{
  "title": "Credit assignment methods for multi-turn agents",
  "question": "Which feedback and cross-action comparisons produce the policy's credit signal?",
  "scope": "Qualitative comparisons of methods, information requirements and task design in the cited paper versions.",
  "keywords": [
    "credit assignment",
    "advantage estimation",
    "multi-turn reinforcement learning",
    "sparse rewards"
  ],
  "dimensions": [
    "Credit unit",
    "Feedback",
    "Comparison mechanism",
    "Additional requirements"
  ],
  "html_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/credit-assignment-methods.html",
  "json_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/credit-assignment-methods.json",
  "csv_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/credit-assignment-methods.csv",
  "bibtex_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/credit-assignment-methods.bib",
  "paper_count": 8,
  "rows": [
    {
      "label": "GiGPO",
      "paper": {
        "name": "GiGPO",
        "version": "2505.10978v1",
        "title": "Group-in-Group Policy Optimization for LLM Agent Training",
        "authors": [
          "Feng, Lang",
          "Xue, Zhenghai",
          "Liu, Tingcong",
          "An, Bo"
        ],
        "publication_date": "2025/05/16",
        "paper_url": "https://arxiv.org/abs/2505.10978v1",
        "html_url": "https://arxiv.org/html/2505.10978v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Episode and action step",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS3.p1",
              "section": "4.3 Group-in-Group Policy Optimization"
            }
          ]
        },
        "Feedback": {
          "value": "Trajectory returns and discounted future rewards",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS1.p1",
              "section": "4.1 Episode Relative Advantages"
            },
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS2.p3",
              "section": "4.2 Step Relative Advantages"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Relative returns among actions at identical anchor states",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS2.p2",
              "section": "4.2 Step Relative Advantages"
            },
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS2.p3",
              "section": "4.2 Step Relative Advantages"
            }
          ]
        },
        "Additional requirements": {
          "value": "Repeated states in collected rollouts; offline grouping",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.10978v1#S4.SS2.p2",
              "section": "4.2 Step Relative Advantages"
            }
          ]
        }
      }
    },
    {
      "label": "Turn-level RL",
      "paper": {
        "name": "Turn-level RL",
        "version": "2505.11821v1",
        "title": "Reinforcing Multi-Turn Reasoning in LLM Agents via Turn-Level Credit Assignment",
        "authors": [
          "Zeng, Siliang",
          "Wei, Quan",
          "Brown, William",
          "Frunza, Oana",
          "Nevmyvaka, Yuriy",
          "Hong, Mingyi"
        ],
        "publication_date": "2025/05/17",
        "paper_url": "https://arxiv.org/abs/2505.11821v1",
        "html_url": "https://arxiv.org/html/2505.11821v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Interaction turn",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.11821v1#S4.SS2.p1",
              "section": "4.2 Proposed Method: Turn-Level Credit Assignment for Multi-Turn LLM Agents"
            }
          ]
        },
        "Feedback": {
          "value": "Turn-level verifiable rewards and final-answer rewards",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.11821v1#S3.SS2.p2",
              "section": "3.2 Reward Design"
            },
            {
              "url": "https://arxiv.org/html/2505.11821v1#S3.SS2.p3",
              "section": "3.2 Reward Design"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Turn-level MDP advantage estimation using intermediate and outcome rewards",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.11821v1#S4.SS2.p1",
              "section": "4.2 Proposed Method: Turn-Level Credit Assignment for Multi-Turn LLM Agents"
            }
          ]
        },
        "Additional requirements": {
          "value": "Task-specific tool-execution and answer-presence verifiers",
          "sources": [
            {
              "url": "https://arxiv.org/html/2505.11821v1#S3.SS2.p2",
              "section": "3.2 Reward Design"
            }
          ]
        }
      }
    },
    {
      "label": "HGPO",
      "paper": {
        "name": "HGPO",
        "version": "2602.22817v1",
        "title": "Hierarchy-of-Groups Policy Optimization for Long-Horizon Agentic Tasks",
        "authors": [
          "He, Shuo",
          "Feng, Lang",
          "Wei, Qi",
          "Cheng, Xin",
          "Feng, Lei",
          "An, Bo"
        ],
        "publication_date": "2026/02/26",
        "paper_url": "https://arxiv.org/abs/2602.22817v1",
        "html_url": "https://arxiv.org/html/2602.22817v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Action step",
          "sources": [
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p4",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            },
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p5",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            }
          ]
        },
        "Feedback": {
          "value": "Sparse terminal rewards and discounted returns",
          "sources": [
            {
              "url": "https://arxiv.org/html/2602.22817v1#S3.p1",
              "section": "3 Preliminaries"
            },
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p4",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Nested groups matching the current state and progressively longer histories",
          "sources": [
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p2",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            },
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p3",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            }
          ]
        },
        "Additional requirements": {
          "value": "State and historical-context matching within existing rollouts",
          "sources": [
            {
              "url": "https://arxiv.org/html/2602.22817v1#S4.SS2.p3",
              "section": "4.2 Hierarchy-of-Groups Policy Optimization"
            }
          ]
        }
      }
    },
    {
      "label": "RTMC",
      "paper": {
        "name": "RTMC",
        "version": "2604.11037v1",
        "title": "RTMC: Step-Level Credit Assignment via Rollout Trees",
        "authors": [
          "Wang, Tao",
          "Zheng, Suhang",
          "Xu, Xiaoxiao"
        ],
        "publication_date": "2026/04/13",
        "paper_url": "https://arxiv.org/abs/2604.11037v1",
        "html_url": "https://arxiv.org/html/2604.11037v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Action segment, shared across its tokens",
          "sources": [
            {
              "url": "https://arxiv.org/html/2604.11037v1#S3.SS1.p1",
              "section": "3.1 From Rollouts to Trees"
            }
          ]
        },
        "Feedback": {
          "value": "Discounted returns from grouped rollouts",
          "sources": [
            {
              "url": "https://arxiv.org/html/2604.11037v1#S3.SS1.p3",
              "section": "3.1 From Rollouts to Trees"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Monte Carlo action and state values in a rollout tree",
          "sources": [
            {
              "url": "https://arxiv.org/html/2604.11037v1#S3.p2",
              "section": "3 Rollout-Tree Monte Carlo Advantage Estimation"
            },
            {
              "url": "https://arxiv.org/html/2604.11037v1#S3.SS1.p3",
              "section": "3.1 From Rollouts to Trees"
            }
          ]
        },
        "Additional requirements": {
          "value": "Task-specific state-action signatures; reuse of collected rollouts",
          "sources": [
            {
              "url": "https://arxiv.org/html/2604.11037v1#S4.p2",
              "section": "4 State-Action Signature Design"
            },
            {
              "url": "https://arxiv.org/html/2604.11037v1#S4.SS1.p1",
              "section": "4.1 Action Signature"
            },
            {
              "url": "https://arxiv.org/html/2604.11037v1#S6.SS1.p2",
              "section": "6.1 Advantages of the Approach"
            }
          ]
        }
      }
    },
    {
      "label": "A3",
      "paper": {
        "name": "A3",
        "version": "2605.08013v1",
        "title": "Learning CLI Agents with Structured Action Credit under Selective Observation",
        "authors": [
          "Su, Haoyang",
          "Wen, Ying"
        ],
        "publication_date": "2026/05/08",
        "paper_url": "https://arxiv.org/abs/2605.08013v1",
        "html_url": "https://arxiv.org/html/2605.08013v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Shell-action turn",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.SS3.p1",
              "section": "3.3 Action Advantage Assignment"
            }
          ]
        },
        "Feedback": {
          "value": "Episode return",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.SS3.p1",
              "section": "3.3 Action Advantage Assignment"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Episode, turn and action-history tree comparisons using shell syntax",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.SS1.p1",
              "section": "3.1 AST measure for CLI agent actions"
            },
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.SS3.p1",
              "section": "3.3 Action Advantage Assignment"
            }
          ]
        },
        "Additional requirements": {
          "value": "Bash parsing and structural action comparisons",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.SS1.p1",
              "section": "3.1 AST measure for CLI agent actions"
            }
          ]
        }
      }
    },
    {
      "label": "TRACE",
      "paper": {
        "name": "TRACE",
        "version": "2607.13988v1",
        "title": "TRACE: Turn-level Reward Assignment via Credit Estimation for Long-Horizon Agents",
        "authors": [
          "Tao, Leitian",
          "Peng, Baolin",
          "Yao, Wenlin",
          "Ge, Tao",
          "Cheng, Hao",
          "Wang, Mike Hang",
          "Gao, Jianfeng",
          "Li, Sharon"
        ],
        "publication_date": "2026/07/15",
        "paper_url": "https://arxiv.org/abs/2607.13988v1",
        "html_url": "https://arxiv.org/html/2607.13988v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Tool-interaction turn; credit shared across its assistant tokens",
          "sources": [
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS1.p1",
              "section": "3.1 Tool-Boundary States"
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS3.SSS0.Px3.p1",
              "section": "Joint optimization with outcome reward."
            }
          ]
        },
        "Feedback": {
          "value": "Final-answer verifier and prefix-conditioned gold-answer likelihoods",
          "sources": [
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS1.p1",
              "section": "3.1 Tool-Boundary States"
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS2.SSS0.Px1.p1",
              "section": "Reference answer score."
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Within-trajectory TD differences of log-ratio answer-likelihood values, with K-step propagation and outcome anchoring",
          "sources": [
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS2.SSS0.Px2.p1",
              "section": "Log-ratio state value."
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS3.SSS0.Px1.p1",
              "section": "One-step TD action credit."
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS3.SSS0.Px2.p1",
              "section": "$K$ -step turn credit."
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS3.SSS0.Px3.p1",
              "section": "Joint optimization with outcome reward."
            }
          ]
        },
        "Additional requirements": {
          "value": "Gold answers and a frozen copy of the initial policy; evaluated on long-horizon search with compact answers",
          "sources": [
            {
              "url": "https://arxiv.org/html/2607.13988v1#S3.SS2.SSS0.Px1.p1",
              "section": "Reference answer score."
            },
            {
              "url": "https://arxiv.org/html/2607.13988v1#S6.p1",
              "section": "6 Limitations"
            }
          ]
        }
      }
    },
    {
      "label": "TCPO",
      "paper": {
        "name": "TCPO",
        "version": "2608.01667v1",
        "title": "TCPO: Turn-Level Credit Policy Optimization",
        "authors": [
          "Liao, Sicong",
          "Chen, Zhi",
          "Tang, Yaohua"
        ],
        "publication_date": "2026/08/03",
        "paper_url": "https://arxiv.org/abs/2608.01667v1",
        "html_url": "https://arxiv.org/html/2608.01667v1"
      },
      "properties": {
        "Credit unit": {
          "value": "Response turn, shared across its tokens",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.01667v1#S3.SS0.SSS1.Px3.p3",
              "section": "Turn-Level Advantage and Token Optimization"
            }
          ]
        },
        "Feedback": {
          "value": "Verifier score after every turn",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.01667v1#S3.SS0.SSS0.Px1.p1",
              "section": "Multi-Turn Verifier Setting"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Retrospective progress, future-best quality and fixed-history alternatives",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.01667v1#S3.p1",
              "section": "3 Method"
            }
          ]
        },
        "Additional requirements": {
          "value": "Selected alternative responses and extra verifier calls; state restoration for AppWorld",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.01667v1#S3.SS0.SSS1.Px2.p3",
              "section": "Counterfactual Estimation and Budgeted Refinement"
            }
          ]
        }
      }
    },
    {
      "label": "IAPO",
      "paper": {
        "name": "IAPO",
        "version": "2608.24588v2",
        "title": "IAPO: Influence-Aware Policy Optimization for Credit Assignment in Multi-Turn Service Agents",
        "authors": [
          "Ren, Bo",
          "Mao, Yirong",
          "Yang, Yi",
          "Que, Wenhui"
        ],
        "publication_date": "2026/08/25",
        "paper_url": "https://arxiv.org/abs/2608.24588v2",
        "html_url": "https://arxiv.org/html/2608.24588v2"
      },
      "properties": {
        "Credit unit": {
          "value": "Trainable assistant action, shared across its tokens",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx4.p1",
              "section": "Method"
            }
          ]
        },
        "Feedback": {
          "value": "Trajectory advantage with observed support and error dependencies",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx4.p1",
              "section": "Method"
            },
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx4.SSx1.p1",
              "section": "Influence-Dependency Graph"
            }
          ]
        },
        "Comparison mechanism": {
          "value": "Positive, bounded routing weights over a within-rollout dependency graph",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx4.p1",
              "section": "Method"
            }
          ]
        },
        "Additional requirements": {
          "value": "Frozen dependency annotator; experiments use Qwen3-32B",
          "sources": [
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx4.SSx3.SSS0.Px3.p1",
              "section": "Routed advantage."
            },
            {
              "url": "https://arxiv.org/html/2608.24588v2#Sx5.SSx1.p3",
              "section": "Experimental Setup"
            }
          ]
        }
      }
    }
  ]
}