{
  "title": "Terminal and software agent benchmark comparison",
  "question": "How do task origin, interaction interface and success criteria differ?",
  "scope": "Qualitative comparisons of methods, information requirements and task design in the cited paper versions.",
  "keywords": [
    "CLI agent benchmark",
    "terminal benchmark",
    "execution feedback",
    "software agent evaluation",
    "ShellOps"
  ],
  "dimensions": [
    "Task source",
    "Interaction interface",
    "Success criteria",
    "Evaluation scope"
  ],
  "html_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/terminal-benchmark-design.html",
  "json_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/terminal-benchmark-design.json",
  "csv_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/terminal-benchmark-design.csv",
  "bibtex_url": "https://hoyant-su-agentic-rl.hf.space/topics/comparisons/terminal-benchmark-design.bib",
  "paper_count": 6,
  "rows": [
    {
      "label": "InterCode",
      "paper": {
        "name": "InterCode",
        "version": "2306.14898v1",
        "title": "InterCode: Standardizing and Benchmarking Interactive Coding with Execution Feedback",
        "authors": [
          "Yang, John",
          "Prabhakar, Akshara",
          "Narasimhan, Karthik",
          "Yao, Shunyu"
        ],
        "publication_date": "2023/06/26",
        "paper_url": "https://arxiv.org/abs/2306.14898v1",
        "html_url": "https://arxiv.org/html/2306.14898v1"
      },
      "properties": {
        "Task source": {
          "value": "NL2Bash and Spider tasks grounded in executable environments",
          "sources": [
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p3",
              "section": "3.3 Implementations"
            },
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p6",
              "section": "3.3 Implementations"
            }
          ]
        },
        "Interaction interface": {
          "value": "Bash and SQL actions with execution observations",
          "sources": [
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p1",
              "section": "3.3 Implementations"
            },
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p2",
              "section": "3.3 Implementations"
            },
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p5",
              "section": "3.3 Implementations"
            }
          ]
        },
        "Success criteria": {
          "value": "Execution output and filesystem changes, or ordered database results",
          "sources": [
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p4",
              "section": "3.3 Implementations"
            },
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p7",
              "section": "3.3 Implementations"
            }
          ]
        },
        "Evaluation scope": {
          "value": "200 Bash tasks and 1,034 SQL tasks in the cited version",
          "sources": [
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p3",
              "section": "3.3 Implementations"
            },
            {
              "url": "https://arxiv.org/html/2306.14898v1#S3.SS3.p6",
              "section": "3.3 Implementations"
            }
          ]
        }
      }
    },
    {
      "label": "SWE-bench",
      "paper": {
        "name": "SWE-bench",
        "version": "2310.06770v1",
        "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
        "authors": [
          "Jimenez, Carlos E.",
          "Yang, John",
          "Wettig, Alexander",
          "Yao, Shunyu",
          "Pei, Kexin",
          "Press, Ofir",
          "Narasimhan, Karthik"
        ],
        "publication_date": "2023/10/10",
        "paper_url": "https://arxiv.org/abs/2310.06770v1",
        "html_url": "https://arxiv.org/html/2310.06770v1"
      },
      "properties": {
        "Task source": {
          "value": "Repository issues paired with resolving pull requests",
          "sources": [
            {
              "url": "https://arxiv.org/html/2310.06770v1#S2.p1",
              "section": "2 SWE-bench"
            }
          ]
        },
        "Interaction interface": {
          "value": "Issue description and codebase as input; code patch as output",
          "sources": [
            {
              "url": "https://arxiv.org/html/2310.06770v1#S2.SS2.p1",
              "section": "2.2 Task Formulation"
            }
          ]
        },
        "Success criteria": {
          "value": "Repository tests associated with the issue",
          "sources": [
            {
              "url": "https://arxiv.org/html/2310.06770v1#S2.p1",
              "section": "2 SWE-bench"
            }
          ]
        },
        "Evaluation scope": {
          "value": "Software issue resolution",
          "sources": [
            {
              "url": "https://arxiv.org/html/2310.06770v1#S2.p1",
              "section": "2 SWE-bench"
            }
          ]
        }
      }
    },
    {
      "label": "Terminal-Bench 2.0",
      "paper": {
        "name": "Terminal-Bench",
        "version": "2601.11868v1",
        "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces",
        "authors": [
          "Merrill, Mike A.",
          "Shaw, Alexander G.",
          "Carlini, Nicholas",
          "Li, Boxuan",
          "Raj, Harsh",
          "Bercovich, Ivan",
          "Shi, Lin",
          "Shin, Jeong Yeon",
          "Walshe, Thomas",
          "Buchanan, E. Kelly",
          "Shen, Junhong",
          "Ye, Guanghao",
          "Lin, Haowei",
          "Poulos, Jason",
          "Wang, Maoyu",
          "Nezhurina, Marianna",
          "Jitsev, Jenia",
          "Lu, Di",
          "Mastromichalakis, Orfeas Menis",
          "Xu, Zhiwei",
          "Chen, Zizhao",
          "Liu, Yue",
          "Zhang, Robert",
          "Chen, Leon Liangyu",
          "Kashyap, Anurag",
          "Uslu, Jan-Lucas",
          "Li, Jeffrey",
          "Wu, Jianbo",
          "Yan, Minghao",
          "Bian, Song",
          "Sharma, Vedang",
          "Sun, Ke",
          "Dillmann, Steven",
          "Anand, Akshay",
          "Lanpouthakoun, Andrew",
          "Koopah, Bardia",
          "Hu, Changran",
          "Guha, Etash",
          "Dreiman, Gabriel H. S.",
          "Zhu, Jiacheng",
          "Krauth, Karl",
          "Zhong, Li",
          "Muennighoff, Niklas",
          "Amanfu, Robert",
          "Tan, Shangyin",
          "Pimpalgaonkar, Shreyas",
          "Aggarwal, Tushar",
          "Lin, Xiangning",
          "Lan, Xin",
          "Zhao, Xuandong",
          "Liang, Yiqing",
          "Wang, Yuanli",
          "Wang, Zilong",
          "Zhou, Changzhi",
          "Heineman, David",
          "Liu, Hange",
          "Trivedi, Harsh",
          "Yang, John",
          "Lin, Junhong",
          "Shetty, Manish",
          "Yang, Michael",
          "Omi, Nabil",
          "Raoof, Negin",
          "Li, Shanda",
          "Zhuo, Terry Yue",
          "Lin, Wuwei",
          "Dai, Yiwei",
          "Wang, Yuxin",
          "Chai, Wenhao",
          "Zhou, Shang",
          "Wahdany, Dariush",
          "She, Ziyu",
          "Hu, Jiaming",
          "Dong, Zhikang",
          "Zhu, Yuxuan",
          "Cui, Sasha",
          "Saiyed, Ahson",
          "Kolbeinsson, Arinbjörn",
          "Hu, Jesse",
          "Rytting, Christopher Michael",
          "Marten, Ryan",
          "Wang, Yixin",
          "Dimakis, Alex",
          "Konwinski, Andy",
          "Schmidt, Ludwig"
        ],
        "publication_date": "2026/01/17",
        "paper_url": "https://arxiv.org/abs/2601.11868v1",
        "html_url": "https://arxiv.org/html/2601.11868v1"
      },
      "properties": {
        "Task source": {
          "value": "Community-authored tasks selected through human review",
          "sources": [
            {
              "url": "https://arxiv.org/html/2601.11868v1#S2.SS2.p1",
              "section": "2.2 Dataset Construction"
            }
          ]
        },
        "Interaction interface": {
          "value": "Container interaction through agent tools; Terminus 2 uses Bash",
          "sources": [
            {
              "url": "https://arxiv.org/html/2601.11868v1#S3.SS1.p1",
              "section": "3.1 Terminus 2"
            },
            {
              "url": "https://arxiv.org/html/2601.11868v1#S3.SS1.p2",
              "section": "3.1 Terminus 2"
            }
          ]
        },
        "Success criteria": {
          "value": "Tests of the final container state against task requirements",
          "sources": [
            {
              "url": "https://arxiv.org/html/2601.11868v1#S2.SS1.p1",
              "section": "2.1 Task Formulation"
            }
          ]
        },
        "Evaluation scope": {
          "value": "89 selected terminal tasks, distributed through Harbor",
          "sources": [
            {
              "url": "https://arxiv.org/html/2601.11868v1#S2.SS2.p1",
              "section": "2.2 Dataset Construction"
            },
            {
              "url": "https://arxiv.org/html/2601.11868v1#S3.SS4.p1",
              "section": "3.4 Harbor"
            }
          ]
        }
      }
    },
    {
      "label": "ShellOps and ShellOps-Pro",
      "paper": {
        "name": "A3",
        "version": "2605.08013v1",
        "title": "Learning CLI Agents with Structured Action Credit under Selective Observation",
        "authors": [
          "Su, Haoyang",
          "Wen, Ying"
        ],
        "publication_date": "2026/05/08",
        "paper_url": "https://arxiv.org/abs/2605.08013v1",
        "html_url": "https://arxiv.org/html/2605.08013v1"
      },
      "properties": {
        "Task source": {
          "value": "Shell-driven tasks with initial and optional target file trees",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S4.SS1.p1",
              "section": "4.1 Dataset Usage and Construction"
            },
            {
              "url": "https://arxiv.org/html/2605.08013v1#S4.SS1.p2",
              "section": "4.1 Dataset Usage and Construction"
            }
          ]
        },
        "Interaction interface": {
          "value": "Shell actions in filesystem workspaces",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S3.p1",
              "section": "3 Method"
            }
          ]
        },
        "Success criteria": {
          "value": "Programmatic rewards over executed outputs and workspace state",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S4.SS1.p1",
              "section": "4.1 Dataset Usage and Construction"
            }
          ]
        },
        "Evaluation scope": {
          "value": "1,624 standard tasks and 150 harder out-of-distribution tasks",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.08013v1#S4.SS1.p2",
              "section": "4.1 Dataset Usage and Construction"
            }
          ]
        }
      }
    },
    {
      "label": "TerminalWorld",
      "paper": {
        "name": "TerminalWorld",
        "version": "2605.22535v1",
        "title": "TerminalWorld: Benchmarking Agents on Real-World Terminal Tasks",
        "authors": [
          "Chu, Zhaoyang",
          "Hu, Jiarui",
          "Jiang, Xingyu",
          "Zou, Pengyu",
          "Li, Han",
          "Peng, Chao",
          "O'Hearn, Peter",
          "Barr, Earl T.",
          "Harman, Mark",
          "Sarro, Federica",
          "Ye, He"
        ],
        "publication_date": "2026/05/21",
        "paper_url": "https://arxiv.org/abs/2605.22535v1",
        "html_url": "https://arxiv.org/html/2605.22535v1"
      },
      "properties": {
        "Task source": {
          "value": "Tasks reconstructed from recorded terminal workflows",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.22535v1#S3.SS3.p1",
              "section": "3.3 Reproducing Executable Environments"
            }
          ]
        },
        "Interaction interface": {
          "value": "Reproduced Docker environments, including multi-service setups",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.22535v1#S3.SS3.p2",
              "section": "3.3 Reproducing Executable Environments"
            }
          ]
        },
        "Success criteria": {
          "value": "Final-state tests checked with complete, empty and partial solutions",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.22535v1#S3.SS4.p2",
              "section": "3.4 Generating Test Suites"
            },
            {
              "url": "https://arxiv.org/html/2605.22535v1#S3.SS4.p4",
              "section": "3.4 Generating Test Suites"
            }
          ]
        },
        "Evaluation scope": {
          "value": "1,530 validated tasks with a separately reviewed Verified subset",
          "sources": [
            {
              "url": "https://arxiv.org/html/2605.22535v1#S3.SS4.p5",
              "section": "3.4 Generating Test Suites"
            },
            {
              "url": "https://arxiv.org/html/2605.22535v1#S4.SS2.p1",
              "section": "4.2 The Verified Subset"
            }
          ]
        }
      }
    },
    {
      "label": "TUA-Bench",
      "paper": {
        "name": "TUA-Bench",
        "version": "2606.28480v1",
        "title": "TUA-Bench: A Benchmark for General-Purpose Terminal-Use Agents",
        "authors": [
          "Chen, Shoufa",
          "Wang, Luyuan",
          "Yang, Xuan",
          "Liu, Zhiheng",
          "Cong, Yuren",
          "Ji, Yuanfeng",
          "Zhou, Feiyan",
          "Zhang, Xiaohui",
          "Yang, Fanny",
          "Zeng, Belinda"
        ],
        "publication_date": "2026/06/26",
        "paper_url": "https://arxiv.org/abs/2606.28480v1",
        "html_url": "https://arxiv.org/html/2606.28480v1"
      },
      "properties": {
        "Task source": {
          "value": "Everyday tasks adapted from OSWorld and expert-designed scientific workflows",
          "sources": [
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS2.SSS1.p1",
              "section": "3.2.1 Everyday Digital Tasks"
            },
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS2.SSS2.p1",
              "section": "3.2.2 Professional Scientific Tasks"
            }
          ]
        },
        "Interaction interface": {
          "value": "Terminal interaction in resettable Linux containers using Harbor",
          "sources": [
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS1.p1",
              "section": "3.1 Task Execution Environment"
            },
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS1.p2",
              "section": "3.1 Task Execution Environment"
            }
          ]
        },
        "Success criteria": {
          "value": "Task-specific verifiers; professional-task design permits programmatic or model judging",
          "sources": [
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS1.p3",
              "section": "3.1 Task Execution Environment"
            },
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS2.SSS2.p1",
              "section": "3.2.2 Professional Scientific Tasks"
            }
          ]
        },
        "Evaluation scope": {
          "value": "120 tasks across five general-purpose computer-use families",
          "sources": [
            {
              "url": "https://arxiv.org/html/2606.28480v1#S3.SS2.SSS3.p1",
              "section": "3.2.3 Task Statistics and Features"
            }
          ]
        }
      }
    }
  ]
}