SamuelChien821/devopsbench-100
DevOpsBench-100 DevOpsBench-100 is a synthetic long-horizon software-engineering / SRE agent benchmark: 100 tasks over one executable world ("NovaCart", a mid-size e-commerce SaaS) with 72 SQLite tables, 1451 seeded rows, a 38-file monorepo with 417 commits, and 97 MCP tools spanning a first-party engineering stack (tickets, PRs, CI, deployments, canaries, migrations, feature flags, metrics, alerts, incidents, chat, knowledge base) plus deliberately disagreeing vendor-shaped… See the full description on the dataset page: https://huggingface.co/datasets/SamuelChien821/devopsbench-100.
02.5k
1[2 {3 "description": "List all services with team, tier, kind, on-call engineer, repo HEAD version and deployed versions.",4 "json_schema": {5 "description": "List all services with team, tier, kind, on-call engineer, repo HEAD version and deployed versions.",6 "name": "list_services",7 "parameters": {8 "properties": {9 "team": {10 "description": "team",11 "type": "string"12 },13 "tier": {14 "description": "tier",15 "type": "integer"16 }17 },18 "required": [],19 "type": "object"20 }21 },22 "name": "list_services",23 "parameters": [24 {25 "default": "None",26 "description": "team",27 "name": "team",28 "required": false,29 "type": "str"30 },31 {32 "default": "None",33 "description": "tier",34 "name": "tier",35 "required": false,36 "type": "int"37 }38 ]39 },40 {41 "description": "Full detail for one service: metadata, deployed config, modules, endpoints, dependencies, current metrics.",42 "json_schema": {43 "description": "Full detail for one service: metadata, deployed config, modules, endpoints, dependencies, current metrics.",44 "name": "get_service",45 "parameters": {46 "properties": {47 "service": {48 "description": "service",49 "type": "string"50 }51 },52 "required": [53 "service"54 ],55 "type": "object"56 }57 },58 "name": "get_service",59 "parameters": [60 {61 "default": null,62 "description": "service",63 "name": "service",64 "required": true,65 "type": "str"66 }67 ]68 },69 {70 "description": "List infrastructure components of the application stack (databases, caches, queues, object stores, CDN).",71 "json_schema": {72 "description": "List infrastructure components of the application stack (databases, caches, queues, object stores, CDN).",73 "name": "list_infra",74 "parameters": {75 "properties": {},76 "required": [],77 "type": "object"78 }79 },80 "name": "list_infra",81 "parameters": []82 },83 {84 "description": "List monorepo files, optionally filtered by service or path substring.",85 "json_schema": {86 "description": "List monorepo files, optionally filtered by service or path substring.",87 "name": "list_files",88 "parameters": {89 "properties": {90 "path_contains": {91 "description": "path_contains",92 "type": "string"93 },94 "service": {95 "description": "service",96 "type": "string"97 }98 },99 "required": [],100 "type": "object"101 }102 },103 "name": "list_files",104 "parameters": [105 {106 "default": "None",107 "description": "service",108 "name": "service",109 "required": false,110 "type": "str"111 },112 {113 "default": "None",114 "description": "path_contains",115 "name": "path_contains",116 "required": false,117 "type": "str"118 }119 ]120 },121 {122 "description": "Read a monorepo source file. Returns its full current content.",123 "json_schema": {124 "description": "Read a monorepo source file. Returns its full current content.",125 "name": "read_file",126 "parameters": {127 "properties": {128 "path": {129 "description": "path",130 "type": "string"131 }132 },133 "required": [134 "path"135 ],136 "type": "object"137 }138 },139 "name": "read_file",140 "parameters": [141 {142 "default": null,143 "description": "path",144 "name": "path",145 "required": true,146 "type": "str"147 }148 ]149 },150 {151 "description": "Search monorepo file contents for a substring; returns matching files with matching line numbers.",152 "json_schema": {153 "description": "Search monorepo file contents for a substring; returns matching files with matching line numbers.",154 "name": "search_code",155 "parameters": {156 "properties": {157 "limit": {158 "description": "limit",159 "type": "integer"160 },161 "query": {162 "description": "query",163 "type": "string"164 },165 "service": {166 "description": "service",167 "type": "string"168 }169 },170 "required": [171 "query"172 ],173 "type": "object"174 }175 },176 "name": "search_code",177 "parameters": [178 {179 "default": null,180 "description": "query",181 "name": "query",182 "required": true,183 "type": "str"184 },185 {186 "default": "None",187 "description": "service",188 "name": "service",189 "required": false,190 "type": "str"191 },192 {193 "default": "20",194 "description": "limit",195 "name": "limit",196 "required": false,197 "type": "int"198 }199 ]200 },201 {202 "description": "Browse monorepo commit history (most recent first).",203 "json_schema": {204 "description": "Browse monorepo commit history (most recent first).",205 "name": "list_commits",206 "parameters": {207 "properties": {208 "limit": {209 "description": "limit",210 "type": "integer"211 },212 "path": {213 "description": "only commits touching this file path",214 "type": "string"215 },216 "query": {217 "description": "substring of the commit message",218 "type": "string"219 },220 "service": {221 "description": "service",222 "type": "string"223 }224 },225 "required": [],226 "type": "object"227 }228 },229 "name": "list_commits",230 "parameters": [231 {232 "default": "None",233 "description": "service",234 "name": "service",235 "required": false,236 "type": "str"237 },238 {239 "default": "None",240 "description": "substring of the commit message",241 "name": "query",242 "required": false,243 "type": "str"244 },245 {246 "default": "None",247 "description": "only commits touching this file path",248 "name": "path",249 "required": false,250 "type": "str"251 },252 {253 "default": "20",254 "description": "limit",255 "name": "limit",256 "required": false,257 "type": "int"258 }259 ]260 },261 {262 "description": "Search the engineering knowledge base (runbooks, policies, design docs, ADRs, postmortems, API specs).",263 "json_schema": {264 "description": "Search the engineering knowledge base (runbooks, policies, design docs, ADRs, postmortems, API specs).",265 "name": "search_docs",266 "parameters": {267 "properties": {268 "kind": {269 "description": "runbook|policy|design_doc|adr|postmortem|api_spec|onboarding",270 "type": "string"271 },272 "limit": {273 "description": "limit",274 "type": "integer"275 },276 "query": {277 "description": "query",278 "type": "string"279 },280 "service": {281 "description": "service",282 "type": "string"283 }284 },285 "required": [],286 "type": "object"287 }288 },289 "name": "search_docs",290 "parameters": [291 {292 "default": "",293 "description": "query",294 "name": "query",295 "required": false,296 "type": "str"297 },298 {299 "default": "None",300 "description": "runbook|policy|design_doc|adr|postmortem|api_spec|onboarding",301 "name": "kind",302 "required": false,303 "type": "str"304 },305 {306 "default": "None",307 "description": "service",308 "name": "service",309 "required": false,310 "type": "str"311 },312 {313 "default": "10",314 "description": "limit",315 "name": "limit",316 "required": false,317 "type": "int"318 }319 ]320 },321 {322 "description": "Read one knowledge-base document in full by doc_id (or exact title).",323 "json_schema": {324 "description": "Read one knowledge-base document in full by doc_id (or exact title).",325 "name": "get_document",326 "parameters": {327 "properties": {328 "doc_id": {329 "description": "doc_id",330 "type": "integer"331 },332 "title": {333 "description": "title",334 "type": "string"335 }336 },337 "required": [],338 "type": "object"339 }340 },341 "name": "get_document",342 "parameters": [343 {344 "default": "None",345 "description": "doc_id",346 "name": "doc_id",347 "required": false,348 "type": "int"349 },350 {351 "default": "None",352 "description": "title",353 "name": "title",354 "required": false,355 "type": "str"356 }357 ]358 },359 {360 "description": "List issue-tracker tickets, optionally filtered by status, service, or type.",361 "json_schema": {362 "description": "List issue-tracker tickets, optionally filtered by status, service, or type.",363 "name": "list_tickets",364 "parameters": {365 "properties": {366 "service": {367 "description": "service",368 "type": "string"369 },370 "status": {371 "description": "status",372 "type": "string"373 },374 "ticket_type": {375 "description": "ticket_type",376 "type": "string"377 }378 },379 "required": [],380 "type": "object"381 }382 },383 "name": "list_tickets",384 "parameters": [385 {386 "default": "None",387 "description": "status",388 "name": "status",389 "required": false,390 "type": "str"391 },392 {393 "default": "None",394 "description": "service",395 "name": "service",396 "required": false,397 "type": "str"398 },399 {400 "default": "None",401 "description": "ticket_type",402 "name": "ticket_type",403 "required": false,404 "type": "str"405 }406 ]407 },408 {409 "description": "Fetch one ticket by key (e.g. ENG-2101).",410 "json_schema": {411 "description": "Fetch one ticket by key (e.g. ENG-2101).",412 "name": "get_ticket",413 "parameters": {414 "properties": {415 "key": {416 "description": "key",417 "type": "string"418 }419 },420 "required": [421 "key"422 ],423 "type": "object"424 }425 },426 "name": "get_ticket",427 "parameters": [428 {429 "default": null,430 "description": "key",431 "name": "key",432 "required": true,433 "type": "str"434 }435 ]436 },437 {438 "description": "List pull requests, optionally filtered by service or status.",439 "json_schema": {440 "description": "List pull requests, optionally filtered by service or status.",441 "name": "list_pull_requests",442 "parameters": {443 "properties": {444 "service": {445 "description": "service",446 "type": "string"447 },448 "status": {449 "description": "status",450 "type": "string"451 }452 },453 "required": [],454 "type": "object"455 }456 },457 "name": "list_pull_requests",458 "parameters": [459 {460 "default": "None",461 "description": "service",462 "name": "service",463 "required": false,464 "type": "str"465 },466 {467 "default": "None",468 "description": "status",469 "name": "status",470 "required": false,471 "type": "str"472 }473 ]474 },475 {476 "description": "Fetch a PR with its structured changes and CI history.",477 "json_schema": {478 "description": "Fetch a PR with its structured changes and CI history.",479 "name": "get_pull_request",480 "parameters": {481 "properties": {482 "pr_number": {483 "description": "pr_number",484 "type": "integer"485 }486 },487 "required": [488 "pr_number"489 ],490 "type": "object"491 }492 },493 "name": "get_pull_request",494 "parameters": [495 {496 "default": null,497 "description": "pr_number",498 "name": "pr_number",499 "required": true,500 "type": "int"501 }502 ]503 },504 {505 "description": "List CI runs (most recent first).",506 "json_schema": {507 "description": "List CI runs (most recent first).",508 "name": "list_ci_runs",509 "parameters": {510 "properties": {511 "limit": {512 "description": "limit",513 "type": "integer"514 },515 "pr_number": {516 "description": "pr_number",517 "type": "integer"518 },519 "service": {520 "description": "service",521 "type": "string"522 }523 },524 "required": [],525 "type": "object"526 }527 },528 "name": "list_ci_runs",529 "parameters": [530 {531 "default": "None",532 "description": "service",533 "name": "service",534 "required": false,535 "type": "str"536 },537 {538 "default": "None",539 "description": "pr_number",540 "name": "pr_number",541 "required": false,542 "type": "int"543 },544 {545 "default": "20",546 "description": "limit",547 "name": "limit",548 "required": false,549 "type": "int"550 }551 ]552 },553 {554 "description": "Fetch one CI run with its per-stage results (build, unit, integration, regression).",555 "json_schema": {556 "description": "Fetch one CI run with its per-stage results (build, unit, integration, regression).",557 "name": "get_ci_run",558 "parameters": {559 "properties": {560 "run_id": {561 "description": "run_id",562 "type": "integer"563 }564 },565 "required": [566 "run_id"567 ],568 "type": "object"569 }570 },571 "name": "get_ci_run",572 "parameters": [573 {574 "default": null,575 "description": "run_id",576 "name": "run_id",577 "required": true,578 "type": "int"579 }580 ]581 },582 {583 "description": "List deployments (most recent first).",584 "json_schema": {585 "description": "List deployments (most recent first).",586 "name": "list_deployments",587 "parameters": {588 "properties": {589 "environment": {590 "description": "environment",591 "type": "string"592 },593 "limit": {594 "description": "limit",595 "type": "integer"596 },597 "service": {598 "description": "service",599 "type": "string"600 }601 },602 "required": [],603 "type": "object"604 }605 },606 "name": "list_deployments",607 "parameters": [608 {609 "default": "None",610 "description": "service",611 "name": "service",612 "required": false,613 "type": "str"614 },615 {616 "default": "None",617 "description": "environment",618 "name": "environment",619 "required": false,620 "type": "str"621 },622 {623 "default": "20",624 "description": "limit",625 "name": "limit",626 "required": false,627 "type": "int"628 }629 ]630 },631 {632 "description": "List database migrations and whether they are applied per environment.",633 "json_schema": {634 "description": "List database migrations and whether they are applied per environment.",635 "name": "list_migrations",636 "parameters": {637 "properties": {638 "environment": {639 "description": "environment",640 "type": "string"641 },642 "service": {643 "description": "service",644 "type": "string"645 }646 },647 "required": [],648 "type": "object"649 }650 },651 "name": "list_migrations",652 "parameters": [653 {654 "default": "None",655 "description": "service",656 "name": "service",657 "required": false,658 "type": "str"659 },660 {661 "default": "None",662 "description": "environment",663 "name": "environment",664 "required": false,665 "type": "str"666 }667 ]668 },669 {670 "description": "Read current production service metrics (recomputed continuously from live traffic).",671 "json_schema": {672 "description": "Read current production service metrics (recomputed continuously from live traffic).",673 "name": "query_metrics",674 "parameters": {675 "properties": {676 "metric": {677 "description": "metric",678 "type": "string"679 },680 "service": {681 "description": "service",682 "type": "string"683 }684 },685 "required": [],686 "type": "object"687 }688 },689 "name": "query_metrics",690 "parameters": [691 {692 "default": "None",693 "description": "service",694 "name": "service",695 "required": false,696 "type": "str"697 },698 {699 "default": "None",700 "description": "metric",701 "name": "metric",702 "required": false,703 "type": "str"704 }705 ]706 },707 {708 "description": "Traffic-generator statistics: request rate per route with the current error rate and p99 of the owning service.",709 "json_schema": {710 "description": "Traffic-generator statistics: request rate per route with the current error rate and p99 of the owning service.",711 "name": "get_traffic_stats",712 "parameters": {713 "properties": {714 "service": {715 "description": "service",716 "type": "string"717 }718 },719 "required": [],720 "type": "object"721 }722 },723 "name": "get_traffic_stats",724 "parameters": [725 {726 "default": "None",727 "description": "service",728 "name": "service",729 "required": false,730 "type": "str"731 }732 ]733 },734 {735 "description": "List SLOs with current values and whether each is breaching.",736 "json_schema": {737 "description": "List SLOs with current values and whether each is breaching.",738 "name": "get_slo_status",739 "parameters": {740 "properties": {741 "service": {742 "description": "service",743 "type": "string"744 }745 },746 "required": [],747 "type": "object"748 }749 },750 "name": "get_slo_status",751 "parameters": [752 {753 "default": "None",754 "description": "service",755 "name": "service",756 "required": false,757 "type": "str"758 }759 ]760 },761 {762 "description": "List alarms, optionally filtered by status (firing|acknowledged|resolved) or service.",763 "json_schema": {764 "description": "List alarms, optionally filtered by status (firing|acknowledged|resolved) or service.",765 "name": "list_alerts",766 "parameters": {767 "properties": {768 "service": {769 "description": "service",770 "type": "string"771 },772 "status": {773 "description": "status",774 "type": "string"775 }776 },777 "required": [],778 "type": "object"779 }780 },781 "name": "list_alerts",782 "parameters": [783 {784 "default": "None",785 "description": "status",786 "name": "status",787 "required": false,788 "type": "str"789 },790 {791 "default": "None",792 "description": "service",793 "name": "service",794 "required": false,795 "type": "str"796 }797 ]798 },799 {800 "description": "Error-tracking issues (Sentry-style): grouped exceptions with culprit and event counts.",801 "json_schema": {802 "description": "Error-tracking issues (Sentry-style): grouped exceptions with culprit and event counts.",803 "name": "list_error_events",804 "parameters": {805 "properties": {806 "service": {807 "description": "service",808 "type": "string"809 },810 "status": {811 "description": "unresolved|resolved",812 "type": "string"813 }814 },815 "required": [],816 "type": "object"817 }818 },819 "name": "list_error_events",820 "parameters": [821 {822 "default": "None",823 "description": "service",824 "name": "service",825 "required": false,826 "type": "str"827 },828 {829 "default": "None",830 "description": "unresolved|resolved",831 "name": "status",832 "required": false,833 "type": "str"834 }835 ]836 },837 {838 "description": "Search application logs by substring, service, or level.",839 "json_schema": {840 "description": "Search application logs by substring, service, or level.",841 "name": "search_logs",842 "parameters": {843 "properties": {844 "level": {845 "description": "level",846 "type": "string"847 },848 "limit": {849 "description": "limit",850 "type": "integer"851 },852 "query": {853 "description": "query",854 "type": "string"855 },856 "service": {857 "description": "service",858 "type": "string"859 }860 },861 "required": [],862 "type": "object"863 }864 },865 "name": "search_logs",866 "parameters": [867 {868 "default": "None",869 "description": "service",870 "name": "service",871 "required": false,872 "type": "str"873 },874 {875 "default": "",876 "description": "query",877 "name": "query",878 "required": false,879 "type": "str"880 },881 {882 "default": "None",883 "description": "level",884 "name": "level",885 "required": false,886 "type": "str"887 },888 {889 "default": "20",890 "description": "limit",891 "name": "limit",892 "required": false,893 "type": "int"894 }895 ]896 },897 {898 "description": "List feature flags with per-environment state.",899 "json_schema": {900 "description": "List feature flags with per-environment state.",901 "name": "list_feature_flags",902 "parameters": {903 "properties": {904 "environment": {905 "description": "environment",906 "type": "string"907 },908 "service": {909 "description": "service",910 "type": "string"911 }912 },913 "required": [],914 "type": "object"915 }916 },917 "name": "list_feature_flags",918 "parameters": [919 {920 "default": "None",921 "description": "service",922 "name": "service",923 "required": false,924 "type": "str"925 },926 {927 "default": "None",928 "description": "environment",929 "name": "environment",930 "required": false,931 "type": "str"932 }933 ]934 },935 {936 "description": "List package dependencies: version at repo HEAD and version deployed in production.",937 "json_schema": {938 "description": "List package dependencies: version at repo HEAD and version deployed in production.",939 "name": "list_packages",940 "parameters": {941 "properties": {942 "service": {943 "description": "service",944 "type": "string"945 }946 },947 "required": [],948 "type": "object"949 }950 },951 "name": "list_packages",952 "parameters": [953 {954 "default": "None",955 "description": "service",956 "name": "service",957 "required": false,958 "type": "str"959 }960 ]961 },962 {963 "description": "List security-scanner findings.",964 "json_schema": {965 "description": "List security-scanner findings.",966 "name": "list_vulnerabilities",967 "parameters": {968 "properties": {969 "service": {970 "description": "service",971 "type": "string"972 },973 "status": {974 "description": "status",975 "type": "string"976 }977 },978 "required": [],979 "type": "object"980 }981 },982 "name": "list_vulnerabilities",983 "parameters": [984 {985 "default": "None",986 "description": "status",987 "name": "status",988 "required": false,989 "type": "str"990 },991 {992 "default": "None",993 "description": "service",994 "name": "service",995 "required": false,996 "type": "str"997 }998 ]999 },1000 {1001 "description": "List API endpoints with repo status, production status, and production traffic share.",1002 "json_schema": {1003 "description": "List API endpoints with repo status, production status, and production traffic share.",1004 "name": "list_api_endpoints",1005 "parameters": {1006 "properties": {1007 "service": {1008 "description": "service",1009 "type": "string"1010 }1011 },1012 "required": [],1013 "type": "object"1014 }1015 },1016 "name": "list_api_endpoints",1017 "parameters": [1018 {1019 "default": "None",1020 "description": "service",1021 "name": "service",1022 "required": false,1023 "type": "str"1024 }1025 ]1026 },1027 {1028 "description": "List the test catalog.",1029 "json_schema": {1030 "description": "List the test catalog.",1031 "name": "list_tests",1032 "parameters": {1033 "properties": {1034 "service": {1035 "description": "service",1036 "type": "string"1037 },1038 "status": {1039 "description": "status",1040 "type": "string"1041 }1042 },1043 "required": [],1044 "type": "object"1045 }1046 },1047 "name": "list_tests",1048 "parameters": [1049 {1050 "default": "None",1051 "description": "service",1052 "name": "service",1053 "required": false,1054 "type": "str"1055 },1056 {1057 "default": "None",1058 "description": "status",1059 "name": "status",1060 "required": false,1061 "type": "str"1062 }1063 ]1064 },1065 {1066 "description": "List incidents.",1067 "json_schema": {1068 "description": "List incidents.",1069 "name": "list_incidents",1070 "parameters": {1071 "properties": {1072 "status": {1073 "description": "status",1074 "type": "string"1075 }1076 },1077 "required": [],1078 "type": "object"1079 }1080 },1081 "name": "list_incidents",1082 "parameters": [1083 {1084 "default": "None",1085 "description": "status",1086 "name": "status",1087 "required": false,1088 "type": "str"1089 }1090 ]1091 },1092 {1093 "description": "Read the public system-status page.",1094 "json_schema": {1095 "description": "Read the public system-status page.",1096 "name": "get_status_page",1097 "parameters": {1098 "properties": {1099 "limit": {1100 "description": "limit",1101 "type": "integer"1102 }1103 },1104 "required": [],1105 "type": "object"1106 }1107 },1108 "name": "get_status_page",1109 "parameters": [1110 {1111 "default": "10",1112 "description": "limit",1113 "name": "limit",1114 "required": false,1115 "type": "int"1116 }1117 ]1118 },1119 {1120 "description": "Read chat messages.",1121 "json_schema": {1122 "description": "Read chat messages.",1123 "name": "list_messages",1124 "parameters": {1125 "properties": {1126 "channel": {1127 "description": "channel",1128 "type": "string"1129 },1130 "limit": {1131 "description": "limit",1132 "type": "integer"1133 }1134 },1135 "required": [],1136 "type": "object"1137 }1138 },1139 "name": "list_messages",1140 "parameters": [1141 {1142 "default": "None",1143 "description": "channel",1144 "name": "channel",1145 "required": false,1146 "type": "str"1147 },1148 {1149 "default": "20",1150 "description": "limit",1151 "name": "limit",1152 "required": false,1153 "type": "int"1154 }1155 ]1156 },1157 {1158 "description": "Create a ticket. Returns the generated key.",1159 "json_schema": {1160 "description": "Create a ticket. Returns the generated key.",1161 "name": "create_ticket",1162 "parameters": {1163 "properties": {1164 "description": {1165 "description": "description",1166 "type": "string"1167 },1168 "priority": {1169 "description": "priority",1170 "type": "string"1171 },1172 "service": {1173 "description": "service",1174 "type": "string"1175 },1176 "ticket_type": {1177 "description": "task|bug|feature|security|incident|postmortem",1178 "enum": [1179 "task",1180 "bug",1181 "feature",1182 "security",1183 "incident",1184 "postmortem"1185 ],1186 "type": "string"1187 },1188 "title": {1189 "description": "title",1190 "type": "string"1191 }1192 },1193 "required": [1194 "title"1195 ],1196 "type": "object"1197 }1198 },1199 "name": "create_ticket",1200 "parameters": [