axisrobotics/Test-Dataset
AXIS held-out 20 — a cross-embodiment few-shot adaptation benchmark 20 tasks never seen in pretraining, 20 demonstrations each, LoRA adaptation, rollout evaluation. This release is the SPECIFICATION and the INDEX, not the demonstration data. It is published first and on purpose: everything here is what you need to render the benchmark on your own embodiment, and none of it depends on our video encoding being finished. Status: the task list is CANDIDATES. The learnability gate… See the full description on the dataset page: https://huggingface.co/datasets/axisrobotics/Test-Dataset.
090
1{2 "schema": "axis_heldout_benchmark_v2",3 "name": "AXIS held-out benchmark",4 "SIZE": {5 "active_tasks": 10,6 "conditional_tasks": 10,7 "why": "A benchmark task must be absent from pretraining. Of 124 quality-eligible tasks only 11 are absent from the stage1_5k training ranges and the 11th (813) is unevaluable, so TEN is the ceiling against today's stage-1 models. The other ten are contaminated -- 157 episodes of the 5,000 -- and become usable only if a future stage-1 run excludes the held-out set.",8 "active": [9 809,10 811,11 866,12 868,13 945,14 953,15 966,16 1426,17 1459,18 188919 ],20 "conditional": [21 810,22 815,23 929,24 970,25 1046,26 1252,27 1427,28 1458,29 1746,30 189131 ]32 },33 "DATA_STATUS": {34 "state": "AWAITING RE-RENDER",35 "camera_rig": "current demos use the superseded rig (hand->cam [-0.074,0,0.0292], fovy 51.9); the pretraining corpus uses [0.05,0,0] / fovy 74.8",36 "appearance": "current demos are randomized per demo (TextureModder + LightingModder), so the 20 demos of a task span 20 visual worlds",37 "request": "render_request/ in this repo covers all 20 tasks"38 },39 "protocol": {40 "adaptation_demos_per_task": 20,41 "eval_episodes_per_task": 50,42 "eval_requires": "--dual-sim",43 "policy": "pi05_base + 20-demo LoRA, constant LR after 100-step warmup",44 "gate": "keep a task if 0.20 <= success <= 0.80"45 },46 "unevaluable": {47 "813": "layout generator fails after 50,000 attempts placing 4 objects in a 0.50 m box",48 "866": "declares DirectedRotationChecker, which is NOT in the checker registry; the harness substitutes AlwaysFalseChecker so the task returns 0/50 under any policy"49 },50 "join_on": "(task_id, attempt_id) -- never episode_index",51 "tasks": [52 {53 "task_id": 809,54 "operation": "separate",55 "name": "separate the sandwich from the juice",56 "status": "active",57 "adaptation_demos": 20,58 "mean_reward_final": 0.922959 },60 {61 "task_id": 810,62 "operation": "separate",63 "name": "separate the sandwich from the napkin box",64 "status": "conditional",65 "adaptation_demos": 20,66 "mean_reward_final": 0.92267 },68 {69 "task_id": 811,70 "operation": "reach",71 "name": "reach target pose",72 "status": "active",73 "adaptation_demos": 20,74 "mean_reward_final": 0.93275 },76 {77 "task_id": 815,78 "operation": "place",79 "name": "place with orientation",80 "status": "conditional",81 "adaptation_demos": 20,82 "mean_reward_final": 0.911583 },84 {85 "task_id": 866,86 "operation": "rotate",87 "name": "rotate the computer mouse",88 "status": "active",89 "adaptation_demos": 20,90 "mean_reward_final": 0.902891 },92 {93 "task_id": 868,94 "operation": "pick",95 "name": "pick single rigid object",96 "status": "active",97 "adaptation_demos": 20,98 "mean_reward_final": 0.94299 },100 {101 "task_id": 929,102 "operation": "put_spatial",103 "name": "put the tea cup on the left side of the tea pot",104 "status": "conditional",105 "adaptation_demos": 20,106 "mean_reward_final": 0.8942107 },108 {109 "task_id": 945,110 "operation": "put_into",111 "name": "put both the potato and the pepper in the bowl",112 "status": "active",113 "adaptation_demos": 20,114 "mean_reward_final": 0.922115 },116 {117 "task_id": 953,118 "operation": "put_spatial",119 "name": "put the muffin on the left side of the cup",120 "status": "active",121 "adaptation_demos": 20,122 "mean_reward_final": 0.9151123 },124 {125 "task_id": 966,126 "operation": "put_on",127 "name": "put the eyeglasses case on the tissue box",128 "status": "active",129 "adaptation_demos": 20,130 "mean_reward_final": 0.9231131 },132 {133 "task_id": 970,134 "operation": "rotate",135 "name": "rotate the eyeglasses case",136 "status": "conditional",137 "adaptation_demos": 20,138 "mean_reward_final": 0.8546139 },140 {141 "task_id": 1046,142 "operation": "put_into",143 "name": "put the toy car into the basket",144 "status": "conditional",145 "adaptation_demos": 20,146 "mean_reward_final": 0.9018147 },148 {149 "task_id": 1252,150 "operation": "remove",151 "name": "remove loose lid",152 "status": "conditional",153 "adaptation_demos": 20,154 "mean_reward_final": 0.8896155 },156 {157 "task_id": 1426,158 "operation": "rearrange",159 "name": "rearrange three items",160 "status": "active",161 "adaptation_demos": 20,162 "mean_reward_final": 0.9132163 },164 {165 "task_id": 1427,166 "operation": "rearrange",167 "name": "rearrange three items 2",168 "status": "conditional",169 "adaptation_demos": 20,170 "mean_reward_final": 0.9097171 },172 {173 "task_id": 1458,174 "operation": "insert",175 "name": "insert utensil into holder",176 "status": "conditional",177 "adaptation_demos": 20,178 "mean_reward_final": 0.917179 },180 {181 "task_id": 1459,182 "operation": "insert",183 "name": "fit lid onto container",184 "status": "active",185 "adaptation_demos": 20,186 "mean_reward_final": 0.904187 },188 {189 "task_id": 1746,190 "operation": "put_on",191 "name": "put the cube block on the poker",192 "status": "conditional",193 "adaptation_demos": 20,194 "mean_reward_final": 0.914195 },196 {197 "task_id": 1889,198 "operation": "put_beside",199 "name": "put the kiwi beside the banana",200 "status": "active",201 "adaptation_demos": 20,202 "mean_reward_final": 0.917203 },204 {205 "task_id": 1891,206 "operation": "put_beside",207 "name": "put the orange beside the banana",208 "status": "conditional",209 "adaptation_demos": 20,210 "mean_reward_final": 0.9076211 }212 ]213}