TaxonomyProject/SimulationMetadata
PercepTax — Simulation Metadata Supporting metadata for the PercepTax benchmark. This repository holds the object taxonomy, object lists / placement rules, and per-scene 3D annotations used to render and annotate the simulated split of PercepTax. It is a collection of JSON/CSV/PNG files (not a load_dataset-style tabular dataset). Contents taxonomy/ # the physical-property taxonomy ├── taxonomy.json # full taxonomy… See the full description on the dataset page: https://huggingface.co/datasets/TaxonomyProject/SimulationMetadata.
12.3k
1{2 "scene_name": "Bakery",3 "view_id": "l024_r005",4 "image_path": "/home/xwang378/scratch/2025/Taxonomy/Data/simulationImage/jiawei/Bakery/l024_r005/lit.png",5 "camera": {6 "location": [7 255,8 -56,9 12610 ],11 "rotation": [12 0,13 -59.999999999999986,14 015 ],16 "fov": 90,17 "c2w": [18 [19 0.5000000000000002,20 -0.8660254037844384,21 0,22 023 ],24 [25 0.8660254037844384,26 0.5000000000000002,27 0,28 029 ],30 [31 0,32 0,33 1,34 035 ],36 [37 255,38 -56,39 126,40 141 ]42 ],43 "width": 1920,44 "height": 1080,45 "fxfycxcy": [46 960.0000000000001,47 960.0000000000002,48 960.0,49 540.050 ]51 },52 "background": {53 "window": [54 {55 "object_id": "SM_wall_01_281",56 "bbox_2d": [57 217,58 0,59 1919,60 107961 ],62 "bbox_3d": {63 "aabb": {64 "center": [65 0,66 0,67 199.2298889160156268 ],69 "extent": [70 448.35015869140625,71 456.7044982910156,72 200.5044555664062573 ]74 },75 "obb": {76 "center": [77 0,78 0,79 199.2298889160156280 ],81 "extent": [82 448.35015869140625,83 456.7044982910156,84 200.5044555664062585 ],86 "rotation": [87 0,88 0,89 090 ]91 },92 "location": [93 0,94 0,95 096 ],97 "rotation": [98 0,99 0,100 0101 ],102 "scale": [103 1,104 1,105 1106 ]107 },108 "color": [109 0,110 255,111 127112 ]113 }114 ],115 "ground": [116 {117 "object_id": "SM_ground_9",118 "bbox_2d": [119 357,120 882,121 1168,122 1079123 ],124 "bbox_3d": {125 "aabb": {126 "center": [127 0,128 0.704864501953125,129 11.041000366210938130 ],131 "extent": [132 447.2875671386719,133 455.69342041015625,134 8.675445556640625135 ]136 },137 "obb": {138 "center": [139 0,140 0.704864501953125,141 11.041000366210938142 ],143 "extent": [144 447.2875671386719,145 455.69342041015625,146 8.675445556640625147 ],148 "rotation": [149 0,150 0,151 0152 ]153 },154 "location": [155 0,156 -0.000762939453125,157 11.041000366210938158 ],159 "rotation": [160 0,161 0,162 0163 ],164 "scale": [165 1,166 1,167 1168 ]169 },170 "color": [171 191,172 0,173 255174 ]175 }176 ]177 },178 "foreground": {179 "grill": [180 {181 "object_id": "SM_grill2",182 "bbox_2d": [183 1583,184 0,185 1919,186 1079187 ],188 "bbox_3d": {189 "aabb": {190 "center": [191 385.2553100585887,192 -75.62591934204102,193 108.73949432373047194 ],195 "extent": [196 28.420366367019028,197 35.869838411538495,198 90.03331756591797199 ]200 },201 "obb": {202 "center": [203 385.2553100585887,204 -75.62591934204102,205 108.73949432373047206 ],207 "extent": [208 28.420366367019028,209 35.869838411538495,210 90.03331756591797211 ],212 "rotation": [213 0,214 90.00007629394531,215 0216 ]217 },218 "location": [219 385.25531005859375,220 -75.62592315673828,221 20222 ],223 "rotation": [224 0,225 90.00007629394531,226 0227 ],228 "scale": [229 1,230 1,231 1232 ]233 },234 "color": [235 127,236 0,237 255238 ]239 }240 ],241 "offeecart": [242 {243 "object_id": "SM_offeecart_343",244 "bbox_2d": [245 870,246 318,247 1582,248 1079249 ],250 "bbox_3d": {251 "aabb": {252 "center": [253 381.5960205253525,254 -207.50344660440354,255 95.30928149775127256 ],257 "extent": [258 46.82246629397008,259 94.56324563513309,260 74.93345305891899261 ]262 },263 "obb": {264 "center": [265 381.5960205253525,266 -207.50344660440354,267 95.30928149775127268 ],269 "extent": [270 46.82246629397008,271 94.56324563513309,272 74.93345305891899273 ],274 "rotation": [275 0,276 90.00011444091797,277 0278 ]279 },280 "location": [281 379.77606201171875,282 -180.17184448242188,283 19.999954223632812284 ],285 "rotation": [286 0,287 90.00011444091797,288 0289 ],290 "scale": [291 0.8725000023841858,292 0.8725000023841858,293 0.8725000023841858294 ]295 },296 "color": [297 0,298 0,299 63300 ]301 }302 ],303 "steel": [304 {305 "object_id": "SM_steel_01_349",306 "bbox_2d": [307 237,308 276,309 432,310 947311 ],312 "bbox_3d": {313 "aabb": {314 "center": [315 240.58107749462158,316 -385.25631504767193,317 105.9680404663086318 ],319 "extent": [320 22.462900420585342,321 45.113007901773415,322 86.05392456054688323 ]324 },325 "obb": {326 "center": [327 240.58107749462158,328 -385.25631504767193,329 105.9680404663086330 ],331 "extent": [332 22.462900420585342,333 45.113007901773415,334 86.05392456054688335 ],336 "rotation": [337 0,338 -87.00657653808594,339 0340 ]341 },342 "location": [343 240.58099365234375,344 -385.254638671875,345 19.999984741210938346 ],347 "rotation": [348 0,349 -87.00657653808594,350 0351 ],352 "scale": [353 1,354 1,355 1356 ]357 },358 "color": [359 0,360 127,361 63362 ]363 }364 ],365 "fireextinguisher": [366 {367 "object_id": "SM_fireextinguisher3",368 "bbox_2d": [369 151,370 869,371 391,372 1079373 ],374 "bbox_3d": {375 "aabb": {376 "center": [377 245.4195556640625,378 -162.7714385986328,379 55.871395111083984380 ],381 "extent": [382 15.709892233551386,383 16.09848102098374,384 37.336402893066406385 ]386 },387 "obb": {388 "center": [389 245.4195556640625,390 -162.7714385986328,391 55.871395111083984392 ],393 "extent": [394 15.709892233551386,395 16.09848102098374,396 37.336402893066406397 ],398 "rotation": [399 0,400 -50.00006484985351,401 0402 ]403 },404 "location": [405 245.4195556640625,406 -162.7714385986328,407 55.87139892578125408 ],409 "rotation": [410 0,411 -50.00006484985351,412 0413 ],414 "scale": [415 1,416 1,417 1418 ]419 },420 "color": [421 255,422 255,423 63424 ]425 }426 ],427 "water dispenser": [428 {429 "object_id": "SM_wdispenser_366",430 "bbox_2d": [431 881,432 397,433 1052,434 975435 ],436 "bbox_3d": {437 "aabb": {438 "center": [439 383.06593916718987,440 -271.71814150316305,441 89.93828582763672442 ],443 "extent": [444 23.43082607754019,445 26.610365686922364,446 72.6376953125447 ]448 },449 "obb": {450 "center": [451 383.06593916718987,452 -271.71814150316305,453 89.93828582763672454 ],455 "extent": [456 23.43082607754019,457 26.610365686922364,458 72.6376953125459 ],460 "rotation": [461 0,462 -10.000549316406248,463 0464 ]465 },466 "location": [467 383.06591796875,468 -271.71826171875,469 89.93828582763672470 ],471 "rotation": [472 0,473 -10.000549316406248,474 0475 ],476 "scale": [477 1,478 1,479 1480 ]481 },482 "color": [483 127,484 0,485 63486 ]487 }488 ],489 "camera": [490 {491 "object_id": "SM_camera2",492 "bbox_2d": [493 438,494 0,495 442,496 1497 ],498 "bbox_3d": {499 "aabb": {500 "center": [501 264.1894836425921,502 -407.4629306793213,503 327.66476345062256504 ],505 "extent": [506 7.672381458117172,507 27.3011983454835,508 14.869946479797363509 ]510 },511 "obb": {512 "center": [513 264.1894836425921,514 -407.4629306793213,515 327.66476345062256516 ],517 "extent": [518 7.672381458117172,519 27.3011983454835,520 14.869946479797363521 ],522 "rotation": [523 0,524 -3.814697265625e-05,525 0526 ]527 },528 "location": [529 264.1894836425781,530 -407.46295166015625,531 327.6647644042969532 ],533 "rotation": [534 0,535 -3.814697265625e-05,536 0537 ],538 "scale": [539 1,540 1,541 1542 ]543 },544 "color": [545 127,546 255,547 63548 ]549 }550 ],551 "exit door": [552 {553 "object_id": "SM_exitdoor_379",554 "bbox_2d": [555 84,556 0,557 216,558 1079559 ],560 "bbox_3d": {561 "aabb": {562 "center": [563 213.24381917873689,564 -226.70896911621094,565 134.8391164401255566 ],567 "extent": [568 11.549467334564724,569 64.80331382804889,570 129.2236785888672571 ]572 },573 "obb": {574 "center": [575 213.24381917873689,576 -226.70896911621094,577 134.8391164401255578 ],579 "extent": [580 11.549467334564724,581 64.80331382804889,582 129.2236785888672583 ],584 "rotation": [585 0,586 0,587 0588 ]589 },590 "location": [591 214.8571014404297,592 -226.70896911621094,593 4.860979080200195594 ],595 "rotation": [596 0,597 0,598 0599 ],600 "scale": [601 1.0691914558410645,602 1.1697558164596558,603 1.153782844543457604 ]605 },606 "color": [607 0,608 63,609 127610 ]611 }612 ],613 "cloth": [614 {615 "object_id": "SM_cloth_06_398",616 "bbox_2d": [617 873,618 290,619 993,620 701621 ],622 "bbox_3d": {623 "aabb": {624 "center": [625 411.7523904654584,626 -334.5108947753906,627 122.06141194308293628 ],629 "extent": [630 31.999095554303494,631 15.56591796875,632 91.9655633581435633 ]634 },635 "obb": {636 "center": [637 411.7523904654584,638 -334.5108947753906,639 122.06141194308293640 ],641 "extent": [642 31.999095554303494,643 15.56591796875,644 91.9655633581435645 ],646 "rotation": [647 -7.05279541015625,648 0,649 0650 ]651 },652 "location": [653 420.81671142578125,654 -334.3865661621094,655 202.543701171875656 ],657 "rotation": [658 -7.05279541015625,659 0,660 0661 ],662 "scale": [663 1,664 1,665 1666 ]667 },668 "color": [669 0,670 63,671 63672 ]673 },674 {675 "object_id": "SM_cloth_5",676 "bbox_2d": [677 0,678 32,679 124,680 1079681 ],682 "bbox_3d": {683 "aabb": {684 "center": [685 227.53141768328933,686 -119.40682621360112,687 140.4956999521605688 ],689 "extent": [690 18.47999161274703,691 15.880772346645657,692 95.30916389242543693 ]694 },695 "obb": {696 "center": [697 227.53141768328933,698 -119.40682621360112,699 140.4956999521605700 ],701 "extent": [702 18.47999161274703,703 15.880772346645657,704 95.30916389242543705 ],706 "rotation": [707 3.074985027313233,708 -0.7657650113105776,709 -0.041014000773429884710 ]711 },712 "location": [713 224.62899780273438,714 -119.4305648803711,715 227.77149963378906716 ],717 "rotation": [718 3.074985027313233,719 -0.7657650113105776,720 -0.041014000773429884721 ],722 "scale": [723 1,724 1,725 1726 ]727 },728 "color": [729 127,730 63,731 63732 ]733 },734 {735 "object_id": "SM_cloth_6",736 "bbox_2d": [737 0,738 0,739 275,740 963741 ],742 "bbox_3d": {743 "aabb": {744 "center": [745 228.01023610238462,746 -149.97324215797198,747 139.8963253703865748 ],749 "extent": [750 21.795398950061013,751 16.623589526621004,752 93.2516195136919753 ]754 },755 "obb": {756 "center": [757 228.01023610238462,758 -149.97324215797198,759 139.8963253703865760 ],761 "extent": [762 21.795398950061013,763 16.623589526621004,764 93.2516195136919765 ],766 "rotation": [767 3.074985027313233,768 -0.7657650113105776,769 -0.041014000773429884770 ]771 },772 "location": [773 224.62899780273438,774 -149.990966796875,775 227.77149963378906776 ],777 "rotation": [778 3.074985027313233,779 -0.7657650113105776,780 -0.041014000773429884781 ],782 "scale": [783 1,784 1,785 1786 ]787 },788 "color": [789 63,790 0,791 0792 ]793 },794 {795 "object_id": "SM_cloth_10",796 "bbox_2d": [797 986,798 278,799 1028,800 333801 ],802 "bbox_3d": {803 "aabb": {804 "center": [805 413.32803751233575,806 -310.1559143066406,807 142.01436851787005808 ],809 "extent": [810 23.98763368928877,811 18.4566650390625,812 68.84829342316803813 ]814 },815 "obb": {816 "center": [817 413.32803751233575,818 -310.1559143066406,819 142.01436851787003820 ],821 "extent": [822 23.98763368928877,823 18.4566650390625,824 68.84829342316803825 ],826 "rotation": [827 -7.052794933319092,828 0,829 0830 ]831 },832 "location": [833 420.81671142578125,834 -310.1559143066406,835 202.543701171875836 ],837 "rotation": [838 -7.052794933319092,839 0,840 0841 ],842 "scale": [843 1,844 1,845 1846 ]847 },848 "color": [849 223,850 0,851 191852 ]853 }854 ],855 "ladder": [856 {857 "object_id": "SM_ladder2",858 "bbox_2d": [859 212,860 0,861 372,862 1066863 ],864 "bbox_3d": {865 "aabb": {866 "center": [867 231.2604099797911,868 -296.61433036763526,869 163.7025774325777870 ],871 "extent": [872 17.375431804216475,873 44.58013502962143,874 144.20106613466515875 ]876 },877 "obb": {878 "center": [879 231.2604099797911,880 -296.61433036763526,881 163.70257743257773882 ],883 "extent": [884 17.375431804216475,885 44.58013502962143,886 144.20106613466515887 ],888 "rotation": [889 0,890 -89.99986267089847,891 -2.1566863059997567892 ]893 },894 "location": [895 237.65826416015625,896 -292.5789794921875,897 20.000015258789062898 ],899 "rotation": [900 0,901 -89.99986267089847,902 -2.1566863059997567903 ],904 "scale": [905 1,906 1,907 1908 ]909 },910 "color": [911 63,912 0,913 63914 ]915 }916 ],917 "stack of food": [918 {919 "object_id": "SM_stack_of_food_515",920 "bbox_2d": [921 426,922 557,923 903,924 915925 ],926 "bbox_3d": {927 "aabb": {928 "center": [929 339.9145812988281,930 -390.86053466796875,931 69.61038208007812932 ],933 "extent": [934 81.03489445663894,935 44.01318928684174,936 50.09919738769531937 ]938 },939 "obb": {940 "center": [941 339.9145812988281,942 -390.86053466796875,943 69.61038208007812944 ],945 "extent": [946 81.03489445663894,947 44.01318928684174,948 50.09919738769531949 ],950 "rotation": [951 0,952 -178.55352783203125,953 0954 ]955 },956 "location": [957 339.9145812988281,958 -390.86053466796875,959 69.61038208007812960 ],961 "rotation": [962 0,963 -178.55352783203125,964 0965 ],966 "scale": [967 1,968 1,969 1970 ]971 },972 "color": [973 63,974 63,975 191976 ]977 }978 ],979 "trash": [980 {981 "object_id": "SM_trash_02_521",982 "bbox_2d": [983 501,984 1004,985 767,986 1079987 ],988 "bbox_3d": {989 "aabb": {990 "center": [991 292.1772155761719,992 -252.17471313476562,993 20.835508823394775994 ],995 "extent": [996 28.486328125,997 26.103759765625,998 0.8280215263366699999 ]1000 },1001 "obb": {1002 "center": [1003 292.1772155761719,1004 -252.17471313476562,1005 20.8355088233947751006 ],1007 "extent": [1008 28.486328125,1009 26.103759765625,1010 0.82802152633666991011 ],1012 "rotation": [1013 0,1014 0,1015 01016 ]1017 },1018 "location": [1019 292.1772155761719,1020 -252.17471313476562,1021 20.8355083465576171022 ],1023 "rotation": [1024 0,1025 0,1026 01027 ],1028 "scale": [1029 1,1030 1,1031 11032 ]1033 },1034 "color": [1035 0,1036 191,1037 2551038 ]1039 }1040 ],1041 "rag": [1042 {1043 "object_id": "SM_rag_04_18",1044 "bbox_2d": [1045 823,1046 841,1047 1083,1048 9291049 ],1050 "bbox_3d": {1051 "aabb": {1052 "center": [1053 396.71624388259596,1054 -311.9612175926489,1055 24.289909362792971056 ],1057 "extent": [1058 49.977114531506686,1059 57.24855124197603,1060 4.1431722640991211061 ]1062 },1063 "obb": {1064 "center": [1065 396.71624388259596,1066 -311.9612175926489,1067 24.289909362792971068 ],1069 "extent": [1070 49.977114531506686,1071 57.24855124197603,1072 4.1431722640991211073 ],1074 "rotation": [1075 0,1076 -14.847186088562012,1077 01078 ]1079 },1080 "location": [1081 397.7102966308594,1082 -314.9366455078125,1083 24.289909362792971084 ],1085 "rotation": [1086 0,1087 -14.847186088562012,1088 01089 ],1090 "scale": [1091 1,1092 1,1093 11094 ]1095 },1096 "color": [1097 223,1098 0,1099 631100 ]1101 }1102 ],1103 "signboard": [1104 {1105 "object_id": "SM_signboard2_31",1106 "bbox_2d": [1107 362,1108 724,1109 524,1110 9961111 ],1112 "bbox_3d": {1113 "aabb": {1114 "center": [1115 262.8511073776462,1116 -330.8132155256824,1117 48.563442230224611118 ],1119 "extent": [1120 19.27591159772119,1121 13.624070188210283,1122 28.9499702453613281123 ]1124 },1125 "obb": {1126 "center": [1127 262.8511073776462,1128 -330.8132155256824,1129 48.563442230224611130 ],1131 "extent": [1132 19.27591159772119,1133 13.624070188210283,1134 28.9499702453613281135 ],1136 "rotation": [1137 0,1138 79.14710235595702,1139 01140 ]1141 },1142 "location": [1143 262.7721862792969,1144 -330.9164123535156,1145 19.7164459228515621146 ],1147 "rotation": [1148 0,1149 79.14710235595702,1150 01151 ],1152 "scale": [1153 1,1154 1,1155 11156 ]1157 },1158 "color": [1159 223,1160 0,1161 1591162 ]1163 }1164 ],1165 "box": [1166 {1167 "object_id": "SM_box_43",1168 "bbox_2d": [1169 352,1170 509,1171 421,1172 5951173 ],1174 "bbox_3d": {1175 "aabb": {1176 "center": [1177 240.32976474556892,1178 -367.8424491802709,1179 122.552475929260251180 ],1181 "extent": [1182 18.93478630149049,1183 25.459768463482703,1184 11.812940597534181185 ]1186 },1187 "obb": {1188 "center": [1189 240.32976474556892,1190 -367.8424491802709,1191 122.552475929260251192 ],1193 "extent": [1194 18.93478630149049,1195 25.459768463482703,1196 11.812940597534181197 ],1198 "rotation": [1199 0,1200 2.2834024429321294,