OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
2613k
1{2 "episode_id": "0469935355958167",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Information_Management",13 "app": [14 "Chrome",15 "Simplenote"16 ],17 "meta_task": "Find a recipe for {} and create a shopping list for the main ingredients.",18 "task": "Find a recipe for Margherita pizza and create a shopping list for the main ingredients.",19 "instruction": "Using Chrome, search for a Margherita pizza recipe. Then, open Simplenote and create a shopping list containing the main ingredients for the recipe you found."20 },21 "step_length": 16,22 "steps": [23 {24 "step": 0,25 "screenshot": "0469935355958167_0.png",26 "action": "CLICK",27 "info": [28 [29 581,30 80931 ],32 [33 581,34 80935 ]36 ],37 "ps": "[(581, 809)]"38 },39 {40 "step": 1,41 "screenshot": "0469935355958167_1.png",42 "action": "CLICK",43 "info": [44 [45 55,46 5147 ],48 [49 55,50 5151 ]52 ],53 "ps": "[(55, 51)]"54 },55 {56 "step": 2,57 "screenshot": "0469935355958167_2.png",58 "action": "CLICK",59 "info": [60 [61 211,62 27663 ],64 [65 211,66 27667 ]68 ],69 "ps": "[(211, 276)]"70 },71 {72 "step": 3,73 "screenshot": "0469935355958167_3.png",74 "action": "TEXT",75 "info": "the recipe for Margherita pizza",76 "ps": ""77 },78 {79 "step": 4,80 "screenshot": "0469935355958167_4.png",81 "action": "CLICK",82 "info": [83 [84 639,85 12886 ],87 [88 639,89 12890 ]91 ],92 "ps": "[(639, 128)]"93 },94 {95 "step": 5,96 "screenshot": "0469935355958167_5.png",97 "action": "CLICK",98 "info": [99 [100 160,101 415102 ],103 [104 160,105 415106 ]107 ],108 "ps": "[(160, 415)]"109 },110 {111 "step": 6,112 "screenshot": "0469935355958167_6.png",113 "action": "SCROLL",114 "info": [115 [116 36,117 458118 ],119 [120 999,121 467122 ]123 ],124 "ps": "[(36, 458), (38, 458), (40, 459), (45, 460), (55, 461), (72, 463), (97, 464), (125, 464), (160, 464), (203, 464), (248, 464), (291, 464), (335, 464), (374, 464), (408, 467), (442, 467), (471, 467), (500, 467), (529, 467), (563, 467), (598, 467), (637, 467), (675, 467), (724, 467), (775, 467), (836, 467), (895, 467), (955, 467), (999, 467)]"125 },126 {127 "step": 7,128 "screenshot": "0469935355958167_7.png",129 "action": "SCROLL",130 "info": [131 [132 562,133 675134 ],135 [136 517,137 16138 ]139 ],140 "ps": "[(562, 675), (562, 673), (562, 668), (562, 661), (562, 654), (562, 648), (562, 642), (562, 635), (562, 631), (562, 629), (562, 624), (562, 617), (562, 609), (562, 602), (562, 596), (562, 586), (562, 577), (562, 568), (562, 559), (562, 551), (562, 544), (562, 537), (562, 530), (562, 526), (562, 520), (562, 516), (562, 510), (562, 506), (562, 500), (562, 496), (562, 490), (562, 486), (562, 481), (562, 476), (562, 472), (562, 465), (562, 458), (562, 449), (562, 440), (562, 432), (562, 423), (562, 416), (562, 409), (562, 402), (562, 395), (562, 389), (562, 382), (562, 377), (562, 370), (562, 364), (562, 357), (562, 350), (562, 345), (562, 339), (562, 334), (562, 327), (562, 322), (562, 315), (562, 309), (562, 302), (562, 296), (562, 290), (562, 286), (562, 282), (562, 280), (562, 276), (562, 274), (562, 271), (562, 268), (562, 265), (562, 263), (562, 261), (562, 257), (562, 255), (562, 252), (562, 249), (562, 247), (562, 243), (562, 242), (562, 240), (562, 238), (562, 237), (562, 236), (562, 234), (562, 232), (562, 228), (562, 224), (562, 222), (562, 220), (562, 217), (562, 216), (562, 214), (562, 210), (562, 207), (562, 203), (562, 196), (562, 187), (562, 179), (562, 168), (562, 159), (562, 150), (562, 142), (562, 134), (562, 127), (562, 121), (562, 116), (562, 110), (562, 107), (562, 105), (562, 102), (562, 100), (562, 99), (562, 98), (562, 96), (562, 93), (562, 91), (562, 89), (562, 88), (562, 87), (562, 86), (562, 84), (559, 81), (559, 79), (556, 77), (556, 74), (556, 72), (551, 69), (551, 68), (549, 66), (549, 64), (549, 62), (549, 58), (547, 57), (547, 55), (544, 54), (544, 52), (544, 51), (541, 49), (539, 45), (537, 40), (535, 33), (529, 28), (527, 24), (524, 22), (522, 19), (517, 16)]"141 },142 {143 "step": 8,144 "screenshot": "0469935355958167_8.png",145 "action": "SCROLL",146 "info": [147 [148 486,149 923150 ],151 [152 493,153 122154 ]155 ],156 "ps": "[(486, 923), (486, 920), (486, 917), (486, 915), (486, 911), (486, 904), (486, 897), (486, 890), (486, 881), (481, 871), (481, 860), (478, 849), (474, 839), (474, 827), (474, 814), (474, 801), (474, 789), (474, 778), (474, 769), (474, 761), (474, 754), (474, 748), (474, 741), (474, 733), (474, 727), (474, 718), (474, 709), (474, 701), (474, 692), (474, 682), (474, 675), (474, 669), (474, 660), (474, 649), (474, 638), (474, 627), (474, 616), (474, 603), (476, 589), (478, 576), (483, 565), (486, 551), (486, 540), (486, 527), (486, 514), (486, 500), (486, 485), (490, 470), (490, 457), (496, 441), (496, 428), (496, 415), (496, 402), (496, 389), (496, 378), (496, 367), (496, 355), (496, 344), (496, 333), (496, 322), (496, 313), (496, 307), (496, 300), (496, 294), (496, 284), (496, 275), (496, 266), (496, 257), (496, 249), (496, 240), (496, 231), (496, 224), (496, 217), (496, 210), (496, 204), (496, 197), (496, 189), (496, 183), (496, 177), (496, 172), (496, 166), (496, 160), (496, 154), (496, 149), (496, 142), (496, 140), (496, 138), (496, 137), (496, 136), (496, 135), (496, 133), (496, 132), (496, 131), (496, 128), (493, 127), (493, 126), (493, 125), (493, 123), (493, 122)]"157 },158 {159 "step": 9,160 "screenshot": "0469935355958167_9.png",161 "action": "CLICK",162 "info": [163 [164 179,165 147166 ],167 [168 179,169 147170 ]171 ],172 "ps": "[(179, 147)]"173 },174 {175 "step": 10,176 "screenshot": "0469935355958167_10.png",177 "action": "CLICK",178 "info": "KEY_HOME",179 "ps": ""180 },181 {182 "step": 11,183 "screenshot": "0469935355958167_11.png",184 "action": "CLICK",185 "info": [186 [187 861,188 241189 ],190 [191 861,192 241193 ]194 ],195 "ps": "[(861, 241)]"196 },197 {198 "step": 12,199 "screenshot": "0469935355958167_12.png",200 "action": "CLICK",201 "info": [202 [203 885,204 913205 ],206 [207 885,208 913209 ]210 ],211 "ps": "[(885, 913)]"212 },213 {214 "step": 13,215 "screenshot": "0469935355958167_13.png",216 "action": "TEXT",217 "info": "the main ingredients for Margherita: a bubbly crust, tomato sauce, mozzarella, basil, oil, salt",218 "ps": ""219 },220 {221 "step": 14,222 "screenshot": "0469935355958167_14.png",223 "action": "CLICK",224 "info": [225 [226 91,227 58228 ],229 [230 91,231 58232 ]233 ],234 "ps": "[(91, 58)]"235 },236 {237 "step": 15,238 "screenshot": "0469935355958167_15.png",239 "action": "COMPLETE",240 "info": "",241 "ps": ""242 }243 ]244}