timlawrenz/gnn-ruby-code-study
GNN Ruby Code Study Systematic study of Graph Neural Network architectures for Ruby code complexity prediction and generation. Paper: Graph Neural Networks for Ruby Code Complexity Prediction and Generation: A Systematic Architecture Study Dataset 22,452 Ruby methods parsed into AST graphs with 74-dimensional node features. Split Samples File Train 19,084 dataset/train.jsonl Validation 3,368 dataset/val.jsonl Each JSONL record contains:… See the full description on the dataset page: https://huggingface.co/datasets/timlawrenz/gnn-ruby-code-study.
0101
1{2 "gnn-architecture-comparison": {3 "sage-baseline": {4 "experiment": "gnn-architecture-comparison",5 "arm_name": "sage-baseline",6 "description": "GraphSAGE baseline (original architecture)",7 "instance_id": 34817102,8 "gpu_info": "NVIDIA GeForce RTX 4090, 23028 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 251Gi 57Gi 136Gi 51Mi 58Gi 191Gi\nModel name: AMD EPYC 7763 64-Core Processor",9 "metrics": {10 "val_mae": 4.7816,11 "val_mse": 68.0714,12 "val_r2": 0.6354,13 "best_val_loss": 68.0577,14 "conv_type": "SAGE",15 "hidden_dim": 64,16 "num_layers": 3,17 "dropout": 0.1,18 "learning_rate": 0.001,19 "epochs": 5020 },21 "exit_code": 0,22 "error": "",23 "duration_seconds": 712.9457042200083,24 "timestamp": "2026-04-13T06:20:29.613824+00:00"25 },26 "gcn": {27 "experiment": "gnn-architecture-comparison",28 "arm_name": "gcn",29 "description": "Graph Convolutional Network",30 "instance_id": 34817109,31 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 1.0Ti 156Gi 405Gi 41Gi 445Gi 803Gi\nModel name: AMD EPYC 7B13 64-Core Processor",32 "metrics": {33 "val_mae": 5.3207,34 "val_mse": 81.6099,35 "val_r2": 0.5628,36 "best_val_loss": 81.5149,37 "conv_type": "GCN",38 "hidden_dim": 64,39 "num_layers": 3,40 "dropout": 0.1,41 "learning_rate": 0.001,42 "epochs": 5043 },44 "exit_code": 0,45 "error": "",46 "duration_seconds": 1049.5592152850004,47 "timestamp": "2026-04-13T06:20:29.614482+00:00"48 },49 "gat": {50 "experiment": "gnn-architecture-comparison",51 "arm_name": "gat",52 "description": "Graph Attention Network",53 "instance_id": 34817119,54 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 251Gi 38Gi 82Gi 36Mi 130Gi 210Gi\nModel name: AMD Ryzen Threadripper PRO 3975WX 32-Cores",55 "metrics": {56 "val_mae": 4.9519,57 "val_mse": 73.1851,58 "val_r2": 0.608,59 "best_val_loss": 73.3893,60 "conv_type": "GAT",61 "hidden_dim": 64,62 "num_layers": 3,63 "dropout": 0.1,64 "learning_rate": 0.001,65 "epochs": 5066 },67 "exit_code": 0,68 "error": "",69 "duration_seconds": 478.46807387899025,70 "timestamp": "2026-04-13T06:20:29.614500+00:00"71 },72 "gin": {73 "experiment": "gnn-architecture-comparison",74 "arm_name": "gin",75 "description": "Graph Isomorphism Network",76 "instance_id": 34817126,77 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 5.0Gi 30Gi 17Mi 468Gi 494Gi\nModel name: AMD Ryzen Threadripper PRO 3955WX 16-Cores",78 "metrics": {79 "val_mae": 4.5889,80 "val_mse": 69.2775,81 "val_r2": 0.6289,82 "best_val_loss": 69.3397,83 "conv_type": "GIN",84 "hidden_dim": 64,85 "num_layers": 3,86 "dropout": 0.1,87 "learning_rate": 0.001,88 "epochs": 5089 },90 "exit_code": 0,91 "error": "",92 "duration_seconds": 397.1449088840018,93 "timestamp": "2026-04-13T06:20:29.614511+00:00"94 },95 "graphconv": {96 "experiment": "gnn-architecture-comparison",97 "arm_name": "graphconv",98 "description": "GraphConv (Morris et al.)",99 "instance_id": 34817138,100 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 1.0Ti 137Gi 264Gi 149Mi 606Gi 860Gi\nModel name: AMD EPYC 7V13 64-Core Processor",101 "metrics": {102 "val_mae": 4.8042,103 "val_mse": 68.1418,104 "val_r2": 0.635,105 "best_val_loss": 68.0079,106 "conv_type": "GraphConv",107 "hidden_dim": 64,108 "num_layers": 3,109 "dropout": 0.1,110 "learning_rate": 0.001,111 "epochs": 50112 },113 "exit_code": 0,114 "error": "",115 "duration_seconds": 781.278599569996,116 "timestamp": "2026-04-13T06:20:29.614519+00:00"117 },118 "sage-wide": {119 "experiment": "gnn-architecture-comparison",120 "arm_name": "sage-wide",121 "description": "SAGE with 128 hidden dim",122 "instance_id": 34817145,123 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 566Gi 25Gi 142Gi 171Mi 398Gi 534Gi\nModel name: AMD EPYC 7763 64-Core Processor",124 "metrics": {125 "val_mae": 4.8625,126 "val_mse": 68.1472,127 "val_r2": 0.635,128 "best_val_loss": 68.0128,129 "conv_type": "SAGE",130 "hidden_dim": 128,131 "num_layers": 3,132 "dropout": 0.1,133 "learning_rate": 0.001,134 "epochs": 50135 },136 "exit_code": 0,137 "error": "",138 "duration_seconds": 857.3578274790052,139 "timestamp": "2026-04-13T06:20:29.614526+00:00"140 },141 "gat-wide": {142 "experiment": "gnn-architecture-comparison",143 "arm_name": "gat-wide",144 "description": "GAT with 128 hidden dim",145 "instance_id": 34817151,146 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 2.0Ti 105Gi 470Gi 174Mi 1.4Ti 1.8Ti\nModel name: AMD EPYC 7702 64-Core Processor",147 "metrics": {},148 "exit_code": 124,149 "error": "Training failed (exit 124): STDERR: Timed out after 1200s\nSTDOUT(tail): ",150 "duration_seconds": 1200.0279779760021,151 "timestamp": "2026-04-13T06:20:29.614536+00:00"152 },153 "sage-deep": {154 "experiment": "gnn-architecture-comparison",155 "arm_name": "sage-deep",156 "description": "SAGE with 5 layers",157 "instance_id": 34817159,158 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 340Gi 3.1Gi 0.0Ki 160Gi 159Gi\nModel name: AMD Ryzen Threadripper PRO 3995WX 64-Cores",159 "metrics": {160 "val_mae": 4.0184,161 "val_mse": 54.3718,162 "val_r2": 0.7087,163 "best_val_loss": 54.4308,164 "conv_type": "SAGE",165 "hidden_dim": 64,166 "num_layers": 5,167 "dropout": 0.1,168 "learning_rate": 0.001,169 "epochs": 50170 },171 "exit_code": 0,172 "error": "",173 "duration_seconds": 549.5692676960025,174 "timestamp": "2026-04-13T06:20:29.614542+00:00"175 }176 },177 "gnn-generation-analysis": {178 "improved-loss-gat": {179 "experiment": "gnn-generation-analysis",180 "arm_name": "improved-loss-gat",181 "description": "Improved (cross-entropy) loss, GAT decoder",182 "instance_id": 34818534,183 "gpu_info": "NVIDIA GeForce RTX 4090, 23028 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 251Gi 59Gi 133Gi 51Mi 58Gi 189Gi\nModel name: AMD EPYC 7763 64-Core Processor",184 "metrics": {185 "syntactic_validity_pct": 0.0,186 "val_loss": 7.7184,187 "samples_evaluated": 100,188 "valid_samples": 0,189 "decoder_conv_type": "GAT",190 "hidden_dim": 256,191 "num_layers": 5,192 "loss_fn": "improved",193 "type_weight": 2.0,194 "parent_weight": 1.0,195 "learning_rate": 0.001,196 "epochs": 30197 },198 "exit_code": 0,199 "error": "",200 "duration_seconds": 87.49750811600825,201 "timestamp": "2026-04-13T06:43:41.644479+00:00"202 },203 "simple-loss-gat": {204 "experiment": "gnn-generation-analysis",205 "arm_name": "simple-loss-gat",206 "description": "Simple (MSE) loss, GAT decoder",207 "instance_id": 34818537,208 "gpu_info": "",209 "metrics": {},210 "exit_code": -1,211 "error": "SSH never became ready",212 "duration_seconds": 0.0,213 "timestamp": "2026-04-13T06:43:41.645165+00:00"214 },215 "comprehensive-loss-gat": {216 "experiment": "gnn-generation-analysis",217 "arm_name": "comprehensive-loss-gat",218 "description": "Comprehensive (combined) loss, GAT decoder",219 "instance_id": 34818538,220 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 5.0Gi 30Gi 17Mi 468Gi 494Gi\nModel name: AMD Ryzen Threadripper PRO 3955WX 16-Cores",221 "metrics": {222 "error": "training_failed",223 "exit_code": 1224 },225 "exit_code": 1,226 "error": "setup:\n Optimizer: Adam (lr=0.001)\n Scheduler: ReduceLROnPlateau (patience=5)\n Loss function: Improved Reconstruction Loss\n AMP Enabled: True\n\n\ud83c\udfcb\ufe0f Starting training...\n==================================================\nTraceback (most recent call last):\n File \"/workspace/experiment/train_autoencoder.py\", line 393, in <module>\n main()\n File \"/workspace/experiment/train_autoencoder.py\", line 332, in main\n train_loss = train_epoch(model, train_loader, optimizer, device, args.type_weight, args.parent_weight, scaler, loss_fn=loss_fn)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/workspace/experiment/train_autoencoder.py\", line 65, in train_epoch\n loss = loss_fn(\n ^^^^^^^^\nTypeError: ast_reconstruction_loss_comprehensive() got an unexpected keyword argument 'type_weight'\nERROR: train_autoencoder.py exited with code 1\nMETRICS:{\"error\": \"training_failed\", \"exit_code\": 1}\n",227 "duration_seconds": 16.827261095982976,228 "timestamp": "2026-04-13T06:43:41.645183+00:00"229 },230 "improved-loss-sage": {231 "experiment": "gnn-generation-analysis",232 "arm_name": "improved-loss-sage",233 "description": "Improved loss, SAGE decoder",234 "instance_id": 34818541,235 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 117Gi 138Gi 15Gi 247Gi 367Gi\nModel name: AMD EPYC 7B13 64-Core Processor",236 "metrics": {237 "syntactic_validity_pct": 0.0,238 "val_loss": 7.7897,239 "samples_evaluated": 100,240 "valid_samples": 0,241 "decoder_conv_type": "SAGE",242 "hidden_dim": 256,243 "num_layers": 5,244 "loss_fn": "improved",245 "type_weight": 2.0,246 "parent_weight": 1.0,247 "learning_rate": 0.001,248 "epochs": 30249 },250 "exit_code": 0,251 "error": "",252 "duration_seconds": 143.7897430199955,253 "timestamp": "2026-04-13T06:43:41.645193+00:00"254 },255 "improved-loss-gin": {256 "experiment": "gnn-generation-analysis",257 "arm_name": "improved-loss-gin",258 "description": "Improved loss, GIN decoder",259 "instance_id": 34818545,260 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 566Gi 25Gi 142Gi 171Mi 398Gi 534Gi\nModel name: AMD EPYC 7763 64-Core Processor",261 "metrics": {262 "syntactic_validity_pct": 0.0,263 "val_loss": 7.8025,264 "samples_evaluated": 100,265 "valid_samples": 0,266 "decoder_conv_type": "GIN",267 "hidden_dim": 256,268 "num_layers": 5,269 "loss_fn": "improved",270 "type_weight": 2.0,271 "parent_weight": 1.0,272 "learning_rate": 0.001,273 "epochs": 30274 },275 "exit_code": 0,276 "error": "",277 "duration_seconds": 82.56781898299232,278 "timestamp": "2026-04-13T06:43:41.645201+00:00"279 },280 "improved-loss-gcn": {281 "experiment": "gnn-generation-analysis",282 "arm_name": "improved-loss-gcn",283 "description": "Improved loss, GCN decoder",284 "instance_id": 34818549,285 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 1.0Ti 219Gi 271Gi 627Mi 516Gi 776Gi\nModel name: AMD EPYC 7C13 64-Core Processor",286 "metrics": {287 "syntactic_validity_pct": 0.0,288 "val_loss": 7.7638,289 "samples_evaluated": 100,290 "valid_samples": 0,291 "decoder_conv_type": "GCN",292 "hidden_dim": 256,293 "num_layers": 5,294 "loss_fn": "improved",295 "type_weight": 2.0,296 "parent_weight": 1.0,297 "learning_rate": 0.001,298 "epochs": 30299 },300 "exit_code": 0,301 "error": "",302 "duration_seconds": 79.23930556201958,303 "timestamp": "2026-04-13T06:43:41.645208+00:00"304 },305 "improved-loss-gat-wide": {306 "experiment": "gnn-generation-analysis",307 "arm_name": "improved-loss-gat-wide",308 "description": "Improved loss, GAT decoder, hidden_dim=512",309 "instance_id": 34818556,310 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 2.0Ti 131Gi 444Gi 182Mi 1.4Ti 1.8Ti\nModel name: AMD EPYC 7702 64-Core Processor",311 "metrics": {312 "syntactic_validity_pct": 0.0,313 "val_loss": 7.7262,314 "samples_evaluated": 100,315 "valid_samples": 0,316 "decoder_conv_type": "GAT",317 "hidden_dim": 512,318 "num_layers": 5,319 "loss_fn": "improved",320 "type_weight": 2.0,321 "parent_weight": 1.0,322 "learning_rate": 0.001,323 "epochs": 30324 },325 "exit_code": 0,326 "error": "",327 "duration_seconds": 259.52819745699526,328 "timestamp": "2026-04-13T06:43:41.645217+00:00"329 }330 },331 "gnn-decoder-topology": {332 "chain-gat": {333 "experiment": "gnn-decoder-topology",334 "arm_name": "chain-gat",335 "description": "Chain edges (legacy baseline), GAT decoder",336 "instance_id": 34818971,337 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 1.0Ti 157Gi 403Gi 41Gi 446Gi 802Gi\nModel name: AMD EPYC 7B13 64-Core Processor",338 "metrics": {339 "syntactic_validity_pct": 0.0,340 "val_loss": 7.7151,341 "samples_evaluated": 100,342 "valid_samples": 0,343 "decoder_edge_mode": "chain",344 "decoder_conv_type": "GAT",345 "hidden_dim": 256,346 "num_layers": 5,347 "loss_fn": "improved",348 "type_weight": 2.0,349 "parent_weight": 1.0,350 "learning_rate": 0.001,351 "epochs": 30352 },353 "exit_code": 0,354 "error": "",355 "duration_seconds": 93.43292950798059,356 "timestamp": "2026-04-13T06:52:18.887589+00:00"357 },358 "teacher-forced-gat": {359 "experiment": "gnn-decoder-topology",360 "arm_name": "teacher-forced-gat",361 "description": "Teacher-forced tree edges, GAT decoder",362 "instance_id": 34818979,363 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 93Gi 210Gi 16Gi 199Gi 389Gi\nModel name: AMD EPYC 7B12 64-Core Processor",364 "metrics": {365 "syntactic_validity_pct": 0.0,366 "val_loss": 7.7059,367 "samples_evaluated": 100,368 "valid_samples": 0,369 "decoder_edge_mode": "teacher_forced",370 "decoder_conv_type": "GAT",371 "hidden_dim": 256,372 "num_layers": 5,373 "loss_fn": "improved",374 "type_weight": 2.0,375 "parent_weight": 1.0,376 "learning_rate": 0.001,377 "epochs": 30378 },379 "exit_code": 0,380 "error": "",381 "duration_seconds": 102.00913854799,382 "timestamp": "2026-04-13T06:52:18.888261+00:00"383 },384 "iterative-gat": {385 "experiment": "gnn-decoder-topology",386 "arm_name": "iterative-gat",387 "description": "Iterative predict\u2192refine, GAT decoder",388 "instance_id": 34818982,389 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 3.6Gi 379Gi 27Mi 120Gi 495Gi\nModel name: AMD EPYC 7282 16-Core Processor",390 "metrics": {391 "syntactic_validity_pct": 0.0,392 "val_loss": 7.7649,393 "samples_evaluated": 100,394 "valid_samples": 0,395 "decoder_edge_mode": "iterative",396 "decoder_conv_type": "GAT",397 "hidden_dim": 256,398 "num_layers": 5,399 "loss_fn": "improved",400 "type_weight": 2.0,401 "parent_weight": 1.0,402 "learning_rate": 0.001,403 "epochs": 30404 },405 "exit_code": 0,406 "error": "",407 "duration_seconds": 102.66292616000283,408 "timestamp": "2026-04-13T06:52:18.888279+00:00"409 },410 "teacher-forced-sage": {411 "experiment": "gnn-decoder-topology",412 "arm_name": "teacher-forced-sage",413 "description": "Teacher-forced tree edges, SAGE decoder",414 "instance_id": 34818986,415 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 566Gi 25Gi 143Gi 171Mi 398Gi 534Gi\nModel name: AMD EPYC 7763 64-Core Processor",416 "metrics": {417 "syntactic_validity_pct": 0.0,418 "val_loss": 7.7987,419 "samples_evaluated": 100,420 "valid_samples": 0,421 "decoder_edge_mode": "teacher_forced",422 "decoder_conv_type": "SAGE",423 "hidden_dim": 256,424 "num_layers": 5,425 "loss_fn": "improved",426 "type_weight": 2.0,427 "parent_weight": 1.0,428 "learning_rate": 0.001,429 "epochs": 30430 },431 "exit_code": 0,432 "error": "",433 "duration_seconds": 85.2959101049928,434 "timestamp": "2026-04-13T06:52:18.888290+00:00"435 },436 "teacher-forced-gin": {437 "experiment": "gnn-decoder-topology",438 "arm_name": "teacher-forced-gin",439 "description": "Teacher-forced tree edges, GIN decoder",440 "instance_id": 34818988,441 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 4.6Gi 109Gi 9.0Mi 389Gi 494Gi\nModel name: AMD Ryzen Threadripper PRO 3955WX 16-Cores",442 "metrics": {443 "syntactic_validity_pct": 7.0,444 "val_loss": 8.3833,445 "samples_evaluated": 100,446 "valid_samples": 7,447 "decoder_edge_mode": "teacher_forced",448 "decoder_conv_type": "GIN",449 "hidden_dim": 256,450 "num_layers": 5,451 "loss_fn": "improved",452 "type_weight": 2.0,453 "parent_weight": 1.0,454 "learning_rate": 0.001,455 "epochs": 30456 },457 "exit_code": 0,458 "error": "",459 "duration_seconds": 57.38957904101699,460 "timestamp": "2026-04-13T06:52:18.888298+00:00"461 },462 "teacher-forced-gat-comprehensive": {463 "experiment": "gnn-decoder-topology",464 "arm_name": "teacher-forced-gat-comprehensive",465 "description": "Teacher-forced, GAT, comprehensive loss",466 "instance_id": 34818993,467 "gpu_info": "NVIDIA GeForce RTX 4090, 24564 MiB, 1, 16\n---\n total used free shared buff/cache available\nMem: 503Gi 25Gi 5.7Gi 252Mi 472Gi 474Gi\nModel name: AMD EPYC 7K62 48-Core Processor",468 "metrics": {469 "error": "training_failed",470 "exit_code": 1471 },472 "exit_code": 1,473 "error": "setup:\n Optimizer: Adam (lr=0.001)\n Scheduler: ReduceLROnPlateau (patience=5)\n Loss function: Improved Reconstruction Loss\n AMP Enabled: True\n\n\ud83c\udfcb\ufe0f Starting training...\n==================================================\nTraceback (most recent call last):\n File \"/workspace/experiment/train_autoencoder.py\", line 393, in <module>\n main()\n File \"/workspace/experiment/train_autoencoder.py\", line 332, in main\n train_loss = train_epoch(model, train_loader, optimizer, device, args.type_weight, args.parent_weight, scaler, loss_fn=loss_fn)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/workspace/experiment/train_autoencoder.py\", line 65, in train_epoch\n loss = loss_fn(\n ^^^^^^^^\nTypeError: ast_reconstruction_loss_comprehensive() got an unexpected keyword argument 'type_weight'\nERROR: train_autoencoder.py exited with code 1\nMETRICS:{\"error\": \"training_failed\", \"exit_code\": 1}\n",474 "duration_seconds": 21.593328246992314,475 "timestamp": "2026-04-13T06:52:18.888305+00:00"476 }477 }478}479