sajalmadan09/bert-tiny-native-cpp

1{2  "hardware": "Apple M4, 10 cores, 16GB RAM, macOS (see https://github.com/Sajalmadan09/sajal-labs/tree/main/research/environment.md)",3  "methodology": "https://github.com/Sajalmadan09/sajal-labs/tree/main/research/experiments/exp11-real-pretrained-transformer/results.md, https://github.com/Sajalmadan09/sajal-labs/tree/main/research/experiments/exp12-warm-latency-width-depth/results.md, https://github.com/Sajalmadan09/sajal-labs/tree/main/research/experiments/exp13-width-threshold/results.md, https://github.com/Sajalmadan09/sajal-labs/tree/main/research/experiments/exp14-wordpiece-native/results.md",4  "equivalence_native_vs_pytorch": {5    "note": "native encoder + tokenizer vs. HF BertModel + BertTokenizerFast, 10 real sentences (lengths 4-25 tokens)",6    "hidden_state_max_abs_error": 9.5367431640625e-06,7    "pooled_output_max_abs_error": 2.1904706954956055e-06,8    "mean_cosine_similarity_pooled": 1.0000000357627879  },10  "tokenizer_equivalence": "17/17 test sentences (incl. contractions, hyphens, OOV words, emails) produced byte-identical token IDs to HF's real tokenizer \u2014 see exp14",11  "cold_invocation_ms_p50_precomputed_tokens": {12    "native": 8.747625000069092,13    "onnx_runtime_cpu": 87.30802099944412,14    "pytorch_plus_transformers": 5095.87887499947115  },16  "cold_invocation_ms_p50_raw_text_full_pipeline": {17    "native": 11.36558350026462,18    "onnx_runtime_cpu_plus_lean_tokenizer": 95.72114549973776,19    "onnx_runtime_cpu_plus_transformers_tokenizer": 2495.6148125002073,20    "pytorch_plus_transformers": 4973.90716650079421  },22  "warm_loop_ms_p50": {23    "native": 0.11565,24    "onnx_runtime_cpu": 0.1342290006505209,25    "pytorch_eager": 0.2760830011538928426  },27  "warm_loop_note": "Unlike cold invocation, native's warm-loop edge is NOT unconditional \u2014 it depends on model width; see exp12/exp13. At this model's hidden_size=128 it's below the measured ~250 crossover, so native keeps a real edge."28}