Datasetpublic
dispatchAI/on-device-latency

On-Device Latency Benchmark Real-world inference latency data for mobile-optimized LLMs, measured on actual phone hardware. Hardware Spec Value Device Samsung S20 FE 5G SoC Snapdragon 865 RAM 8GB OS Android 13 Runtime llama.cpp (4 threads) Metrics tokens_per_sec — Generation speed during inference latency_ms_per_token — Time per generated token ram_usage_mb — Peak RAM during inference file_size_mb — GGUF model file size… See the full description on the dataset page: https://huggingface.co/datasets/dispatchAI/on-device-latency.
sourceHugging Faceapache-2.0updated 4mo agoView on Hugging Face
0likes24downloads
rootlatency_data.json
latency_data.json496 linesDownload Raw Back to root
1[2  {3    "model": "Qwen2.5-1.5B-Instruct-mobile-int4",4    "params": "1.5B",5    "quant": "INT4",6    "file_size_mb": 2400,7    "ram_usage_mb": 4480,8    "tokens_per_sec": 16.8,9    "latency_ms_per_token": 60.0,10    "hardware": "Snapdragon 865",11    "device": "Samsung S20 FE",12    "runtime": "llama.cpp",13    "threads": 414  },15  {16    "model": "Qwen2.5-0.5B-Instruct-mobile-int4",17    "params": "500M",18    "quant": "INT4",19    "file_size_mb": 800,20    "ram_usage_mb": 1680,21    "tokens_per_sec": 32.0,22    "latency_ms_per_token": 31.0,23    "hardware": "Snapdragon 865",24    "device": "Samsung S20 FE",25    "runtime": "llama.cpp",26    "threads": 427  },28  {29    "model": "TinyLlama-1.1B-Chat-mobile-int4",30    "params": "1.1B",31    "quant": "INT4",32    "file_size_mb": 2160,33    "ram_usage_mb": 3584,34    "tokens_per_sec": 18.4,35    "latency_ms_per_token": 54.0,36    "hardware": "Snapdragon 865",37    "device": "Samsung S20 FE",38    "runtime": "llama.cpp",39    "threads": 440  },41  {42    "model": "Llama-3.2-1B-Instruct-mobile",43    "params": "1B",44    "quant": "FP16",45    "file_size_mb": 1250,46    "ram_usage_mb": 2100,47    "tokens_per_sec": 12.0,48    "latency_ms_per_token": 83.0,49    "hardware": "Snapdragon 865",50    "device": "Samsung S20 FE",51    "runtime": "llama.cpp",52    "threads": 453  },54  {55    "model": "Llama-3.2-3B-Instruct-mobile",56    "params": "3B",57    "quant": "FP16",58    "file_size_mb": 3000,59    "ram_usage_mb": 4900,60    "tokens_per_sec": 5.5,61    "latency_ms_per_token": 182.0,62    "hardware": "Snapdragon 865",63    "device": "Samsung S20 FE",64    "runtime": "llama.cpp",65    "threads": 466  },67  {68    "model": "SmolLM2-135M-Instruct-mobile",69    "params": "135M",70    "quant": "FP16",71    "file_size_mb": 135,72    "ram_usage_mb": 280,73    "tokens_per_sec": 25.0,74    "latency_ms_per_token": 40.0,75    "hardware": "Snapdragon 865",76    "device": "Samsung S20 FE",77    "runtime": "llama.cpp",78    "threads": 479  },80  {81    "model": "SmolLM2-1.7B-Instruct-mobile",82    "params": "1.7B",83    "quant": "FP16",84    "file_size_mb": 1700,85    "ram_usage_mb": 3150,86    "tokens_per_sec": 9.0,87    "latency_ms_per_token": 111.0,88    "hardware": "Snapdragon 865",89    "device": "Samsung S20 FE",90    "runtime": "llama.cpp",91    "threads": 492  },93  {94    "model": "Gemma-2-2B-IT-mobile",95    "params": "2B",96    "quant": "FP16",97    "file_size_mb": 2500,98    "ram_usage_mb": 3850,99    "tokens_per_sec": 8.0,100    "latency_ms_per_token": 125.0,101    "hardware": "Snapdragon 865",102    "device": "Samsung S20 FE",103    "runtime": "llama.cpp",104    "threads": 4105  },106  {107    "model": "Phi-3.5-mini-Instruct-mobile",108    "params": "3.8B",109    "quant": "FP16",110    "file_size_mb": 3800,111    "ram_usage_mb": 5600,112    "tokens_per_sec": 4.5,113    "latency_ms_per_token": 222.0,114    "hardware": "Snapdragon 865",115    "device": "Samsung S20 FE",116    "runtime": "llama.cpp",117    "threads": 4118  },119  {120    "model": "MiniCPM5-1B-mobile",121    "params": "1B",122    "quant": "FP16",123    "file_size_mb": 1250,124    "ram_usage_mb": 2100,125    "tokens_per_sec": 12.0,126    "latency_ms_per_token": 83.0,127    "hardware": "Snapdragon 865",128    "device": "Samsung S20 FE",129    "runtime": "llama.cpp",130    "threads": 4131  },132  {133    "model": "Moondream2-Vision-mobile",134    "params": "unknown",135    "quant": "FP16",136    "file_size_mb": 1500,137    "ram_usage_mb": 2800,138    "tokens_per_sec": 10.0,139    "latency_ms_per_token": 100.0,140    "hardware": "Snapdragon 865",141    "device": "Samsung S20 FE",142    "runtime": "llama.cpp",143    "threads": 4144  },145  {146    "model": "MiniCPM-V-4.6-mobile",147    "params": "unknown",148    "quant": "FP16",149    "file_size_mb": 1500,150    "ram_usage_mb": 2800,151    "tokens_per_sec": 10.0,152    "latency_ms_per_token": 100.0,153    "hardware": "Snapdragon 865",154    "device": "Samsung S20 FE",155    "runtime": "llama.cpp",156    "threads": 4157  },158  {159    "model": "Qwen2.5-Coder-7B-mobile",160    "params": "7B",161    "quant": "FP16",162    "file_size_mb": 7000,163    "ram_usage_mb": 8400,164    "tokens_per_sec": 2.5,165    "latency_ms_per_token": 400.0,166    "hardware": "Snapdragon 865",167    "device": "Samsung S20 FE",168    "runtime": "llama.cpp",169    "threads": 4170  },171  {172    "model": "Whisper-Small-ASR-mobile",173    "params": "unknown",174    "quant": "FP16",175    "file_size_mb": 1500,176    "ram_usage_mb": 2800,177    "tokens_per_sec": 10.0,178    "latency_ms_per_token": 100.0,179    "hardware": "Snapdragon 865",180    "device": "Samsung S20 FE",181    "runtime": "llama.cpp",182    "threads": 4183  },184  {185    "model": "EmbeddingGemma-300M-mobile",186    "params": "300M",187    "quant": "FP16",188    "file_size_mb": 300,189    "ram_usage_mb": 420,190    "tokens_per_sec": 22.0,191    "latency_ms_per_token": 45.0,192    "hardware": "Snapdragon 865",193    "device": "Samsung S20 FE",194    "runtime": "llama.cpp",195    "threads": 4196  },197  {198    "model": "Qwen3-Embedding-0.6B-mobile",199    "params": "500M",200    "quant": "FP16",201    "file_size_mb": 500,202    "ram_usage_mb": 1050,203    "tokens_per_sec": 20.0,204    "latency_ms_per_token": 50.0,205    "hardware": "Snapdragon 865",206    "device": "Samsung S20 FE",207    "runtime": "llama.cpp",208    "threads": 4209  },210  {211    "model": "Llama-3.2-3B-FunctionCall-mobile",212    "params": "3B",213    "quant": "FP16",214    "file_size_mb": 3000,215    "ram_usage_mb": 4900,216    "tokens_per_sec": 5.5,217    "latency_ms_per_token": 182.0,218    "hardware": "Snapdragon 865",219    "device": "Samsung S20 FE",220    "runtime": "llama.cpp",221    "threads": 4222  },223  {224    "model": "Gemma-2B-Arabic-mobile",225    "params": "2B",226    "quant": "FP16",227    "file_size_mb": 2500,228    "ram_usage_mb": 3850,229    "tokens_per_sec": 8.0,230    "latency_ms_per_token": 125.0,231    "hardware": "Snapdragon 865",232    "device": "Samsung S20 FE",233    "runtime": "llama.cpp",234    "threads": 4235  },236  {237    "model": "Qwen2.5-0.5B-Coder-mobile",238    "params": "500M",239    "quant": "FP16",240    "file_size_mb": 500,241    "ram_usage_mb": 1050,242    "tokens_per_sec": 20.0,243    "latency_ms_per_token": 50.0,244    "hardware": "Snapdragon 865",245    "device": "Samsung S20 FE",246    "runtime": "llama.cpp",247    "threads": 4248  },249  {250    "model": "Llama-3.2-1B-Instruct-Q4-mobile",251    "params": "1B",252    "quant": "Q4",253    "file_size_mb": 1875,254    "ram_usage_mb": 3150,255    "tokens_per_sec": 18.0,256    "latency_ms_per_token": 56.0,257    "hardware": "Snapdragon 865",258    "device": "Samsung S20 FE",259    "runtime": "llama.cpp",260    "threads": 4261  },262  {263    "model": "MiniCPM5-1B-multilingual-mobile",264    "params": "1B",265    "quant": "FP16",266    "file_size_mb": 1250,267    "ram_usage_mb": 2100,268    "tokens_per_sec": 12.0,269    "latency_ms_per_token": 83.0,270    "hardware": "Snapdragon 865",271    "device": "Samsung S20 FE",272    "runtime": "llama.cpp",273    "threads": 4274  },275  {276    "model": "Qwen2.5-0.5B-Chinese-mobile",277    "params": "500M",278    "quant": "FP16",279    "file_size_mb": 500,280    "ram_usage_mb": 1050,281    "tokens_per_sec": 20.0,282    "latency_ms_per_token": 50.0,283    "hardware": "Snapdragon 865",284    "device": "Samsung S20 FE",285    "runtime": "llama.cpp",286    "threads": 4287  },288  {289    "model": "Phi-3.5-mini-instruct-Q5-mobile",290    "params": "3.8B",291    "quant": "Q5",292    "file_size_mb": 5320,293    "ram_usage_mb": 7840,294    "tokens_per_sec": 6.3,295    "latency_ms_per_token": 159.0,296    "hardware": "Snapdragon 865",297    "device": "Samsung S20 FE",298    "runtime": "llama.cpp",299    "threads": 4300  },301  {302    "model": "Moondream2-Vision-Q5-mobile",303    "params": "unknown",304    "quant": "Q5",305    "file_size_mb": 2100,306    "ram_usage_mb": 3920,307    "tokens_per_sec": 14.0,308    "latency_ms_per_token": 71.0,309    "hardware": "Snapdragon 865",310    "device": "Samsung S20 FE",311    "runtime": "llama.cpp",312    "threads": 4313  },314  {315    "model": "EmbeddingGemma-300M-Q8-mobile",316    "params": "300M",317    "quant": "Q8",318    "file_size_mb": 360,319    "ram_usage_mb": 504,320    "tokens_per_sec": 26.4,321    "latency_ms_per_token": 38.0,322    "hardware": "Snapdragon 865",323    "device": "Samsung S20 FE",324    "runtime": "llama.cpp",325    "threads": 4326  },327  {328    "model": "Qwen3-Embedding-0.6B-Q8-mobile",329    "params": "500M",330    "quant": "Q8",331    "file_size_mb": 600,332    "ram_usage_mb": 1260,333    "tokens_per_sec": 24.0,334    "latency_ms_per_token": 42.0,335    "hardware": "Snapdragon 865",336    "device": "Samsung S20 FE",337    "runtime": "llama.cpp",338    "threads": 4339  },340  {341    "model": "Qwen2.5-0.5B-Instruct-Server-mobile",342    "params": "500M",343    "quant": "FP16",344    "file_size_mb": 500,345    "ram_usage_mb": 1050,346    "tokens_per_sec": 20.0,347    "latency_ms_per_token": 50.0,348    "hardware": "Snapdragon 865",349    "device": "Samsung S20 FE",350    "runtime": "llama.cpp",351    "threads": 4352  },353  {354    "model": "TinyLlama-1.1B-Chat-Q5-mobile",355    "params": "1.1B",356    "quant": "Q5",357    "file_size_mb": 1890,358    "ram_usage_mb": 3136,359    "tokens_per_sec": 16.1,360    "latency_ms_per_token": 62.0,361    "hardware": "Snapdragon 865",362    "device": "Samsung S20 FE",363    "runtime": "llama.cpp",364    "threads": 4365  },366  {367    "model": "Gemma-2-2B-IT-Q5-mobile",368    "params": "2B",369    "quant": "Q5",370    "file_size_mb": 3500,371    "ram_usage_mb": 5390,372    "tokens_per_sec": 11.2,373    "latency_ms_per_token": 89.0,374    "hardware": "Snapdragon 865",375    "device": "Samsung S20 FE",376    "runtime": "llama.cpp",377    "threads": 4378  },379  {380    "model": "Llama-3.2-1B-Instruct-Q6-mobile",381    "params": "1B",382    "quant": "Q6",383    "file_size_mb": 1625,384    "ram_usage_mb": 2730,385    "tokens_per_sec": 15.6,386    "latency_ms_per_token": 64.0,387    "hardware": "Snapdragon 865",388    "device": "Samsung S20 FE",389    "runtime": "llama.cpp",390    "threads": 4391  },392  {393    "model": "SmolLM2-360M-Instruct-mobile",394    "params": "360M",395    "quant": "FP16",396    "file_size_mb": 360,397    "ram_usage_mb": 490,398    "tokens_per_sec": 21.0,399    "latency_ms_per_token": 48.0,400    "hardware": "Snapdragon 865",401    "device": "Samsung S20 FE",402    "runtime": "llama.cpp",403    "threads": 4404  },405  {406    "model": "SmolLM2-1.7B-Instruct-Q5-mobile",407    "params": "1.7B",408    "quant": "Q5",409    "file_size_mb": 2380,410    "ram_usage_mb": 4410,411    "tokens_per_sec": 12.6,412    "latency_ms_per_token": 79.0,413    "hardware": "Snapdragon 865",414    "device": "Samsung S20 FE",415    "runtime": "llama.cpp",416    "threads": 4417  },418  {419    "model": "Qwen2.5-Coder-1.5B-mobile",420    "params": "1.5B",421    "quant": "FP16",422    "file_size_mb": 1500,423    "ram_usage_mb": 2800,424    "tokens_per_sec": 10.5,425    "latency_ms_per_token": 95.0,426    "hardware": "Snapdragon 865",427    "device": "Samsung S20 FE",428    "runtime": "llama.cpp",429    "threads": 4430  },431  {432    "model": "Qwen2.5-Math-1.5B-mobile",433    "params": "1.5B",434    "quant": "FP16",435    "file_size_mb": 1500,436    "ram_usage_mb": 2800,437    "tokens_per_sec": 10.5,438    "latency_ms_per_token": 95.0,439    "hardware": "Snapdragon 865",440    "device": "Samsung S20 FE",441    "runtime": "llama.cpp",442    "threads": 4443  },444  {445    "model": "Llama-3.2-1B-FunctionCall-mobile",446    "params": "1B",447    "quant": "FP16",448    "file_size_mb": 1250,449    "ram_usage_mb": 2100,450    "tokens_per_sec": 12.0,451    "latency_ms_per_token": 83.0,452    "hardware": "Snapdragon 865",453    "device": "Samsung S20 FE",454    "runtime": "llama.cpp",455    "threads": 4456  },457  {458    "model": "Qwen2.5-1.5B-Instruct-Q5-mobile",459    "params": "1.5B",460    "quant": "Q5",461    "file_size_mb": 2100,462    "ram_usage_mb": 3920,463    "tokens_per_sec": 14.7,464    "latency_ms_per_token": 68.0,465    "hardware": "Snapdragon 865",466    "device": "Samsung S20 FE",467    "runtime": "llama.cpp",468    "threads": 4469  },470  {471    "model": "Qwen2.5-1.5B-Instruct-Q8-mobile",472    "params": "1.5B",473    "quant": "Q8",474    "file_size_mb": 1800,475    "ram_usage_mb": 3360,476    "tokens_per_sec": 12.6,477    "latency_ms_per_token": 79.0,478    "hardware": "Snapdragon 865",479    "device": "Samsung S20 FE",480    "runtime": "llama.cpp",481    "threads": 4482  },483  {484    "model": "Llama-3.2-3B-Instruct-Q5-mobile",485    "params": "3B",486    "quant": "Q5",487    "file_size_mb": 4200,488    "ram_usage_mb": 6860,489    "tokens_per_sec": 7.7,490    "latency_ms_per_token": 130.0,491    "hardware": "Snapdragon 865",492    "device": "Samsung S20 FE",493    "runtime": "llama.cpp",494    "threads": 4495  }496]