{"slug":"gateway-overhead-benchmark-2026-10","title":"Gateway overhead benchmark, October 2026","asOf":"2026-10-25","columns":["host","path","p50_ms","p95_ms","p99_ms","added_p50_ms","rps"],"rows":[["quiet host","direct",0.58,1.07,3.56,0,3542],["quiet host","miss",2.1,4.49,7.24,1.52,641],["quiet host","exact_hit",1.08,1.95,4.89,0.5,1670],["quiet host","semantic_hit",1.22,2.08,4.7,0.64,null],["quiet host","streaming first byte (added)",1.98,3.58,4.27,1.98,null],["production host (shared, under load)","direct",0.66,1.59,4.21,0,3666],["production host (shared, under load)","miss",2.83,4.85,8.22,2.17,625],["production host (shared, under load)","exact_hit",1.24,2.3,4.9,0.58,1453],["quiet host","LiteLLM Proxy miss (same box, same mock)",16.98,21.81,null,16.25,null]],"summary":{"added_miss_p50_ms":1.52,"added_miss_p95_ms":3.42,"added_exact_hit_p50_ms":0.5,"streaming_added_first_byte_p50_ms":1.98,"litellm_added_p50_ms":16.25,"litellm_version":"LiteLLM: Current Version = 1.106.4","concurrency":32,"requests":2000,"node":"v24.22.1"},"generated_at":"2026-10-26T20:26:26.823Z","license":"CC-BY-4.0","canonical":"https://tokenzero.sh/reports/gateway-overhead-benchmark-2026-10"}