[{"data":1,"prerenderedAt":80},["ShallowReactive",2],{"term-t\u002Ftail-latency":3,"related-t\u002Ftail-latency":60},{"id":4,"title":5,"acronym":6,"body":7,"category":40,"description":41,"difficulty":42,"extension":43,"letter":44,"meta":45,"navigation":46,"path":47,"related":48,"seo":54,"sitemap":55,"stem":58,"subcategory":6,"__hash__":59},"terms\u002Fterms\u002Ft\u002Ftail-latency.md","Tail Latency",null,{"type":8,"value":9,"toc":33},"minimark",[10,15,19,23,26,30],[11,12,14],"h2",{"id":13},"eli5-the-vibe-check","ELI5 - The Vibe Check",[16,17,18],"p",{},"Your average response time might look great, but tail latency is about that unlucky slice of requests, the slowest 1 percent or so, that take way longer than everyone else. Here's the kicker: if a single user action fans out to 50 backend calls, even a rare 1-in-100 slow call means most of those user actions will hit at least one slow call. Averages lie, the tail is where users actually feel pain.",[11,20,22],{"id":21},"real-talk","Real Talk",[16,24,25],{},"Tail latency refers to the response times at the high percentiles of a distribution, typically p95, p99, or p999, which matter more than the mean for user experience because a single user request often fans out into many downstream calls, and the overall response time is bounded by the slowest of them. Systems with many parallel dependencies suffer 'tail latency amplification,' where even a small per-call tail probability compounds into a large fraction of end-to-end requests being slow, which is why SLOs are defined on percentiles, not averages.",[11,27,29],{"id":28},"when-youll-hear-this","When You'll Hear This",[16,31,32],{},"\"P50 looks fine, but p99 tail latency is 4 seconds, that's what's actually driving the complaints.\" \u002F \"With 40 parallel fan-out calls, tail latency amplification means almost every request hits at least one slow one.\"",{"title":34,"searchDepth":35,"depth":35,"links":36},"",2,[37,38,39],{"id":13,"depth":35,"text":14},{"id":21,"depth":35,"text":22},{"id":28,"depth":35,"text":29},"architecture","Your average response time might look great, but tail latency is about that unlucky slice of requests, the slowest 1 percent or so, that take way longer th...","intermediate","md","t",{},true,"\u002Fterms\u002Ft\u002Ftail-latency",[49,50,51,52,53],"Hedged Request","Latency Budget","Load Shedding","Circuit Breaker","Time to First Token",{"title":5,"description":41},{"changefreq":56,"priority":57},"weekly",0.7,"terms\u002Ft\u002Ftail-latency","fRJ_o-QIYf85rp4mQcRvPU0zcFz_dOU6Hi3efVtcIlM",[61,64,68,72,76],{"title":52,"path":62,"acronym":6,"category":40,"difficulty":42,"description":63},"\u002Fterms\u002Fc\u002Fcircuit-breaker","Circuit Breaker is like the electrical circuit breaker in your house.",{"title":49,"path":65,"acronym":6,"category":40,"difficulty":66,"description":67},"\u002Fterms\u002Fh\u002Fhedged-request","advanced","A hedged request is a bet-hedging trick against tail latency: if the first response hasn't come back within a short window, you fire off a duplicate reques...",{"title":50,"path":69,"acronym":6,"category":70,"difficulty":42,"description":71},"\u002Fterms\u002Fl\u002Flatency-budget","ai","A latency budget is deciding upfront how many milliseconds each part of your AI pipeline is allowed to spend before the whole thing feels slow to a user.",{"title":51,"path":73,"acronym":6,"category":74,"difficulty":66,"description":75},"\u002Fterms\u002Fl\u002Fload-shedding","backend","Load shedding is intentionally dropping some requests when your server is overwhelmed so it can keep serving the rest.",{"title":53,"path":77,"acronym":78,"category":70,"difficulty":42,"description":79},"\u002Fterms\u002Ft\u002Ftime-to-first-token","TTFT","Time to First Token is how long a user stares at a blank screen before the AI starts typing back.",1792528129173]