[{"data":1,"prerenderedAt":81},["ShallowReactive",2],{"term-s\u002Fspeculative-decoding":3,"related-s\u002Fspeculative-decoding":60},{"id":4,"title":5,"acronym":6,"body":7,"category":40,"description":41,"difficulty":42,"extension":43,"letter":44,"meta":45,"navigation":46,"path":47,"related":48,"seo":54,"sitemap":55,"stem":58,"subcategory":6,"__hash__":59},"terms\u002Fterms\u002Fs\u002Fspeculative-decoding.md","Speculative Decoding",null,{"type":8,"value":9,"toc":33},"minimark",[10,15,19,23,26,30],[11,12,14],"h2",{"id":13},"eli5-the-vibe-check","ELI5 - The Vibe Check",[16,17,18],"p",{},"Speculative decoding is like having a fast, slightly reckless intern draft several sentences at once while the careful senior engineer just checks and approves them instead of writing every word from scratch. A small, cheap model guesses the next few tokens, and the big model verifies them all in a single pass. When the guesses are good, you get the big model's quality at a fraction of its usual speed.",[11,20,22],{"id":21},"real-talk","Real Talk",[16,24,25],{},"Speculative decoding accelerates autoregressive LLM inference by using a small draft model to propose multiple future tokens, which the larger target model then verifies in parallel in one forward pass instead of generating one token at a time. Accepted tokens are kept and rejected ones are regenerated by the target model, preserving the target model's exact output distribution while cutting wall-clock latency, often by two to three times.",[11,27,29],{"id":28},"when-youll-hear-this","When You'll Hear This",[16,31,32],{},"\"We turned on speculative decoding and cut p50 latency almost in half.\" \u002F \"The draft model's acceptance rate dropped on code, so the speedup shrank.\"",{"title":34,"searchDepth":35,"depth":35,"links":36},"",2,[37,38,39],{"id":13,"depth":35,"text":14},{"id":21,"depth":35,"text":22},{"id":28,"depth":35,"text":29},"ai","Speculative decoding is like having a fast, slightly reckless intern draft several sentences at once while the careful senior engineer just checks and appr...","advanced","md","s",{},true,"\u002Fterms\u002Fs\u002Fspeculative-decoding",[49,50,51,52,53],"Inference","Token","LLM","Time to First Token","Quantization",{"title":5,"description":41},{"changefreq":56,"priority":57},"weekly",0.7,"terms\u002Fs\u002Fspeculative-decoding","whUhbo3SFXyvtHWXLP5xuzK0MlI0B9Cnk9FvW_G5f_o",[61,65,70,73,77],{"title":49,"path":62,"acronym":6,"category":40,"difficulty":63,"description":64},"\u002Fterms\u002Fi\u002Finference","intermediate","Inference is when the AI actually runs and generates output — as opposed to training, which is when it's learning.",{"title":51,"path":66,"acronym":67,"category":40,"difficulty":68,"description":69},"\u002Fterms\u002Fl\u002Fllm","Large Language Model","beginner","An LLM is a humongous AI that read basically the entire internet and learned to predict what words come next, really really well.",{"title":53,"path":71,"acronym":6,"category":40,"difficulty":42,"description":72},"\u002Fterms\u002Fq\u002Fquantization","Quantization is the art of making AI models smaller and faster by using less precise numbers.",{"title":52,"path":74,"acronym":75,"category":40,"difficulty":63,"description":76},"\u002Fterms\u002Ft\u002Ftime-to-first-token","TTFT","Time to First Token is how long a user stares at a blank screen before the AI starts typing back.",{"title":50,"path":78,"acronym":6,"category":79,"difficulty":68,"description":80},"\u002Fterms\u002Ft\u002Ftoken","vibecoding","In AI-land, a token is a chunk of text — roughly 3\u002F4 of a word.",1792528125713]