[{"data":1,"prerenderedAt":81},["ShallowReactive",2],{"term-t\u002Ftensor-parallelism":3,"related-t\u002Ftensor-parallelism":60},{"id":4,"title":5,"acronym":6,"body":7,"category":40,"description":41,"difficulty":42,"extension":43,"letter":44,"meta":45,"navigation":46,"path":47,"related":48,"seo":54,"sitemap":55,"stem":58,"subcategory":6,"__hash__":59},"terms\u002Fterms\u002Ft\u002Ftensor-parallelism.md","Tensor Parallelism",null,{"type":8,"value":9,"toc":33},"minimark",[10,15,19,23,26,30],[11,12,14],"h2",{"id":13},"eli5-the-vibe-check","ELI5 - The Vibe Check",[16,17,18],"p",{},"Some models are too big to fit on a single GPU, full stop, no amount of quantization saves you. Tensor parallelism solves this by slicing individual weight matrices into pieces and spreading them across multiple GPUs, so each one holds only a fraction of every layer and they constantly compare notes mid-computation. It's less like giving each GPU its own task and more like several people jointly holding one enormous whiteboard.",[11,20,22],{"id":21},"real-talk","Real Talk",[16,24,25],{},"Tensor parallelism splits individual weight matrices within each transformer layer across multiple GPUs, requiring high-bandwidth interconnects (like NVLink) since every forward pass needs frequent all-reduce communication between devices. It differs from pipeline parallelism, which assigns whole layers to different GPUs, and from data parallelism, which replicates the full model and splits the batch. Frameworks like Megatron-LM and DeepSpeed implement it for training and serving models too large for a single device.",[11,27,29],{"id":28},"when-youll-hear-this","When You'll Hear This",[16,31,32],{},"\"The 70B model needs tensor parallelism across 4 GPUs just to load the weights.\" \u002F \"Without NVLink, tensor parallelism communication overhead kills your throughput.\"",{"title":34,"searchDepth":35,"depth":35,"links":36},"",2,[37,38,39],{"id":13,"depth":35,"text":14},{"id":21,"depth":35,"text":22},{"id":28,"depth":35,"text":29},"ai","Some models are too big to fit on a single GPU, full stop, no amount of quantization saves you.","advanced","md","t",{},true,"\u002Fterms\u002Ft\u002Ftensor-parallelism",[49,50,51,52,53],"Sharding","GPU","Mixture of Experts","Model Serving","Weights",{"title":5,"description":41},{"changefreq":56,"priority":57},"weekly",0.7,"terms\u002Ft\u002Ftensor-parallelism","Nq1RLmwRPdLIfoNbHfJaidAPfrH7W43cdBPqDZ1DCFA",[61,66,70,74,78],{"title":50,"path":62,"acronym":63,"category":40,"difficulty":64,"description":65},"\u002Fterms\u002Fg\u002Fgpu","Graphics Processing Unit","beginner","A GPU was originally built for rendering graphics in games, but turns out it's also perfect for AI.",{"title":51,"path":67,"acronym":68,"category":40,"difficulty":42,"description":69},"\u002Fterms\u002Fm\u002Fmixture-of-experts","MoE","Mixture of Experts is like having a team of specialists instead of one generalist.",{"title":52,"path":71,"acronym":6,"category":40,"difficulty":72,"description":73},"\u002Fterms\u002Fm\u002Fmodel-serving","intermediate","Model serving is the infrastructure that takes a trained AI model and makes it available as a fast, reliable API.",{"title":49,"path":75,"acronym":6,"category":76,"difficulty":42,"description":77},"\u002Fterms\u002Fs\u002Fsharding","database","Sharding splits your database across multiple servers based on some rule — like user IDs 1-1M on server 1, 1M-2M on server 2.",{"title":53,"path":79,"acronym":6,"category":40,"difficulty":72,"description":80},"\u002Fterms\u002Fw\u002Fweights","Weights are the numbers inside a neural network that determine what it knows and how it behaves — they're the AI's 'brain cells.",1792528129764]