[{"data":1,"prerenderedAt":209},["ShallowReactive",2],{"term-l\u002Fllm-evals":3,"related-l\u002Fllm-evals":195},{"id":4,"title":5,"acronym":6,"body":7,"category":174,"description":175,"difficulty":176,"extension":177,"letter":178,"meta":179,"navigation":180,"path":181,"related":182,"seo":188,"sitemap":189,"stem":192,"subcategory":193,"__hash__":194},"terms\u002Fterms\u002Fl\u002Fllm-evals.md","LLM Evals","Evals",{"type":8,"value":9,"toc":168},"minimark",[10,15,19,23,26,30,157,161,164],[11,12,14],"h2",{"id":13},"eli5-the-vibe-check","ELI5 - The Vibe Check",[16,17,18],"p",{},"LLM evals are the test suite for a model or a prompt, except instead of asserting that 2 plus 2 equals 4, you're often asking 'is this answer actually good?' which is a much fuzzier question. Evals can be simple exact-match checks, or they can be another LLM acting as a judge, or a human rating outputs. Ship a prompt change without evals and you're just vibing in production.",[11,20,22],{"id":21},"real-talk","Real Talk",[16,24,25],{},"LLM evals are structured test suites that measure a model's or prompt's output quality against a dataset of inputs with defined success criteria, ranging from exact-match and rubric-based scoring to LLM-as-judge and human evaluation. Eval frameworks like promptfoo, Braintrust, and OpenAI Evals let teams catch regressions when swapping models, changing prompts, or updating RAG pipelines before those changes reach production.",[11,27,29],{"id":28},"show-me-the-code","Show Me The Code",[31,32,37],"pre",{"className":33,"code":34,"language":35,"meta":36,"style":36},"language-yaml shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","# promptfoo-style eval config\ntests:\n  - vars: { question: \"What is our refund window?\" }\n    assert:\n      - type: contains\n        value: \"30 days\"\n      - type: llm-rubric\n        value: \"answer is polite and on-topic\"\n","yaml","",[38,39,40,49,60,93,101,115,131,143],"code",{"__ignoreMap":36},[41,42,45],"span",{"class":43,"line":44},"line",1,[41,46,48],{"class":47},"sHwdD","# promptfoo-style eval config\n",[41,50,52,56],{"class":43,"line":51},2,[41,53,55],{"class":54},"swJcz","tests",[41,57,59],{"class":58},"sMK4o",":\n",[41,61,63,66,69,72,75,78,80,83,87,90],{"class":43,"line":62},3,[41,64,65],{"class":58},"  -",[41,67,68],{"class":54}," vars",[41,70,71],{"class":58},":",[41,73,74],{"class":58}," {",[41,76,77],{"class":54}," question",[41,79,71],{"class":58},[41,81,82],{"class":58}," \"",[41,84,86],{"class":85},"sfazB","What is our refund window?",[41,88,89],{"class":58},"\"",[41,91,92],{"class":58}," }\n",[41,94,96,99],{"class":43,"line":95},4,[41,97,98],{"class":54},"    assert",[41,100,59],{"class":58},[41,102,104,107,110,112],{"class":43,"line":103},5,[41,105,106],{"class":58},"      -",[41,108,109],{"class":54}," type",[41,111,71],{"class":58},[41,113,114],{"class":85}," contains\n",[41,116,118,121,123,125,128],{"class":43,"line":117},6,[41,119,120],{"class":54},"        value",[41,122,71],{"class":58},[41,124,82],{"class":58},[41,126,127],{"class":85},"30 days",[41,129,130],{"class":58},"\"\n",[41,132,134,136,138,140],{"class":43,"line":133},7,[41,135,106],{"class":58},[41,137,109],{"class":54},[41,139,71],{"class":58},[41,141,142],{"class":85}," llm-rubric\n",[41,144,146,148,150,152,155],{"class":43,"line":145},8,[41,147,120],{"class":54},[41,149,71],{"class":58},[41,151,82],{"class":58},[41,153,154],{"class":85},"answer is polite and on-topic",[41,156,130],{"class":58},[11,158,160],{"id":159},"when-youll-hear-this","When You'll Hear This",[16,162,163],{},"\"We don't ship a prompt change without running it against the eval suite first.\" \u002F \"The new model regressed on our evals for multi-step math.\"",[165,166,167],"style",{},"html pre.shiki code .sHwdD, html code.shiki .sHwdD{--shiki-light:#90A4AE;--shiki-light-font-style:italic;--shiki-default:#546E7A;--shiki-default-font-style:italic;--shiki-dark:#676E95;--shiki-dark-font-style:italic}html pre.shiki code .swJcz, html code.shiki .swJcz{--shiki-light:#E53935;--shiki-default:#F07178;--shiki-dark:#F07178}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":36,"searchDepth":51,"depth":51,"links":169},[170,171,172,173],{"id":13,"depth":51,"text":14},{"id":21,"depth":51,"text":22},{"id":28,"depth":51,"text":29},{"id":159,"depth":51,"text":160},"ai","LLM evals are the test suite for a model or a prompt, except instead of asserting that 2 plus 2 equals 4, you're often asking 'is this answer actually good...","intermediate","md","l",{},true,"\u002Fterms\u002Fl\u002Fllm-evals",[183,184,185,186,187],"Prompt Engineering","Hallucination","Fine-Tuning","Red Teaming","Guardrails",{"title":5,"description":175},{"changefreq":190,"priority":191},"weekly",0.7,"terms\u002Fl\u002Fllm-evals",null,"X6G3x5klM3ATU_HTrvliS29VKFAq9I_aYydwpF7pdkU",[196,199,203,206],{"title":187,"path":197,"acronym":193,"category":174,"difficulty":176,"description":198},"\u002Fterms\u002Fg\u002Fguardrails","Guardrails are the safety nets you put around AI applications — rules and checks that prevent the AI from going rogue.",{"title":184,"path":200,"acronym":193,"category":201,"difficulty":176,"description":202},"\u002Fterms\u002Fh\u002Fhallucination","vibecoding","When an AI confidently makes something up — like citing a library that doesn't exist or generating code that calls a function that was never written.",{"title":183,"path":204,"acronym":193,"category":201,"difficulty":176,"description":205},"\u002Fterms\u002Fp\u002Fprompt-engineering","Prompt engineering is the art of talking to AI so it actually does what you want.",{"title":186,"path":207,"acronym":193,"category":174,"difficulty":176,"description":208},"\u002Fterms\u002Fr\u002Fred-teaming","Red teaming in AI is trying to break the AI on purpose — like hiring someone to try to rob your bank so you can find the security holes.",1792528095421]