def recursive_split(text, max_chars, separators): if len(text) <= max_chars: return [text] for i, sep in enumerate(separators): if sep == "": return [text[j:j + max_chars] for j in range(0, len(text), max_chars)] if sep in text: parts = text.split(sep) chunks = [] current = "" for part in parts: candidate = current + (sep if current else "") + part if len(candidate) <= max_chars: current = candidate else: if current: chunks.append(current) current = part if current: chunks.append(current) result = [] next_seps = separators[i + 1:] for chunk in chunks: if len(chunk) > max_chars: result.extend(recursive_split(chunk, max_chars, next_seps)) else: result.append(chunk) return result return [text] doc1 = "Para one short.\n\nPara two also short.\n\nPara three is here." result1 = recursive_split(doc1, 25, ["\n\n", ". ", " ", ""]) print(f"paragraph case: {len(result1)} chunks") doc2 = "Sentence one. Sentence two is much longer than the others." result2 = recursive_split(doc2, 30, ["\n\n", ". ", " ", ""]) print(f"long-sentence case: {len(result2)} chunks") 2c:T5a7,def recursive_split(text, max_chars, separators): if len(text) <= max_chars: return [text] for i, sep in enumerate(separators): if sep == "": return [text[j:j + max_chars] for j in range(0, len(text), max_chars)] if sep in text: parts = text.split(sep) chunks = [] current = "" for part in parts: candidate = current + (sep if current else "") + part if len(candidate) <= max_chars: current = candidate else: if current: chunks.append(current) current = part if current: chunks.append(current) result = [] next_seps = separators[i + 1:] for chunk in chunks: if len(chunk) > max_chars: result.extend(recursive_split(chunk, max_chars, next_seps)) else: result.append(chunk) return result return [text] # define chunk_document(text, max_chars, overlap, separators) below doc = "First short paragraph.\n\nSecond paragraph here.\n\nAnd third one ends." chunks = chunk_document(doc, 22, 8, ["\n\n", ". ", " ", ""]) print(f"{len(chunks)} chunks, lengths: {[len(c) for c in chunks]}") print(f"first chunk: {chunks[0]!r}") print(f"second chunk starts with: {chunks[1][:23]!r}") 2d:T6d2,def recursive_split(text, max_chars, separators): if len(text) <= max_chars: return [text] for i, sep in enumerate(separators): if sep == "": return [text[j:j + max_chars] for j in range(0, len(text), max_chars)] if sep in text: parts = text.split(sep) chunks = [] current = "" for part in parts: candidate = current + (sep if current else "") + part if len(candidate) <= max_chars: current = candidate else: if current: chunks.append(current) current = part if current: chunks.append(current) result = [] next_seps = separators[i + 1:] for chunk in chunks: if len(chunk) > max_chars: result.extend(recursive_split(chunk, max_chars, next_seps)) else: result.append(chunk) return result return [text] def chunk_document(text, max_chars, overlap, separators): raw_chunks = recursive_split(text, max_chars, separators) if overlap <= 0 or len(raw_chunks) <= 1: return raw_chunks result = [raw_chunks[0]] for prev, current in zip(raw_chunks, raw_chunks[1:]): carry = prev[-overlap:] result.append(carry + current) return result doc = "First short paragraph.\n\nSecond paragraph here.\n\nAnd third one ends." chunks = chunk_document(doc, 22, 8, ["\n\n", ". ", " ", ""]) print(f"{len(chunks)} chunks, lengths: {[len(c) for c in chunks]}") print(f"first chunk: {chunks[0]!r}") print(f"second chunk starts with: {chunks[1][:23]!r}") 2e:Tc4b,# Embedding is where the bill starts Chunking is one-time work. Embedding is the cost that scales with your corpus AND with your traffic — every chunk gets embedded once, every user query gets embedded on every request. Pick the wrong model and a side project that should cost $4/month costs $400. Run the editor. We're faking the API with a stub `embed()` so this runs in your browser. The real call is one line of code: ```python # real version — needs an OpenAI key, doesn't run here. from openai import OpenAI client = OpenAI() vec = client.embeddings.create( model="text-embedding-3-small", input="cancel my subscription", ).data[0].embedding # list of 1536 floats ``` Same shape: text in, list of floats out. The numbers mean nothing to you. They mean "where this text sits in semantic space" to a vector database. ## The cost table that actually matters | Model | Dims | Cost per 1M tokens | Where it shines | |---|---|---|---| | `text-embedding-3-small` (OpenAI) | 1536 | **$0.02** | Default. 95% of use cases. | | `text-embedding-3-large` (OpenAI) | 3072 | $0.13 | Hard retrieval (legal, code, multilingual). 6.5× the cost. | | `voyage-3` (Voyage AI) | 1024 | $0.06 | Strong retrieval benchmarks, esp. domain-specific. | | `all-MiniLM-L6-v2` (sentence-transformers) | 384 | **$0 (local)** | Free if you run it yourself. Slower at query time. | | `cohere-embed-v3` | 1024 | $0.10 | Strong on long documents. | That is roughly the entire embedding market. The interesting line is the first one: `text-embedding-3-small` at **$0.02 per million tokens** is the default you should reach for first. A chunk of 200 tokens costs $0.000004 to embed. A 10,000-chunk corpus costs **4 cents** to embed end-to-end. ## The rule of thumb - **Corpus under 100k chunks, English, general topics**: `text-embedding-3-small`. Done. Move on. - **Specialized domain** (legal, medical, code, non-English): start with `text-embedding-3-small`, build a 50-question eval set (see the evals chapter), measure retrieval accuracy. Only upgrade to `large` if you're below 80%. - **Privacy-sensitive or air-gapped**: `all-MiniLM-L6-v2` running locally via sentence-transformers. Free, no data leaves your box. - **Massive corpus** (10M+ chunks): the $0.02 vs $0.13 gap becomes real money. Use `small` unless your eval set proves you need more. ## The mistake nobody catches The trap that bites every team: **you must embed your documents and your queries with the same model**. If you embed 10k chunks with `text-embedding-3-small` and then a junior engineer embeds the user query with `text-embedding-3-large`, the vectors aren't in the same space. Cosine similarity returns garbage. Retrieval picks random chunks. The model gets fed nonsense. You get a Slack message about "RAG hallucinating again." Step 6 is exactly this bug. You'll fix it. ## What you'll build A tiny similarity engine using cosine math by hand (Pyodide has no numpy in the default kernel, so we use stdlib). By the end you've got `rank_by_similarity(query_vec, doc_vecs)` and a 5-FAQ retrieval pipeline that returns the top-2 closest matches to a question.2f:T8b5,# Cosine similarity in 60 seconds You don't need linear algebra to use embeddings. You need one function. It's called cosine similarity and it answers one question: > Do these two vectors point in roughly the same direction? That's it. Two pieces of text that mean similar things produce vectors that point similar directions. Two unrelated pieces of text produce vectors pointing different directions. The formula is three lines of stdlib Python — run the editor. ## What the numbers mean | Score | Interpretation | |---|---| | 1.0 | Identical direction (vectors of the same text) | | 0.85–0.99 | Strong semantic match — usually a hit | | 0.5–0.85 | Related topic, possibly the right chunk | | 0.2–0.5 | Tangentially related — probably noise | | Below 0.2 | Unrelated — definitely noise | These ranges shift with the model. `text-embedding-3-small` tends to give higher absolute scores than `voyage-3`. **Always look at the relative order, not the absolute number.** The top-K matters, the score is a sanity check. ## Why "cosine" and not regular distance? Two reasons. First, embeddings have hundreds or thousands of dimensions, and in high-dimensional space, *direction* carries meaning while *magnitude* mostly carries length (more tokens → bigger vector). You want direction. Second, cosine is bounded — it's always between -1 and 1, so you can compare scores across queries. Euclidean distance is unbounded and depends on vector length. ## The pre-flight you have to remember Cosine similarity assumes vectors are pointing **out from the origin**. If one vector is at the origin (all zeros) the formula divides by zero. In practice this means: - Don't embed empty strings. - Don't embed strings of only whitespace. - Trim and validate your input before calling the API. A surprising number of RAG bugs are "we embedded an empty string once and now the index has a zero vector that matches every query equally poorly." ## What you'll do next Step 4: given three vectors and a query, predict which one is closest. Step 5: fill in the cosine formula yourself. Step 6 and 7: fix two bugs every junior engineer ships. Step 8: write the ranker. Step 9: build a tiny FAQ search engine end-to-end.30:T4a8,import math def cosine(a, b): dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(x * x for x in a)) nb = math.sqrt(sum(y * y for y in b)) return dot / (na * nb) # "model A" — password-ish on dim 0, cancel-ish on dim 1, export-ish on dim 2. def embed_a(text): t = text.lower() v = [0.0, 0.0, 0.0, 0.0] if "password" in t or "reset" in t: v[0] = 1.0 if "cancel" in t or "subscription" in t: v[1] = 1.0 if "export" in t or "data" in t: v[2] = 1.0 if not any(v): v[3] = 1.0 return v # "model B" — same words but mapped to DIFFERENT axes. different vector space. def embed_b(text): t = text.lower() v = [0.0, 0.0, 0.0, 0.0] if "password" in t or "reset" in t: v[2] = 1.0 if "cancel" in t or "subscription" in t: v[3] = 1.0 if "export" in t or "data" in t: v[0] = 1.0 if not any(v): v[1] = 1.0 return v docs = ["reset my password", "cancel my subscription", "export my data"] doc_vecs = [embed_a(d) for d in docs] # bug: query embedded with the WRONG model. mixes vector spaces. query_vec = embed_b("reset my password") for text, vec in zip(docs, doc_vecs): print(f"{cosine(query_vec, vec):.4f} {text}") 31:T404,import math def cosine(a, b): dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(x * x for x in a)) nb = math.sqrt(sum(y * y for y in b)) return dot / (na * nb) def top_k(query_vec, faqs, k): scored = [(text, cosine(query_vec, vec)) for text, vec in faqs] scored.sort(key=lambda t: -t[1]) return scored[:k] faqs = [ ("how do I reset my password?", [0.9, 0.1, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0]), ("how do I cancel my subscription?", [0.1, 0.9, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0]), ("how do I upgrade my plan?", [0.1, 0.7, 0.2, 0.0, 0.0, 0.1, 0.0, 0.1]), ("how do I export my data?", [0.0, 0.0, 0.9, 0.1, 0.0, 0.0, 0.1, 0.0]), ("what are your business hours?", [0.0, 0.0, 0.0, 0.0, 0.9, 0.1, 0.0, 0.1]), ] query_text = "I forgot my password" query_vec = [0.85, 0.1, 0.0, 0.0, 0.15, 0.0, 0.0, 0.05] results = top_k(query_vec, faqs, 2) print(f"query: {query_text!r}") print("top 2 matches:") for text, score in results: print(f" {score:.4f} {text}") 32:T9f1,# Top-k retrieves the closest chunks. The closest chunks are not always the right answer. You've already built the parts: chunked the document, embedded the chunks, embedded the query, computed cosine similarity. The naive last step is `sorted(scored, reverse=True)[:3]`. Take the top three. Done. That's the move that makes a demo work and production fail. The top-3 has three rows whether your corpus is full of the answer or empty of it. The third-best chunk in an empty-corpus query is useless garbage, and the model will dutifully use it. ## Two patterns you ship instead **Pattern 1: distance threshold.** Before slicing the top-k, drop any candidate whose score is below a cutoff (typical: 0.5 for cosine similarity on a well-aligned embedding model). Run the editor. `chunk_56` at 0.12 is clearly noise; the threshold drops it before the model ever sees it. The cutoff depends on the embedding model: - `text-embedding-3-small`: ~0.4-0.5 for "this is relevant" - `text-embedding-3-large`: ~0.5-0.6 (tighter distribution) - `voyage-3` and similar reranker-aware models: ~0.7+ - A model you fine-tuned: measure on a held-out set. The threshold is the most important number in your RAG pipeline. Most people never tune it. **Pattern 2: rerank.** Top-k by cosine gets you a candidate set of maybe 20 chunks. A reranker (a small model whose job is to score relevance directly, not as a side effect of embedding distance) re-ranks those 20 into the top 3 you actually send to the LLM. Cohere Rerank, Jina Reranker, and Voyage Rerank are the usual choices. They're cheap and they double the quality of retrieval without changing anything else in the pipeline. ## What goes wrong without these The four classic failure shapes for naive top-k: 1. **Tangential hits.** Question: "what's our refund policy?" Top-3 includes a chunk that mentions refunds but is about Q4 finance reporting. Model writes "Per our Q4 report, refunds are at 12%." 2. **Duplicates.** The same paragraph appears in two documents. Top-3 is [doc_a/p3, doc_b/p3, doc_a/p4]. You wasted a slot on a copy. 3. **Empty corpus.** No relevant chunk exists, top-3 returns three garbage chunks, the model treats them as authoritative and hallucinates from noise. 4. **Order matters.** The model attends more strongly to the FIRST chunk in the prompt than the third. If your sort is unstable, the answer quality flickers between identical-similarity ties. This lesson teaches the three fixes that catch all four failures: threshold, dedupe, stable sort.33:T9e2,# The threshold is the single most important number in your pipeline Run the editor. Naive top-5 returns all five chunks, including the clearly-noise `chunk_56` at 0.12. The thresholded version filters first, returning only the three relevant chunks even though you asked for up to 5. That's the property you want in production. **Top-k caps the upper bound. The threshold sets the lower bound.** Use both. ## The cutoff is per-model The right threshold depends on the embedding model's similarity distribution. Here's the calibration you'll actually do: 1. Take 50 query+chunk pairs you KNOW are relevant. Compute cosine. 2. Take 50 query+chunk pairs you KNOW are irrelevant. Compute cosine. 3. The threshold is the value that separates them best. For most modern embedding models, the right cutoff lands somewhere between 0.4 and 0.7. Anything lower than 0.3 is essentially "model guessed semantic similarity from grammar alone," not from meaning. Anything higher than 0.9 means "near-duplicate" — usually not what you want for retrieval breadth. ## What "score is below threshold" means in practice When the threshold filters every candidate (no result clears the bar), you have three options: 1. **Return empty.** The model gets no retrieved context. Your prompt should handle this case explicitly: "If you don't have enough information, say so." This is the right move. 2. **Return the best you have anyway.** The model will hallucinate from garbage. This is the bad move. 3. **Fall back to a different retriever** (keyword search, BM25, a broader corpus). This is the expensive-but-right move. Option 1 is what production RAG systems do. The "I don't know" response feels worse than a confident wrong answer, but it's catastrophically less expensive. ## The threshold's siblings: dedupe and stable sort A threshold catches noise. Dedupe catches redundancy. Stable sort catches order-flicker. You need all three before you ship. Dedupe pattern: ```python seen = set() deduped = [] for c, s in sorted(scored, key=lambda x: x[1], reverse=True): if c not in seen: seen.add(c) deduped.append((c, s)) ``` Stable sort is automatic in Python — `sorted` is stable, so ties preserve insertion order. That means if you sort by score and the top-3 has two 0.87s, they come out in the order they were inserted. Predictable. The reason this matters: when you re-run the same query twice and the top-3 reshuffles between calls, the LLM's answer shifts too. Stable sort kills that flicker.34:T461,import math def cosine(a, b): dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(x * x for x in a)) nb = math.sqrt(sum(y * y for y in b)) return dot / (na * nb) if na and nb else 0.0 query_vec = [1.0, 0.0, 0.0, 0.0] good_index = [ ("policy/refund_terms", [0.95, 0.05, 0.0, 0.0]), # ~1.0 ("policy/refund_terms", [0.92, 0.08, 0.0, 0.0]), # dup, lower ("policy/refund_eligibility", [0.80, 0.20, 0.0, 0.0]), # ~0.97 ("hr/onboarding", [0.10, 0.90, 0.0, 0.0]), # ~0.11 ] noisy_index = [ ("noise_1", [0.10, 0.50, 0.40, 0.0]), # ~0.18 ("noise_2", [0.05, 0.30, 0.60, 0.0]), # ~0.07 ("noise_3", [0.00, 0.10, 0.80, 0.0]), # ~0.00 ] one_hit_index = [ ("policy/refund_terms", [0.99, 0.01, 0.0, 0.0]), # ~1.0 ("hr/onboarding", [0.10, 0.90, 0.0, 0.0]), # ~0.11 ] # define rag_retrieve(query_vec, indexed, k, threshold) below print(rag_retrieve(query_vec, good_index, k=3, threshold=0.5)) print(rag_retrieve(query_vec, noisy_index, k=3, threshold=0.5)) print(rag_retrieve(query_vec, one_hit_index, k=3, threshold=0.5)) 35:T58d,import math def cosine(a, b): dot = sum(x * y for x, y in zip(a, b)) na = math.sqrt(sum(x * x for x in a)) nb = math.sqrt(sum(y * y for y in b)) return dot / (na * nb) if na and nb else 0.0 query_vec = [1.0, 0.0, 0.0, 0.0] good_index = [ ("policy/refund_terms", [0.95, 0.05, 0.0, 0.0]), ("policy/refund_terms", [0.92, 0.08, 0.0, 0.0]), ("policy/refund_eligibility", [0.80, 0.20, 0.0, 0.0]), ("hr/onboarding", [0.10, 0.90, 0.0, 0.0]), ] noisy_index = [ ("noise_1", [0.10, 0.50, 0.40, 0.0]), ("noise_2", [0.05, 0.30, 0.60, 0.0]), ("noise_3", [0.00, 0.10, 0.80, 0.0]), ] one_hit_index = [ ("policy/refund_terms", [0.99, 0.01, 0.0, 0.0]), ("hr/onboarding", [0.10, 0.90, 0.0, 0.0]), ] def rag_retrieve(query_vec, indexed, k, threshold): scored = [(cid, cosine(query_vec, v)) for cid, v in indexed] filtered = [(cid, s) for cid, s in scored if s >= threshold] ranked = sorted(filtered, key=lambda x: x[1], reverse=True) seen = set() out = [] for cid, s in ranked: if cid in seen: continue seen.add(cid) out.append(cid) if len(out) == k: break return out print(rag_retrieve(query_vec, good_index, k=3, threshold=0.5)) print(rag_retrieve(query_vec, noisy_index, k=3, threshold=0.5)) print(rag_retrieve(query_vec, one_hit_index, k=3, threshold=0.5)) 36:T1a69,# The decision that's killed more AI startups than any model swap This lesson is a theory lesson. There is barely any code. That is on purpose. The previous four lessons of this chapter taught you how to build RAG — chunking, embeddings, retrieval, ranking. This lesson zooms out and asks a different question: **should you be building RAG at all?** Every team building an AI product right now is, knowingly or not, picking some combination of architectural patterns for getting knowledge into the model. The combination they pick determines their cost curve, their freshness story, their latency budget, and whether they're still in business in 18 months. The choice is invisible in the demo. It shows up at scale. ## A note on the framing before we start What follows is one practitioner's heuristic, not a canonical taxonomy. The field has not settled on a single map of how to get knowledge into a language model at inference time, and credible practitioners disagree about how many distinct categories there are. You will see at least five candidates in serious writing on the topic: - RAG (retrieve at query) - long-context (stuff the prompt) - fine-tune (bake into weights) - tool-calling into live systems (retrieval as a tool the agent invokes) - knowledge-graph traversal and multi-hop agentic retrieval We teach the first three as primary buckets because they are the ones whose tradeoffs determine the architecture of most products today, and we treat hybrid (fine-tune base + RAG augmentation) as a real fourth class because the vertical-AI products that combine them are not edge cases. Tool-calling and KG traversal show up later in the curriculum (ch15 and ch16) as their own concerns. The rubric in step 07 names where this simplified map breaks down. ## The four buckets we'll teach to The architecture pivots on which of these your product depends on: 1. **Pull-from-source (RAG)**. Keep the knowledge in a vector store (or a SQL table, or a search index). At query time, retrieve the relevant chunks and stuff them into the prompt. The model never sees the corpus during training. It just sees today's slice. 2. **Stuff-the-context (long-context)**. Put the entire corpus in the prompt. Gemini's 2M-token context window made this a real option for whole-codebase and whole-contract use cases. With prompt caching, the marginal cost per query collapses. 3. **Bake-into-the-model (fine-tune)**. Train the model on your corpus so the knowledge lives in the weights. The model doesn't need to be told anything at inference time — it already "knows." Style, format, and domain language all transfer. 4. **Hybrid (fine-tune base + RAG augmentation)**. The combination most vertical AI products converge on once they hit scale — a fine-tuned base for style and domain language, plus RAG on top for facts that change. Operationally the most expensive of the four; commercially the most defensible for products where both style AND freshness matter (Harvey, vertical medical AI). These look like alternatives. They're really tradeoffs along five axes, and the right answer is almost always "use the combination whose tradeoffs match the product." ## The five tradeoff axes Every product evaluation reduces to these five questions: 1. **Freshness.** How often does the corpus change? Hourly? Quarterly? Never? RAG dominates when freshness is hours-to-days. Long-context dominates when the corpus is stable for the session. Fine-tune is hopeless when freshness matters at all — the knowledge in the weights is frozen at training time. 2. **Governance.** Can you point at the specific source the model used? RAG gives you citations for free (you know which chunks went in). Long-context gives you the whole document as the citation. Fine-tune gives you nothing — the model can't tell you which training example produced an output. 3. **Cost per call.** RAG's cost is retrieval + a small prompt. Long-context's cost is the full prompt every call, unless the prompt cache hits. Fine-tune's cost is a small inference call, but the training cost amortizes only if you call the model a lot. 4. **Latency budget.** RAG adds a retrieval hop (50-500ms). Long- context adds prompt-processing time (linear in token count, but cached on Anthropic's models if you're careful). Fine-tune is the lowest-latency option per call. 5. **Capability gap.** Is the model bad at your domain language, your formatting conventions, your house style? RAG can't fix that — it can only give the model the right facts. Fine-tune is the only fork that fixes capability gaps in the model itself. ## Why the wrong fork kills startups The graveyard is full of teams that picked the wrong fork. A few shapes of failure that keep repeating: - **Fine-tuned on facts.** The legal-tech that fine-tuned GPT-4 on case law in 2024 got crushed when GPT-5.5 with a 1M context window shipped in 2026. The fine-tune was a freshness liability — every new case meant retraining. Long-context (or RAG) would have aged better. - **RAG'd a stable corpus.** A company built a retrieval pipeline for a 200-page product manual that hasn't changed in three years. Six engineers, twelve months, a vector store. The whole manual fits in a 200K-token context window and would cost less per query with prompt caching. They built infra for a freshness problem they didn't have. - **Long-contexted a hot corpus.** A startup tried to keep a daily- updating customer database in the prompt. Cache misses on every update, cost went vertical, latency tanked. Should have been RAG. The "RAG is dead" provocation — a December 2024 talk by Nik Pash (Cline) in Jason Liu's RAG speaker series, later echoed in Liu's own 2025 "RAG is dead for agents" posts — landed because it pointed at one real failure mode (overbuilt RAG when long-context or agentic search would do) and got read as "stop using RAG at all." It's neither. It's: **pick the fork that matches the corpus**. ## What this lesson is The next five reading steps walk through each fork with named examples, then synthesize a five-axis rubric you can apply to any product. The two MC steps test whether you can pick the right fork for a scenario. The write and checkpoint steps codify the rubric into a `pick_fork(product_spec)` function you can paste into your own planning docs. By the end you should be able to read any AI product launch and predict which combination they picked, and whether the picked combination matches the corpus shape. That intuition is worth more than any specific RAG library you'll learn — and worth more than the rubric itself, which is a heuristic, not a decision procedure.37:T14a3,# Fork 1: RAG — when the corpus changes faster than you can retrain RAG is the right answer when your corpus has a heartbeat. New documents land daily. Old documents get edited. Records get added. Tickets get resolved. The truth lives somewhere that is NOT the model's training data, and it's moving. Glean is the canonical example. A 2026 enterprise search company at $200M+ ARR (as of late 2025, growing fast) whose entire moat is "we index your messy enterprise graph (Slack, Drive, Notion, Linear, Jira, Confluence) and the model answers from THAT, not from what it learned at training time." Glean cannot fine-tune. The customer's corpus changes every hour. Glean cannot stuff the whole thing in context — the average enterprise has hundreds of millions of documents. RAG is the only shape that fits. ## Why RAG wins on freshness The freshness story is simple: a new document hits the index, gets embedded, and is queryable the moment indexing completes. No retraining. No model swap. No engineer in the loop. The corpus and the model are decoupled. Compare that to fine-tune, where adding a new document means a training run, eval, deploy, and a rollback plan if quality drops. Or to long-context, where the new document either fits in the window (and gets included on every call, even when irrelevant) or doesn't fit at all. RAG is the freshness fork. If the answer to "how often does the corpus change?" is "more than once a week," RAG is almost always the right starting point. ## The classic case: a support bot reading docs that change weekly Imagine you ship a SaaS product. Your docs are 300 pages. Your eng team ships features twice a week, and the docs team updates 5-15 pages per release. Customers ask questions like "how do I configure SSO for Okta?" and the answer is in section 4.7, which got updated last Tuesday. A RAG bot: - Watches the docs repo. When a markdown file changes, re-chunks and re-embeds it. - At query time, embeds the user question, retrieves the top 5 chunks, and stuffs them in the prompt. - Cites the section the answer came from. The customer can click through to verify. The bot is freshness-aligned. The customer asks today's question and gets today's answer. The docs team's update is live in minutes. A fine-tuned bot can't do this. Every doc update needs a training run. By the time you've fine-tuned, the docs have changed again. A long-context bot could do it (stuff all 300 pages in the prompt every call) but at 300K tokens per call, you're paying $0.90 per question on Sonnet-class models, and your latency is 4-8 seconds per response. RAG cuts that to $0.02 and 1 second. ## The RAG cost curve RAG's cost has three components: 1. **Indexing cost.** Pay once per document update. Tiny per-doc (a few cents at most) but linear in corpus size. 2. **Retrieval cost.** Pay per query. Embeddings of the query (a fraction of a cent) plus a vector search (free if you self-host, pennies if you don't). 3. **Generation cost.** Pay per query. The prompt is small — just the retrieved chunks (5-20K tokens typically) plus the question. For Glean's shape (massive corpus, ~10 queries per user per day, millions of users), this curve is unbeatable. For a 200-page product manual queried 500 times a day, the curve looks different — retrieval still wins on freshness but loses on simplicity if the manual fits in context. ## When RAG breaks down RAG fails predictably in three shapes: - **The retrieval misses.** Your chunking shredded the answer across two chunks, the top-5 didn't include either, and the model hallucinates. This is the failure mode @@pd-path:path.ref:context-and-retrieval@@ spent four lessons fixing — chunking, embeddings, ranking, rerank. - **The corpus is too small to need it.** If your whole corpus fits in 200K tokens (most internal handbooks, most policy documents, most onboarding guides), you've built a retrieval pipeline for a freshness problem you didn't have. Just put the whole thing in the prompt and cache it. - **The corpus changes so fast that the index lags reality.** A trading floor where prices update every 100ms is not a RAG problem. It's a real-time inference problem, and you read from the live source at query time, not from an index. Anthropic's contextual retrieval paper (September 2024) is the production-grade RAG playbook. Anthropic added a step before chunking where each chunk is prefixed with a contextual summary ("this chunk is from section 4.7 of the SSO configuration docs and discusses the Okta SAML setup"). The technique reduced retrieval failures by 35% (embeddings alone), 49% (with BM25), or 67% (with reranking) on their benchmarks. If you're building serious RAG, you read that paper and copy the contextual-retrieval step before you touch anything else. ## What this fork teaches - **RAG is the freshness fork.** Pick it when the corpus has a heartbeat. Don't pick it when the corpus is stable. - **Glean is the proof point.** Multi-source, ever-changing, enterprise-grade. No other fork could have built that company. - **RAG is not "easier" than the other forks.** Production RAG is a 12-month engineering project. The "tutorial RAG" you see on YouTube ships in a weekend and breaks at scale.38:T176d,# Fork 2: Long-context — when the whole corpus fits and stays put The big shift in 2024-2026 was context windows getting genuinely large. Gemini 1.5 Pro shipped a 1M-token context window in February 2024. Gemini 1.5 Pro went to 2M in 2024; Gemini 2.0 Pro shipped with 2M in early 2025. Anthropic's Sonnet 4.6 sits at 1M. GPT-5.5 ships with 1M. This changed the calculus for a specific shape of product: **stable corpus, small enough to fit, queried many times per session**. Before long context, those products were RAG by default because nothing else fit. After long context, they're often better-served by just stuffing the whole thing in the prompt and letting prompt caching make it cheap. ## The "just shove it all in there" pattern The pattern is exactly what it sounds like: - Load the entire corpus into the system prompt at session start. - Use prompt caching so the corpus is processed once and cached. - Run as many queries against the cached prompt as the session needs. When it works, the per-query cost approaches the cost of just the question + the answer. The corpus, having been processed once and cached, is effectively free on the second through Nth query. Anthropic's prompt cache is the key infra here. Two important numbers to know: - **Cache TTL: 5 minutes (default).** A cached prompt stays warm for 5 minutes after its last use. The 1h option is available as an explicit opt-in (`"cache_control": {"type": "ephemeral", "ttl": "1h"}`) at 2× input write cost. The design question is the gap between cache reads: bursty sessions can fit the default window, while workflows with longer tool or human pauses may need the 1-hour TTL. - **Cache read cost: ~10% of base input cost.** When the cache hits, you pay roughly a tenth of what you'd pay to send those tokens fresh. This is what makes the pattern economical. The 5-minute TTL is the single most important number for designing long-context products. If your queries are bursty within a 5-min window (a user asking 10 questions about a contract in one sitting), the pattern is incredibly cheap. If queries are spread across hours, cache misses dominate and the cost explodes. ## The classic case: a legal-review bot loading the whole contract once A legal-tech startup builds a bot that reviews commercial contracts. The user uploads a contract (200K-500K tokens), then asks a series of questions: - "What's the termination clause?" - "Are there any indemnification terms I should flag?" - "Compare the liability cap to industry standard for SaaS deals." - "Pull out every renewal-related clause." Old shape (pre-long-context): chunk the contract, embed it, run a RAG pipeline on each question. This works, but you lose context across chunks. Asking about "renewal terms" might miss a clause buried in a definitions section that the chunker put in a different bucket. New shape (long-context): load the whole contract into a cached system prompt. Every question hits the full document. The model sees the whole thing on every query. Cross-document reasoning ("the indemnification cap in section 7 conflicts with the warranty in section 12") becomes possible because both sections are in context. Per-query cost on Sonnet 4.6 with cache hits: roughly $0.05-0.15 for a 300K-token contract being queried in a 5-min window. The same workload via RAG would have been $0.02-0.05 per query — RAG is still cheaper per call, but it can't answer the cross-document questions, and the architecture is 10× simpler. ## When long-context wins The dominance conditions for long-context: - **Corpus is stable for the session.** Once loaded, it doesn't change. Legal contracts, codebases-at-a-commit, single onboarding manuals, single research papers. - **Corpus fits in the window.** Modern windows are 1-2M tokens. Anything bigger needs RAG by physics. - **Query patterns are bursty.** Multiple queries within the 5-min cache TTL. A user doing a deep review session, not a customer asking one question and leaving. - **You need cross-corpus reasoning.** The model needs to see the whole document to answer the question. RAG fragments this. When all four conditions hit, long-context dominates RAG on both quality and architectural simplicity. The infra is "an API call." There is no vector store. There is no chunker. There is no embedding pipeline. There is no rerank step. ## When long-context breaks Long-context fails predictably: - **Corpus too big.** A 10M-token enterprise wiki doesn't fit, so you're back to RAG by physics. Most enterprise products live here. - **Cache misses dominate.** If queries are spread over hours, the 5-min TTL means most queries pay full freight. The cost explodes fast — a 500K-token prompt at full price is $1.50-3.00 per query on Sonnet-class models. - **The corpus changes mid-session.** Every change invalidates the cache, forcing a full re-process on the next query. - **The "needle in a haystack" failure.** Empirically, even large- context models lose track of specific facts when the prompt is near the limit. Gemini's own evals show retrieval accuracy degrading past 500K tokens. The model has the data, but doesn't reliably find it. The "RAG is dead" style of argument is really about long-context's encroachment on RAG's territory. It is useful on the trend (long-context took some small-corpus stable-doc use cases) and wrong as a blanket frame (RAG is still the right fork for freshness and huge-corpus territory). ## What this fork teaches - **Long-context is the simplicity fork.** When it fits, the infra is one API call. No vector store, no chunker, no rerank. - **The gap between cache reads is the make-or-break number.** Design for bursty queries within the default window, or opt into a longer TTL when long pauses are part of the product. - **2M tokens isn't infinite.** Most enterprise corpora still need RAG. Long-context took the small-corpus territory. RAG kept the big-corpus territory.39:T193d,# Fork 3: Fine-tune — when style or format is the product Fine-tuning is the most misunderstood fork. Most teams reach for fine-tune when they want the model to "know" their data. That is almost always the wrong reason. Fine-tune is bad at facts. Fine-tune is bad at freshness. Fine-tune is bad at citations. Fine-tune is bad at debugging. What fine-tune is good at: **changing how the model writes**. Domain language, formatting conventions, house style, output structure, tone. The capability gap, not the knowledge gap. ## Harvey AI is the canonical example Harvey AI, the legal-tech unicorn at $5B+ valuation, made fine- tuning legible to the AI industry. Harvey's earlier strategy (2023-early 2024) was to fine-tune on legal style. Their pitch wasn't "we know more case law than GPT-4." Their pitch was: **we know how lawyers write**. Specifically: - The structure of a legal memo (issue, rule, application, conclusion). - The voice ("the court found" vs "the court held"). - The citation format (Bluebook). - The hedge language ("plaintiffs may be entitled to" vs "the plaintiff is entitled to"). - The willingness to flag uncertainty in a specific way that matches how partner-track associates flag uncertainty. None of that is a knowledge gap. GPT-4 had read case law. What GPT-4 didn't have was the rhythm of how lawyers want their outputs to read. By May 2025, Harvey publicly pivoted to using multiple foundation models (Anthropic, Google, OpenAI) with task-specific routing and workflow orchestration, after frontier reasoning models beat their custom model on their own BigLaw Bench eval. The lesson is the same: fine-tune for style and format, not facts — but the receipts say even style-fine-tuning lost ground when frontier reasoning improved. This is the right shape for fine-tune. The product wedge is the style and format. The facts are still going to be wrong sometimes (no model is fact-clean on case law), but the OUTPUT looks authoritative in the way the customer needs. ## Why fine-tune fails for facts Fine-tune fails on facts for three reasons that compound: 1. **Frozen knowledge.** The model knows what was in its training data at training time. New case law decided yesterday is invisible to the model. RAG would have caught it; fine-tune can't. Every retraining cycle is a chance to fix this, but most teams retrain quarterly at best. 2. **No citations.** When the model outputs a fact, you cannot point to where the fact came from. Was it in your training set? Was it hallucinated? Was it adjacent to your training set and the model interpolated? Nobody knows. RAG hands you the chunk that produced the answer. Fine-tune hands you a vibe. 3. **Hallucination still happens.** Fine-tuning on facts does NOT make the model stop hallucinating. It makes the model hallucinate in your domain language, with your formatting, confidently. This is sometimes worse than baseline because the output looks authoritative. The teams that fine-tuned on facts in 2023-2024 mostly got crushed when retrieval-augmented systems caught up. The teams that fine- tuned on style and format kept their moat. ## The cautionary tale: the legal-tech that fine-tuned and got crushed A legal-tech startup (not Harvey) raised a $30M Series A in 2024 on the back of "we fine-tuned a model on US case law." Their demo was beautiful. The model could discuss landmark cases, cite them in Bluebook format, and write briefs that read like a junior associate wrote them. Two problems hit them in 2026: 1. **GPT-5.5 ships with 1M-token context and the legal-eval gap closed.** Long-context with proper case-law retrieval matched their fine-tuned model on style and BEAT it on facts (because the long-context system could pull in cases that hadn't existed when they fine-tuned). 2. **Their retraining cycle was 8 weeks.** Every 8 weeks they retrained. In between, the model was wrong about anything new. Their customers (BigLaw associates) noticed. The product's freshness reputation eroded. By Q2 2026 their renewal rate had fallen below 40%. They pivoted to a RAG-over-fine-tune-base architecture (fine-tune for style, RAG for current case law) but by then the market had moved. The lesson is not "don't fine-tune." The lesson is **fine-tune for the right axis**. Style, format, domain language — yes. Facts — no. ## When fine-tune wins The dominance conditions for fine-tune: - **The capability gap is in HOW the model writes.** Tone, structure, format, domain language. The model needs to produce outputs that match a specific voice or template. - **The output template is stable.** Bluebook citations don't change weekly. Legal memo structure doesn't change quarterly. The thing you're fine-tuning ON has to be stable enough that your training data is still relevant 6-12 months out. - **You have the data.** Fine-tuning on a few hundred examples rarely moves the needle. Harvey fine-tuned on millions. You need a corpus of well-formatted, vetted examples in the target style. - **You're willing to operate a retraining pipeline.** Fine-tune is the only fork that has an MLOps cost. You need an eval harness, a retraining cadence, rollback plans. ## The honest case for hybrid (fine-tune + RAG) The shape that wins in practice for vertical AI products is often **fine-tune for style + RAG for facts**. Harvey-shaped companies look like this internally: a fine-tuned base model that writes like a lawyer, with RAG pulling in current case law at query time. This is the architecture the legal-tech-that-got-crushed pivoted to. It's the one Harvey actually ships. It's the one most "AI for [vertical]" companies converge on after they've shipped enough to see where each fork wins. The decision rubric for hybrid is: pick fine-tune as the BASE if style is critical, pick RAG as the augmentation if freshness is critical. Most professional-services verticals (law, accounting, medical, consulting) have both, so they end up hybrid. ## What this fork teaches - **Fine-tune is the capability fork.** Pick it when style, format, or domain language is the product. Don't pick it for facts. - **Harvey is the proof point.** They fine-tuned on style and built a $5B company. The competitors who fine-tuned on facts mostly went under. - **Hybrid is the realistic answer for vertical AI.** Fine-tune the base for style, layer RAG on top for facts.3a:T40e,# 2027 scenario. three forks, three possible "futures get bad for me" stories. futures_2027 = { "RAG": { "context_windows": "10M tokens commodity, prompt cache TTL up to 1 hour", "infra": "vector stores commoditized, $0/mo on shared infra", "competitor_move": "long-context took the small-corpus market", }, "long_context": { "context_windows": "10M tokens commodity, prompt cache TTL up to 1 hour", "infra": "near-zero marginal cost on cached prompts", "competitor_move": "took most of RAG's small-stable-corpus territory", }, "fine_tune": { "training": "fine-tune-as-a-service: $50 to train, deploy in 5 min", "base_models": "GPT-6 + Claude 5 ship with 1M-token cached context that matches most fine-tunes on style", "competitor_move": "long-context + few-shot prompting closes most of the style gap", }, } for fork, conditions in futures_2027.items(): print(f"{fork}:") for k, v in conditions.items(): print(f" {k}: {v}") 3b:T9a2,# Inside the RAG fork: vector, graph, or both If you picked RAG, there's a second decision hiding inside it that vendors love to blur: *how* do you find the relevant material? The honest answer depends on the **shape of the question**, not on which technique is newer. **Direct lookup** — "what does the refund policy say?" The answer lives in one or two passages that *sound like* the question. Vector similarity search, the RAG you already know, handles this well. Done. **Multi-hop** — "where is Acme's *parent company* based?" No single chunk contains the answer. One document says Acme is owned by Globex; a different document says Globex is based in Berlin. Similarity search retrieves chunks that resemble the question — and no chunk resembles a question whose answer spans two documents. This is where retrieval built on *relationships* earns its keep: follow the `owned_by` edge, then the `based_in` edge. The editor does exactly that hop, in ten lines, over the same list-of-edge-dicts shape from @@pd-path:path.ref:read-the-data-shapes@@. **Neighborhood** — "show me everything connected to this incident." Also relationship-shaped: you want the edges around one node, not the passages similar to a phrase. **Global summary** — "what are the recurring themes across all our retros?" Neither trick works alone: similarity retrieves fragments, and edge-hopping needs you to know where to start. Summary questions want aggregation over the whole corpus, which is a different (and more expensive) pipeline entirely. ## The honest version of the "GraphRAG" pitch You'll hear that graph-based retrieval beats vector retrieval. **It doesn't, in general** — it beats vector retrieval *on relationship-shaped questions*, costs real effort to build (someone must extract those edges, with the provenance discipline from chapter 14), and does nothing extra for direct lookups, which are most questions in most products. Hybrid systems use vector search to find an entry point, then walk edges from there — paying both costs to cover both shapes. So the rubric-before-the-rubric: **listen to your users' questions.** Mostly direct lookups → vector RAG, stop there. A steady diet of "how is X related to Y" and "what's upstream of Z" → that's the signal to start extracting edges. Choosing by question shape is cheap. Choosing by hype costs you the pipeline you didn't need. Run the editor. Two hops, one answer no single document contained.3c:T1a6d,# The rubric — score any product on five axes and the fork usually narrows fast The case studies above are the intuition. Now the synthesis: a five-axis rubric you can run on any AI product spec and get a fork recommendation. This is the rubric the `pick_fork` function in the next step will encode. ## The five axes (again, with thresholds) For any product spec, score these five questions: 1. **Freshness pressure.** How often does the corpus change? - **Never / yearly**: 0 freshness pressure - **Monthly**: low - **Weekly or faster**: HIGH freshness pressure — RAG-leaning 2. **Corpus size.** How much text would the model need to see to answer any reasonable question? - **Under 200K tokens**: fits in context, long-context-eligible - **200K-2M tokens**: borderline, depends on cache patterns - **Over 2M tokens**: doesn't fit, RAG by physics 3. **Style criticality.** Does the product live or die by HOW the model writes? - **Generic prose is fine**: low style criticality - **Some domain language needed (handled by prompting)**: medium - **The output must read like an expert in the domain wrote it**: HIGH — fine-tune-leaning 4. **Cost budget per call.** What can you afford per query? - **Under $0.05/call**: tight — RAG or fine-tune - **$0.05-$0.50/call**: comfortable — long-context viable - **Over $0.50/call**: rich — any fork works on cost alone 5. **Latency budget.** How fast must the response come back? - **Under 1 second**: tight — fine-tune leads, RAG close, long-context risky - **1-5 seconds**: comfortable — all three viable - **Over 5 seconds**: rich — any fork works on latency alone ## How to score You don't add the axes. You read them as a profile and pick the fork whose strengths match. Some patterns: - **High freshness + huge corpus**: RAG, no question. Glean shape. - **Low freshness + small corpus + bursty queries**: long-context with cache. Single-contract review shape. - **High style criticality + stable output template**: fine-tune. Harvey shape. - **High style + needs current facts**: hybrid (fine-tune + RAG). Vertical AI shape. - **Tight cost + tight latency + small stable corpus**: fine-tune if you can amortize training, else RAG. The decision is rarely close. When you write down the five axes for a real product, three of them are loud and two are quiet, and the loud three usually point at one fork. ## Where teams get it wrong The three failure modes from step 01, mapped to the rubric: 1. **The legal-tech that fine-tuned on facts** had HIGH freshness pressure (case law updates constantly) and picked fine-tune, whose freshness is the worst of the three. They scored axis 1 wrong (assumed monthly was fine; it wasn't) and picked the fork that punished them for it. 2. **The team that RAG'd a stable 200-page manual** had ZERO freshness pressure and SMALL corpus. They should have scored axis 1 at zero and axis 2 at "fits in context" and picked long-context. They picked RAG because "everyone builds RAG." 3. **The startup that long-contexted a daily-changing customer DB** had HIGH freshness AND a tight latency budget. They should have picked RAG. They picked long-context because the demo was simpler, and the cost curve killed them at scale. The pattern: teams pick the fork they're familiar with, then bend the product to fit the fork. The rubric inverts that. Score the product first; the fork tends to fall out. ## What this rubric doesn't capture The five axes are useful but not exhaustive. The rubric is a heuristic, not a decision procedure. It deliberately ignores: - **Organizational capacity.** Does your team have the ML engineering bench to operate a fine-tune pipeline (data curation, training infra, eval harness, drift monitoring)? If not, fine-tune is more expensive than the axes alone suggest. - **Regulatory constraints.** Can you legally ship customer data through a third-party fine-tune API? Some industries (health, finance, EU consumer data) make fine-tune practically unavailable regardless of what the rubric says. - **Product-lifecycle stage.** A pre-PMF startup and a Fortune 500 mid-rollout get different defaults from the same scores. The startup wants the cheapest fastest path to validate the idea. The Fortune 500 wants the path that survives audit. - **The fourth option of "hybrid with a roadmap."** Sometimes the right answer is "RAG today, plan to migrate to long-context once the corpus stabilizes in six months" — a sequence the rubric won't suggest because it scores a single moment in time. Run the five axes first to narrow the field. Then check these four considerations before committing. When two of them push hard against the rubric's recommendation, the recommendation loses. ## When hybrid is the right answer Hybrid (fine-tune base + RAG augmentation) wins when TWO of the five axes point in different directions: - HIGH style criticality (push toward fine-tune) AND - HIGH freshness pressure (push toward RAG) This is exactly the vertical-AI shape (Harvey, certain medical AI products, certain accounting products). The style demand says fine-tune. The freshness demand says RAG. Hybrid lets both win. The cost of hybrid is operational complexity: you have a fine- tuning pipeline AND a retrieval pipeline AND the orchestration between them. Most teams should NOT pick hybrid until the single-fork version has shipped and shown a specific failure mode that hybrid fixes. ## The rubric, in one paragraph For any AI product, ask: how often does the corpus change, how big is it, does the model's writing style need to match an expert, what's the cost budget, what's the latency budget. If the corpus changes weekly or is over 2M tokens, RAG. If it's stable and under 2M and queries are bursty, long-context with cache. If style is the product, fine-tune. If freshness AND style both matter, hybrid. The five numbers narrow the field fast — then check the org-capacity, regulatory, lifecycle, and roadmap considerations before committing. ## What the next two steps do Step 08 codifies this rubric into a `pick_fork(product_spec)` function. You write the score-based rules; the function returns one of `"rag"`, `"long_context"`, `"fine_tune"`, or `"hybrid"`. Step 09 inverts the rubric: given a product AND a chosen fork, does the fork fit? `evaluate_strategy(product, fork)` returns `(ok, reason)`. This is the rubric you'd run on someone else's pitch deck to decide whether their architecture survives contact with reality. By the end you have a rubric you can paste into a planning doc and a function that codifies it. That's worth more than any specific RAG library you'll learn.3d:T4ca,Write `pick_fork(spec)` that takes a product spec (dict) and returns one of: `"rag"`, `"long_context"`, `"fine_tune"`, `"hybrid"`. Spec fields you'll be given: - `corpus_size_tokens`: int (how big the corpus is in tokens) - `update_frequency_days`: int (how often the corpus changes; 9999 means "essentially never") - `style_critical`: bool (does the output need to read like a domain expert wrote it?) - `cost_per_call_max`: float (USD ceiling per query) - `latency_budget_ms`: int (max acceptable latency) Apply the rubric in this order (first match wins): 1. If `corpus_size_tokens > 2_000_000`: return `"rag"` (doesn't fit in any context window, RAG by physics). 2. If `style_critical` AND `update_frequency_days <= 7`: return `"hybrid"` (style + freshness both matter). 3. If `style_critical`: return `"fine_tune"` (style is the product, corpus is stable enough). 4. If `update_frequency_days <= 7`: return `"rag"` (freshness pressure, no style demand). 5. If `corpus_size_tokens <= 2_000_000`: return `"long_context"` (stable, fits in window). 6. Fallback: return `"rag"`. Two products demoed. Expected output: ``` Glean (enterprise search): rag Harvey (legal AI): hybrid ``` 3e:T54a,Final drill. Invert the rubric. Given a product spec AND a fork someone picked, decide whether the fork fits the product. Write `evaluate_strategy(product, fork)` that returns a tuple `(ok: bool, reason: str)`. The product spec has the same fields as in step 08: `corpus_size_tokens`, `update_frequency_days`, `style_critical`, `cost_per_call_max`, `latency_budget_ms`. The fork is one of: `"rag"`, `"long_context"`, `"fine_tune"`, `"hybrid"`. Apply these checks (first failure wins; if none fail, return ok=True): - If fork is `"long_context"` AND `corpus_size_tokens > 2_000_000`: return `(False, "corpus too large for long-context window")` - If fork is `"fine_tune"` AND `update_frequency_days <= 7`: return `(False, "freshness too high for fine-tune")` - If fork is `"rag"` AND `corpus_size_tokens <= 200_000` AND `update_frequency_days >= 90`: return `(False, "RAG overkill for small stable corpus")` - If fork is `"hybrid"` AND NOT `style_critical`: return `(False, "hybrid wasted without style demand")` - Otherwise return `(True, "fork fits product")` Three product-fork pairs run. Expected output: ``` Harvey + fine_tune: (False, 'freshness too high for fine-tune') Glean + long_context: (False, 'corpus too large for long-context window') Handbook bot + rag: (False, 'RAG overkill for small stable corpus') ``` 3f:T5ee,harvey = { "name": "Harvey", "corpus_size_tokens": 1_500_000, "update_frequency_days": 3, "style_critical": True, "cost_per_call_max": 0.50, "latency_budget_ms": 5000, } glean = { "name": "Glean", "corpus_size_tokens": 50_000_000_000, "update_frequency_days": 1, "style_critical": False, "cost_per_call_max": 0.05, "latency_budget_ms": 2000, } handbook_bot = { "name": "Handbook bot", "corpus_size_tokens": 80_000, "update_frequency_days": 365, "style_critical": False, "cost_per_call_max": 0.05, "latency_budget_ms": 2000, } def evaluate_strategy(product, fork): size = product.get("corpus_size_tokens", 0) freshness = product.get("update_frequency_days", 9999) style = product.get("style_critical", False) if fork == "long_context" and size > 2_000_000: return (False, "corpus too large for long-context window") if fork == "fine_tune" and freshness <= 7: return (False, "freshness too high for fine-tune") if fork == "rag" and size <= 200_000 and freshness >= 90: return (False, "RAG overkill for small stable corpus") if fork == "hybrid" and not style: return (False, "hybrid wasted without style demand") return (True, "fork fits product") print(f"Harvey + fine_tune: {evaluate_strategy(harvey, 'fine_tune')}") print(f"Glean + long_context: {evaluate_strategy(glean, 'long_context')}") print(f"Handbook bot + rag: {evaluate_strategy(handbook_bot, 'rag')}") 40:Ta0e,# "It feels relevant" is not a metric You tuned the threshold, deduped, stabilized the sort. Then you typed five queries into your own retriever, nodded at the results, and called it good. That's not evaluation. That's vibes with extra steps — and vibes don't tell you whether last Tuesday's re-chunking made retrieval better or quietly broke it. The fix costs an afternoon: an **eval set**. Twenty or thirty real queries, each paired with the chunk ids a human says are the right answer. That's the whole artifact — a list of (query, gold ids) pairs. Build it from the question shapes your users actually ask (the fork lesson's rubric applies here too: mostly direct lookups → your eval set should be mostly direct lookups). Then three numbers fall out: - **recall@k** — of the chunks a human marked relevant, what fraction showed up in your top-k? Missing gold chunks means the model never sees the answer. This is the number that predicts hallucination. - **precision@k** — of the k chunks you sent, what fraction was relevant? Low precision means you're paying tokens to feed the model distractions. - **MRR** (mean reciprocal rank) — how high did the *first* correct chunk rank, averaged as 1/rank? Rank 1 scores 1.0, rank 3 scores 0.33. It matters because the model attends hardest to the first chunk in the prompt. Run the editor. One retrieved chunk is gold, one gold chunk is missing, one retrieved chunk is filler — and the two metrics disagree about how bad that is. They usually do. Recall and precision pull in opposite directions as you turn the k knob, which is exactly why you need both. ## The other half: what a vector DB row actually is Strip the marketing and a vector database stores rows of exactly three things: ```python {"id": "policy/p2", "vec": [0.12, -0.4, ...], "meta": {"tenant": "acme", "year": 2026}} ``` An **id**, a **vector**, and a **payload** (metadata). That's it. The product you pay for adds fast approximate search over millions of vectors — but the *shape* is a list of dicts, which is why this lesson can model one honestly in fifteen lines of stdlib Python. The payload is where the second decision of this lesson lives: **filter-then-search or search-then-filter?** When a query says "acme's refund policy," you can filter rows to `tenant == "acme"` first and rank only those — or rank everything and filter the top-k afterward. One of these silently returns nothing when the global top-k happens to be full of other tenants' documents. You'll break and fix that exact bug in a few steps.41:T636,def dot(a, b): return sum(x * y for x, y in zip(a, b)) rows = [ {"id": "acme/refunds", "vec": [0.9, 0.1, 0.0], "meta": {"tenant": "acme"}}, {"id": "acme/sla", "vec": [0.1, 0.9, 0.1], "meta": {"tenant": "acme"}}, {"id": "acme/pricing", "vec": [0.1, 0.2, 0.9], "meta": {"tenant": "acme"}}, {"id": "acme/onboarding", "vec": [0.4, 0.5, 0.1], "meta": {"tenant": "acme"}}, {"id": "globex/refunds", "vec": [0.95, 0.05, 0.0], "meta": {"tenant": "globex"}}, ] eval_set = [ {"q": [1.0, 0.0, 0.0], "tenant": "acme", "relevant": {"acme/refunds"}}, {"q": [0.0, 1.0, 0.0], "tenant": "acme", "relevant": {"acme/sla", "acme/pricing", "acme/onboarding"}}, {"q": [0.4, 0.5, 0.1], "tenant": "acme", "relevant": {"acme/onboarding"}}, ] def search(q_vec, rows, tenant, k): mine = [r for r in rows if r["meta"]["tenant"] == tenant] ranked = sorted(mine, key=lambda r: dot(q_vec, r["vec"]), reverse=True) return [r["id"] for r in ranked[:k]] def evaluate(eval_set, rows, k): recall_total = 0.0 rr_total = 0.0 for case in eval_set: ids = search(case["q"], rows, case["tenant"], k) hits = sum(1 for cid in ids if cid in case["relevant"]) recall_total += hits / len(case["relevant"]) for rank, cid in enumerate(ids, start=1): if cid in case["relevant"]: rr_total += 1 / rank break n = len(eval_set) return recall_total / n, rr_total / n r, m = evaluate(eval_set, rows, k=2) print(f"recall@2: {r:.2f}") print(f"mrr@2: {m:.2f}") 1c:[["$","$L24",null,{"payload":{"catalog":"Chunking that respects structure — don't shred your own documents (step 2/9) · context and retrieval · promptdojo","byPath":{"developer":"Chunking that respects structure — don't shred your own documents (step 2/9) · context and retrieval · promptdojo","copywriter":"Chunking that respects structure — don't shred your own documents (step 2/9) · context and retrieval · promptdojo","data-analyst":"Chunking that respects structure — don't shred your own documents (step 2/9) · context and retrieval · promptdojo","lawyer":"Chunking that respects structure — don't shred your own documents (step 2/9) · context and retrieval · promptdojo"}}}],["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"$25"}}],["$","$L26",null,{"tree":{"toc":{"version":"0.1.0","generatedAt":"1970-01-01T00:00:00.000Z","chapters":[{"number":0,"slug":"before-you-build","title":"from worker to builder","blurb":"stop doing repeated work by hand. this chapter is the map: the builder loop, old workplace deliverables turned into site artifacts, and the reusable tool card. if you are starting here, those moves are the lesson. if you already ran a studio, you already used the loop.","status":"live","lessonCount":10,"stepCount":14,"liveLessonCount":10,"liveStepCount":14,"estMinutes":74,"xpTotal":36},{"number":1,"slug":"read-the-script","title":"read the script — your first code-reading lab","blurb":"ai hands you small scripts. this lab covers the three reads that matter: what the names hold, whether the function actually returns, and which branch of the if-chain really fires. if you are starting here, this labels the reads. if you already ran a studio, you already wrote these.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":21,"xpTotal":78},{"number":2,"slug":"read-the-data-shapes","title":"read the data shapes — the json ai just dumped","blurb":"every api response, config file, and llm reply is lists and dicts nested in a tree. read the shape on sight, walk it with a loop, and catch the wrong-layer bug ai loves to ship.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":23,"xpTotal":80},{"number":3,"slug":"read-the-failure","title":"read the failure — cursor wrote this and crashed","blurb":"when python crashes, it tells you exactly what happened and where. most non-engineers panic at the wall of text — and ai 'fixes' the crash by hiding it. learn to read the failure, then catch only what's worth catching.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":24,"xpTotal":83},{"number":4,"slug":"read-the-pipeline","title":"read the pipeline — files in, api calls out","blurb":"every real script reads a file or calls an api — usually both. learn the with-block, the status-code families, and the retry rules: the seatbelts ai forgets to put on.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":26,"xpTotal":83},{"number":7,"slug":"mutation-and-state","title":"mutation — why your code mysteriously breaks","blurb":"when a list inside a function changes the list outside the function, that's mutation. ai does this constantly without flagging it, and it's the bug class that takes the longest to find.","status":"live","lessonCount":2,"stepCount":9,"liveLessonCount":2,"liveStepCount":9,"estMinutes":10,"xpTotal":32},{"number":13,"slug":"llm-apis","title":"llm apis — talking to claude and openai","blurb":"every ai feature you ship eventually calls a model api. learn the messages pattern, how to read the response, and the four lines ai writes every single time.","status":"live","lessonCount":4,"stepCount":27,"liveLessonCount":4,"liveStepCount":27,"estMinutes":45,"xpTotal":87},{"number":14,"slug":"structured-output","title":"structured output — making the model return json","blurb":"free-form text breaks every pipeline. learn the schema-first pattern ai uses to get reliable json back, validate it with pydantic, and catch the model's lies before they hit prod.","status":"live","lessonCount":3,"stepCount":25,"liveLessonCount":3,"liveStepCount":25,"estMinutes":38,"xpTotal":80},{"number":15,"slug":"mcp","title":"mcp — the model context protocol","blurb":"mcp is the new standard for plugging tools and data sources into ai agents. learn what an mcp server actually is, how claude code lists tools, and why this is replacing one-off integrations everywhere.","status":"live","lessonCount":3,"stepCount":26,"liveLessonCount":3,"liveStepCount":26,"estMinutes":40,"xpTotal":85},{"number":16,"slug":"agent-loops","title":"agent loops — tool_use and the request/tool/respond cycle","blurb":"an agent isn't magic. it's a while loop. learn the actual cycle claude code, cursor, and every other agent uses: model returns tool_use, you run the tool, you send the result back, repeat until end_turn.","status":"live","lessonCount":6,"stepCount":48,"liveLessonCount":6,"liveStepCount":48,"estMinutes":71,"xpTotal":154},{"number":17,"slug":"git-and-github","title":"git and github cli — the ai builder's actual workflow","blurb":"cursor and claude code commit on your behalf. reading those commits — and undoing the bad ones — is your job. learn the three-state model, the commands you'll run every day, and what `gh` does that `git` can't.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":36,"xpTotal":79},{"number":18,"slug":"secrets-and-env","title":"secrets — .env, api keys, and what not to commit","blurb":"ai ships keys to github all the time. learn the .env pattern, why os.getenv is non-negotiable, what to do when a key leaks, and the gitignore lines you need on day one.","status":"live","lessonCount":2,"stepCount":16,"liveLessonCount":2,"liveStepCount":16,"estMinutes":16,"xpTotal":58},{"number":19,"slug":"prompting","title":"prompting ai coding tools","blurb":"an AI coding tool writes and edits with you — Cursor is one brand, Claude Code is another. the difference between a one-shot session and a four-hour spiral is almost always the first prompt.","status":"live","lessonCount":5,"stepCount":36,"liveLessonCount":5,"liveStepCount":36,"estMinutes":59,"xpTotal":118},{"number":20,"slug":"agent-traces","title":"reading agent traces and telemetry","blurb":"when an agent fails, the trace tells you exactly where. learn to read tool calls, tool results, and stop reasons — the json breadcrumbs every agent leaves behind.","status":"live","lessonCount":3,"stepCount":25,"liveLessonCount":3,"liveStepCount":25,"estMinutes":30,"xpTotal":88},{"number":21,"slug":"evals","title":"test the agent before you trust it","blurb":"same input, check the output, pass or fail. don't ship on a demo that felt right.","status":"live","lessonCount":5,"stepCount":34,"liveLessonCount":5,"liveStepCount":34,"estMinutes":57,"xpTotal":112},{"number":22,"slug":"context-and-retrieval","title":"context and retrieval — feeding the model real data","blurb":"rag without the overengineering. chunking, embeddings, vector search, and the small set of patterns that make a model answer from your data instead of its training set.","status":"live","lessonCount":5,"stepCount":44,"liveLessonCount":5,"liveStepCount":44,"estMinutes":75,"xpTotal":144},{"number":23,"slug":"production-tradeoffs","title":"production tradeoffs — cost, latency, quality","blurb":"the three numbers every shipped llm feature lives or dies by. token math, caching, streaming, batching, and the small set of decisions that move the product more than a model swap ever will.","status":"live","lessonCount":4,"stepCount":33,"liveLessonCount":4,"liveStepCount":33,"estMinutes":57,"xpTotal":111},{"number":24,"slug":"debugging-output","title":"debugging broken ai output","blurb":"when the model lies to your customer. the methodology for narrowing down what went wrong, the four most-common breakage classes, and the discipline that separates 'we shipped a fix' from 'we blamed the model and shrugged'.","status":"live","lessonCount":4,"stepCount":34,"liveLessonCount":4,"liveStepCount":34,"estMinutes":61,"xpTotal":112},{"number":25,"slug":"capstone","title":"mid-path build — ship the CLI agent","blurb":"wire it all together. the prompt, the call, the validation, the trace, the eval, the MCP tool. a path checkpoint, not the end of the course: the smallest end-to-end llm feature you could ship to a real user. retrieval and prompt-cache cost work live in later chapters — extend this build with them when you scale past the demo input set.","status":"live","lessonCount":7,"stepCount":62,"liveLessonCount":7,"liveStepCount":62,"estMinutes":100,"xpTotal":219},{"number":26,"slug":"agent-harnesses","title":"agent harnesses — the layer between you and the raw api","blurb":"claude code, cursor, aider, codex cli — they're all the same four layers wrapped around the same model api. learn what those layers are, what each adds, and what you'd build yourself if you had to. when the harness changes the world, the rollback lives in the repo — a runbook a junior can execute at 3am.","status":"live","lessonCount":5,"stepCount":37,"liveLessonCount":5,"liveStepCount":37,"estMinutes":69,"xpTotal":119},{"number":27,"slug":"ai-image-generation","title":"ai image generation — from prompt to production asset","blurb":"the 2026 image model landscape, the prompts that work, and the pipeline that turns one good idea into a hundred ready-to-ship images. nano banana 2 for volume, nano banana pro for fidelity, flux, midjourney, ideogram, gpt-image-2. when each wins and what they cost.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":60,"xpTotal":67},{"number":28,"slug":"ai-video-generation","title":"ai video generation — omni, veo 3.1, sora, and the second wave","blurb":"video is the hardest content type to generate, the most expensive, and the most strategically interesting. learn the 2026 model lineup, the camera-control patterns that separate slop from craft, and the cost math that decides whether your idea is viable.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":50,"xpTotal":61},{"number":29,"slug":"programmatic-design","title":"programmatic design — code-driven video and the new design pipeline","blurb":"ai generates raw assets; code stitches them into something shippable. hyperframes, remotion, claude design — when each tool wins, how they combine, and the data-driven workflows that turn one template into a hundred videos.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":46,"xpTotal":66},{"number":30,"slug":"harness-engineering","title":"harness engineering — the discipline behind the model","blurb":"every coding agent is a model plus a harness. the model is bought; the harness is engineered. learn the craft: how to ratchet rules from failures, fight context rot, design long-horizon loops, wire hooks as enforcement, and read the haas shift that's reshaping what you build vs buy.","status":"live","lessonCount":6,"stepCount":52,"liveLessonCount":6,"liveStepCount":52,"estMinutes":126,"xpTotal":143},{"number":31,"slug":"intro-to-terminal","title":"intro to terminal — the text way to drive your computer","blurb":"the terminal is a text box: type a name, press enter, the computer does it. by the end you can move around your files, make folders, and read files without the mouse. every later tool chapter assumes this.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":26,"xpTotal":50},{"number":32,"slug":"intro-to-claude-cli","title":"intro to claude cli — the door to real building","blurb":"you've used claude in a chat window. the claude cli is the same model with its hands on your actual files. this chapter installs it, signs you in, and runs your first real command. by the end you've watched an ai read, plan, and change things on your machine, and you know when to reach for the cli instead of the chat box.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":32,"xpTotal":48},{"number":33,"slug":"intro-to-codex-cli","title":"intro to openai codex cli — a second tool in the kit","blurb":"you know the claude cli. the openai codex cli does the same job, an ai working in your terminal on your real files, with a different company behind it. this chapter installs it, signs you in, and runs your first command. most of what you already know carries straight over, so this chapter is mostly about what is different and when to reach for which.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":32,"xpTotal":45},{"number":34,"slug":"team-skills","title":"claude skills for teams — playbooks your whole team shares","blurb":"a claude skill is a packaged set of instructions — your team's playbook — that claude loads when it's relevant, so nobody has to re-explain it. this chapter is for people who manage teams. it covers what a skill is, how a team shares and provisions skills, real examples for hr, legal, and ops work, when a skill beats a one-off prompt, and the governance you need before any skill touches real work.","status":"live","lessonCount":4,"stepCount":28,"liveLessonCount":4,"liveStepCount":28,"estMinutes":41,"xpTotal":60},{"number":35,"slug":"dataframes-numpy-pandas","title":"dataframes with numpy and pandas","blurb":"tables are the working surface of applied ml. learn rows, columns, missing values, joins, aggregates, and the dataframe habits ai-generated notebooks assume.","status":"live","lessonCount":6,"stepCount":42,"liveLessonCount":6,"liveStepCount":42,"estMinutes":43,"xpTotal":156},{"number":36,"slug":"sql-for-ml-datasets","title":"sql for ml datasets","blurb":"most training data starts in a database. learn the select, join, filter, aggregate, and leakage traps that decide whether a model is learning signal or nonsense.","status":"live","lessonCount":4,"stepCount":28,"liveLessonCount":4,"liveStepCount":28,"estMinutes":28,"xpTotal":104},{"number":37,"slug":"dataset-pipelines","title":"dataset formats, ingestion, and validation pipelines","blurb":"a dataset is a product surface. build ingestion, validation, partitions, manifests, and checkpoints so the next run is not a mystery.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":38,"slug":"ml-math-and-stats","title":"ml math and statistics that actually show up","blurb":"vectors, probability, distributions, correlation, and uncertainty are not trivia. they are how you read model behavior without worshipping it.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":39,"slug":"supervised-learning-workflows","title":"supervised learning workflows","blurb":"labels, splits, baselines, training, prediction, and evaluation. the supervised workflow is the first complete model loop.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":40,"xpTotal":130},{"number":40,"slug":"unsupervised-learning-and-embeddings","title":"unsupervised learning, embeddings, and recommenders","blurb":"not every useful model has labels. cluster, compare, retrieve, and recommend by turning examples into useful neighborhoods.","status":"live","lessonCount":4,"stepCount":28,"liveLessonCount":4,"liveStepCount":28,"estMinutes":32,"xpTotal":104},{"number":41,"slug":"metrics-and-error-analysis","title":"metrics, slices, and error analysis","blurb":"accuracy is a blunt instrument. learn confusion matrices, precision, recall, thresholds, slices, and failure notes so model quality has evidence.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":42,"slug":"pytorch-tensors-and-autograd","title":"pytorch tensors and autograd","blurb":"read tensor code without flinching. tensors, shapes, broadcasting, gradients, and autograd are the grammar of modern deep learning scripts.","status":"live","lessonCount":4,"stepCount":28,"liveLessonCount":4,"liveStepCount":28,"estMinutes":30,"xpTotal":104},{"number":43,"slug":"training-loops-and-optimizers","title":"training loops, backprop, optimizers, and schedulers","blurb":"the training loop is where models change. learn loss, gradients, optimizer steps, schedules, checkpoints, and the bugs ai ships there.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":44,"slug":"deep-learning-architectures","title":"cnns, transformers, and useful llm internals","blurb":"architecture literacy for builders: convolution, attention, tokens, decoding, kv cache, quantization, and what those choices do to cost and behavior.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":45,"slug":"feature-experiments-registries","title":"feature pipelines, experiment tracking, and registries","blurb":"features, runs, configs, artifacts, and registries are how ml work becomes repeatable instead of a lucky notebook.","status":"live","lessonCount":5,"stepCount":35,"liveLessonCount":5,"liveStepCount":35,"estMinutes":37,"xpTotal":130},{"number":46,"slug":"model-serving-and-mlops","title":"model serving, ci/cd, and mlops","blurb":"serving a model means handling inputs, versions, routes, batch jobs, ci gates, rollback, and production failures deliberately.","status":"live","lessonCount":6,"stepCount":42,"liveLessonCount":6,"liveStepCount":42,"estMinutes":44,"xpTotal":156},{"number":47,"slug":"monitoring-cloud-portfolio","title":"monitoring, drift, cloud scale, and portfolio launch","blurb":"the last mile: logs, drift, alerts, retraining decisions, cloud cost, gpu constraints, architecture docs, demos, and role stories.","status":"live","lessonCount":6,"stepCount":42,"liveLessonCount":6,"liveStepCount":42,"estMinutes":44,"xpTotal":156},{"number":48,"slug":"graphic-design-studio","title":"graphic design studio — prompt, kit, and fifty on-brand assets","blurb":"how a working designer runs image models: prompts written like creative briefs, a brand bible that travels as structured data — with the design tokens that carry it onto screens — and the seeds, reference locks, and QA filters that make fifty generated assets look like one studio shipped them.","status":"live","lessonCount":3,"stepCount":27,"liveLessonCount":3,"liveStepCount":27,"estMinutes":73,"xpTotal":82},{"number":49,"slug":"campaign-studio","title":"campaign studio — one campaign, end to end, with the gate on","blurb":"brief to voice-locked copy variants to batch plan to calendar to the numbers pass. the whole campaign as data your scripts can check — and an approval gate nothing skips. built for the person who owns the send button.","status":"live","lessonCount":3,"stepCount":22,"liveLessonCount":3,"liveStepCount":22,"estMinutes":59,"xpTotal":69},{"number":50,"slug":"support-studio","title":"support studio — the reply drafter you can trust","blurb":"how a working support agent runs AI: a policy pack the drafter can't talk around, a banned-promises linter that holds every draft, golden eval sets that make each macro earn autonomy one intent at a time, and the escalation line where money, law, and anger stay human.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":61,"xpTotal":71},{"number":51,"slug":"copy-studio","title":"copy studio — voice from evidence, variants at scale, claims that clear","blurb":"a client's voice measured from their own published copy, variant batches that pass the voice gate or die with a named reason, and a claim check that blocks the ship button while any fact is unsourced. corrections-log culture, as code.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":66,"xpTotal":72},{"number":52,"slug":"analytics-studio","title":"analytics studio — messy export to defensible memo","blurb":"how a working analyst runs AI: a cleaning log that accounts for every dropped row, sample-first checks that catch the plausible-but-wrong aggregation, a memo where every number carries the address of the cell that made it, SR 11-7-style challenge against the AI's recommendation, and the rerun test — same data, same code, same memo, or it isn't done.","status":"live","lessonCount":3,"stepCount":22,"liveLessonCount":3,"liveStepCount":22,"estMinutes":61,"xpTotal":74},{"number":53,"slug":"delivery-studio","title":"delivery studio — direct the build without becoming the bottleneck","blurb":"how a working delivery lead runs AI builders: specs with acceptance checks written before the build, a review that runs the checks instead of admiring the demo, status roll-ups computed from acceptance records instead of standup vibes, the not-ready call made on evidence you can read out loud to a steering committee, and an hours-and-dollars receipt — before/after on a named function, finance's loaded rate, a named owner, the accepted thing that shipped — small enough to defend in a staff meeting.","status":"live","lessonCount":4,"stepCount":32,"liveLessonCount":4,"liveStepCount":32,"estMinutes":74,"xpTotal":96},{"number":54,"slug":"people-studio","title":"people studio — draft fast, decide human, prove everything","blurb":"how a working hr specialist runs AI: a data firewall that keeps employee records out of prompts by code instead of willpower, recruiting briefs where every claim cites a resume line, policy drafts that cannot publish without a named legal reviewer, and the adverse-action paper trail that survives a discrimination claim.","status":"live","lessonCount":3,"stepCount":24,"liveLessonCount":3,"liveStepCount":24,"estMinutes":56,"xpTotal":72},{"number":55,"slug":"ops-studio","title":"ops studio — tribal knowledge to running automation","blurb":"how a working ops lead gets the process out of one person's head and into automation that can be trusted: SOPs captured as versioned data with the invented steps hunted down, recurring checklists with an explicit auto-run vs named-approval line, exception queues instead of silent failures, a receipt against the export for every number in the weekly report, and a PII proxy that masks member identifiers before the model and restores them only for the named human who sends.","status":"live","lessonCount":4,"stepCount":32,"liveLessonCount":4,"liveStepCount":32,"estMinutes":80,"xpTotal":96},{"number":56,"slug":"legal-studio","title":"legal studio — the cite-check gate and the log that survives","blurb":"how a working lawyer runs AI: a citation-verification drill where every authority gets opened at the source (the fabricated ones always look right), clause extraction that's worthless without a pinpoint, privilege rules written as a routing table instead of remembered under deadline, and the review log you'd want in the record when opposing counsel moves for sanctions.","status":"live","lessonCount":3,"stepCount":22,"liveLessonCount":3,"liveStepCount":22,"estMinutes":60,"xpTotal":69},{"number":57,"slug":"agent-studio","title":"agent studio — ship one agent feature before you stand up","blurb":"one sitting, one agent feature, end to end: a tool schema that makes the dangerous call a validation error, a loop that meters its own spend, golden cases wired before the router works, a trace you debug from instead of vibes, and the ratchet that turns tonight's incident into tomorrow's eval. practice at production tempo.","status":"live","lessonCount":3,"stepCount":23,"liveLessonCount":3,"liveStepCount":23,"estMinutes":66,"xpTotal":73}]},"accessToc":{"chapters":[{"slug":"before-you-build","lessons":[{"slug":"from-worker-to-builder","steps":[{"id":"before-you-build/from-worker-to-builder/s01-01-intro"}]},{"slug":"upgrade-dull-artifacts-into-interactive-sites","steps":[{"id":"before-you-build/upgrade-dull-artifacts-into-interactive-sites/s01-01-intro"},{"id":"before-you-build/upgrade-dull-artifacts-into-interactive-sites/s02-02-build-brief"},{"id":"before-you-build/upgrade-dull-artifacts-into-interactive-sites/s03-03-your-artifact-brief"}]},{"slug":"why-building-matters-now","steps":[{"id":"before-you-build/why-building-matters-now/s01-01-intro"}]},{"slug":"the-builder-loop","steps":[{"id":"before-you-build/the-builder-loop/s01-01-intro"}]},{"slug":"your-first-builder-brief","steps":[{"id":"before-you-build/your-first-builder-brief/s01-01-intro"}]},{"slug":"what-the-ai-is-doing","steps":[{"id":"before-you-build/what-the-ai-is-doing/s01-01-intro"}]},{"slug":"build-your-first-reusable-tool","steps":[{"id":"before-you-build/build-your-first-reusable-tool/s01-01-intro"}]},{"slug":"from-prompt-to-project","steps":[{"id":"before-you-build/from-prompt-to-project/s01-01-intro"}]},{"slug":"why-code-appears-later","steps":[{"id":"before-you-build/why-code-appears-later/s01-01-intro"}]},{"slug":"use-it-again","steps":[{"id":"before-you-build/use-it-again/s01-01-intro"},{"id":"before-you-build/use-it-again/s02-02-which-fix-compounds"},{"id":"before-you-build/use-it-again/s03-03-save-version-two"}]}]},{"slug":"read-the-script","lessons":[{"slug":"names-and-values","steps":[{"id":"read-the-script/names-and-values/s01-01-intro"},{"id":"read-the-script/names-and-values/s02-02-spot-the-type"},{"id":"read-the-script/names-and-values/s03-03-assignment"},{"id":"read-the-script/names-and-values/s04-04-predict-the-score"},{"id":"read-the-script/names-and-values/s05-05-fill-the-name"},{"id":"read-the-script/names-and-values/s06-06-fix-the-typo"},{"id":"read-the-script/names-and-values/s07-07-write-greeting"},{"id":"read-the-script/names-and-values/s08-08-checkpoint"}]},{"slug":"functions-and-the-hallucinated-return","steps":[{"id":"read-the-script/functions-and-the-hallucinated-return/s01-01-intro"},{"id":"read-the-script/functions-and-the-hallucinated-return/s02-02-which-returns"},{"id":"read-the-script/functions-and-the-hallucinated-return/s03-03-return-vs-print"},{"id":"read-the-script/functions-and-the-hallucinated-return/s04-04-predict-the-result"},{"id":"read-the-script/functions-and-the-hallucinated-return/s05-05-fix-the-missing-return"},{"id":"read-the-script/functions-and-the-hallucinated-return/s06-06-fix-print-vs-return"},{"id":"read-the-script/functions-and-the-hallucinated-return/s07-07-write-square"},{"id":"read-the-script/functions-and-the-hallucinated-return/s08-08-checkpoint"}]},{"slug":"decisions-if-elif","steps":[{"id":"read-the-script/decisions-if-elif/s01-01-intro"},{"id":"read-the-script/decisions-if-elif/s02-02-truthiness-quiz"},{"id":"read-the-script/decisions-if-elif/s03-03-eq-vs-is"},{"id":"read-the-script/decisions-if-elif/s04-04-predict-the-empty-list"},{"id":"read-the-script/decisions-if-elif/s05-05-fix-the-zero-bug"},{"id":"read-the-script/decisions-if-elif/s06-06-fix-the-elif-order"},{"id":"read-the-script/decisions-if-elif/s07-07-checkpoint"},{"id":"read-the-script/decisions-if-elif/s08-08-going-deeper"}]}]},{"slug":"read-the-data-shapes","lessons":[{"slug":"the-shapes-ai-dumps-on-you","steps":[{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s01-01-intro"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s02-02-which-is-which"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s03-03-nested-paths"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s04-04-predict-the-deep-key"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s05-05-fill-the-index"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s06-06-fix-the-keyerror"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s07-07-fix-the-list-vs-dict"},{"id":"read-the-data-shapes/the-shapes-ai-dumps-on-you/s08-08-checkpoint"}]},{"slug":"walking-the-shape","steps":[{"id":"read-the-data-shapes/walking-the-shape/s01-01-intro"},{"id":"read-the-data-shapes/walking-the-shape/s02-02-which-loop-shape"},{"id":"read-the-data-shapes/walking-the-shape/s03-03-walking-the-shape"},{"id":"read-the-data-shapes/walking-the-shape/s04-04-predict-the-list-loop"},{"id":"read-the-data-shapes/walking-the-shape/s05-05-fill-the-loop-variable"},{"id":"read-the-data-shapes/walking-the-shape/s06-06-fix-the-dict-loop"},{"id":"read-the-data-shapes/walking-the-shape/s07-07-write-sum-loop"},{"id":"read-the-data-shapes/walking-the-shape/s08-08-checkpoint"}]},{"slug":"comprehensions-and-the-wrong-layer-bug","steps":[{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s01-01-intro"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s02-02-which-comp-shape"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s03-03-with-filter"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s04-04-predict-the-comp"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s05-05-fix-the-wrong-layer"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s06-06-write-comp"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s07-07-checkpoint"},{"id":"read-the-data-shapes/comprehensions-and-the-wrong-layer-bug/s08-08-going-deeper"}]}]},{"slug":"read-the-failure","lessons":[{"slug":"read-bottom-up","steps":[{"id":"read-the-failure/read-bottom-up/s01-01-anatomy"},{"id":"read-the-failure/read-bottom-up/s02-02-which-error"},{"id":"read-the-failure/read-bottom-up/s03-03-the-five-fingerprints"},{"id":"read-the-failure/read-bottom-up/s04-04-predict-the-class"},{"id":"read-the-failure/read-bottom-up/s05-05-fix-nameerror"},{"id":"read-the-failure/read-bottom-up/s06-06-fix-attributeerror"},{"id":"read-the-failure/read-bottom-up/s07-07-write-defensive"},{"id":"read-the-failure/read-bottom-up/s08-08-checkpoint"}]},{"slug":"try-except-what-ai-misuses","steps":[{"id":"read-the-failure/try-except-what-ai-misuses/s01-01-intro"},{"id":"read-the-failure/try-except-what-ai-misuses/s02-02-which-block-runs"},{"id":"read-the-failure/try-except-what-ai-misuses/s03-03-naming-the-error"},{"id":"read-the-failure/try-except-what-ai-misuses/s04-04-predict-the-branch"},{"id":"read-the-failure/try-except-what-ai-misuses/s05-05-fill-the-keyword"},{"id":"read-the-failure/try-except-what-ai-misuses/s06-06-fix-the-bare-except"},{"id":"read-the-failure/try-except-what-ai-misuses/s07-07-write-safe-lookup"},{"id":"read-the-failure/try-except-what-ai-misuses/s08-08-checkpoint"}]},{"slug":"fail-loudly","steps":[{"id":"read-the-failure/fail-loudly/s01-01-intro"},{"id":"read-the-failure/fail-loudly/s02-02-which-raise-fires"},{"id":"read-the-failure/fail-loudly/s03-03-predict-the-raise"},{"id":"read-the-failure/fail-loudly/s04-04-fill-the-raise-keyword"},{"id":"read-the-failure/fail-loudly/s05-05-fix-the-silent-swallow"},{"id":"read-the-failure/fail-loudly/s06-06-write-validate-age"},{"id":"read-the-failure/fail-loudly/s07-07-checkpoint"},{"id":"read-the-failure/fail-loudly/s08-08-going-deeper"}]}]},{"slug":"read-the-pipeline","lessons":[{"slug":"files-without-fear","steps":[{"id":"read-the-pipeline/files-without-fear/s01-01-intro"},{"id":"read-the-pipeline/files-without-fear/s02-02-which-mode"},{"id":"read-the-pipeline/files-without-fear/s03-03-pathlib-and-manifests"},{"id":"read-the-pipeline/files-without-fear/s04-04-predict-the-suffix"},{"id":"read-the-pipeline/files-without-fear/s05-05-fill-the-mode"},{"id":"read-the-pipeline/files-without-fear/s06-06-fix-the-wrong-mode"},{"id":"read-the-pipeline/files-without-fear/s07-07-write-manifest-entry"},{"id":"read-the-pipeline/files-without-fear/s08-08-checkpoint"}]},{"slug":"talking-to-apis","steps":[{"id":"read-the-pipeline/talking-to-apis/s01-01-intro"},{"id":"read-the-pipeline/talking-to-apis/s02-02-which-status-family"},{"id":"read-the-pipeline/talking-to-apis/s03-03-parsing-the-json"},{"id":"read-the-pipeline/talking-to-apis/s04-04-predict-the-key"},{"id":"read-the-pipeline/talking-to-apis/s05-05-fill-the-method"},{"id":"read-the-pipeline/talking-to-apis/s06-06-fix-the-status-check"},{"id":"read-the-pipeline/talking-to-apis/s07-07-write-classify-status"},{"id":"read-the-pipeline/talking-to-apis/s08-08-checkpoint"}]},{"slug":"when-the-pipeline-breaks","steps":[{"id":"read-the-pipeline/when-the-pipeline-breaks/s01-01-intro"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s02-02-honor-the-retry-after"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s03-03-predict-the-status-handling"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s04-04-fill-the-raise-for-status"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s05-05-fix-the-missing-check"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s06-06-fix-the-broad-retry"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s07-07-checkpoint"},{"id":"read-the-pipeline/when-the-pipeline-breaks/s08-08-going-deeper"}]}]},{"slug":"mutation-and-state","lessons":[{"slug":"why-it-breaks","steps":[{"id":"mutation-and-state/why-it-breaks/s01-01-intro"},{"id":"mutation-and-state/why-it-breaks/s02-02-which-mutates"},{"id":"mutation-and-state/why-it-breaks/s03-03-shared-reference"},{"id":"mutation-and-state/why-it-breaks/s04-04-fix-the-shared-list"},{"id":"mutation-and-state/why-it-breaks/s05-05-checkpoint"}]},{"slug":"copy-vs-reference","steps":[{"id":"mutation-and-state/copy-vs-reference/s01-01-deep-copy"},{"id":"mutation-and-state/copy-vs-reference/s02-02-predict-the-shallow"},{"id":"mutation-and-state/copy-vs-reference/s03-03-fix-the-shallow-bug"},{"id":"mutation-and-state/copy-vs-reference/s04-04-checkpoint"}]}]},{"slug":"llm-apis","lessons":[{"slug":"before-the-api-what-the-model-is-doing","steps":[{"id":"llm-apis/before-the-api-what-the-model-is-doing/s01-01-intro"}]},{"slug":"the-messages-pattern","steps":[{"id":"llm-apis/the-messages-pattern/s01-01-intro"},{"id":"llm-apis/the-messages-pattern/s02-02-which-role-is-yours"},{"id":"llm-apis/the-messages-pattern/s03-03-reading-the-response"},{"id":"llm-apis/the-messages-pattern/s04-04-predict-the-text"},{"id":"llm-apis/the-messages-pattern/s05-05-fill-the-role"},{"id":"llm-apis/the-messages-pattern/s06-06-fix-the-message-shape"},{"id":"llm-apis/the-messages-pattern/s07-07-fix-the-response-access"},{"id":"llm-apis/the-messages-pattern/s08-08-write-ask-claude"},{"id":"llm-apis/the-messages-pattern/s09-09-checkpoint"}]},{"slug":"reading-the-response","steps":[{"id":"llm-apis/reading-the-response/s01-01-intro"},{"id":"llm-apis/reading-the-response/s02-02-which-block-do-you-read"},{"id":"llm-apis/reading-the-response/s03-03-the-five-stop-reasons"},{"id":"llm-apis/reading-the-response/s04-04-predict-the-text"},{"id":"llm-apis/reading-the-response/s05-05-fill-the-text-extract"},{"id":"llm-apis/reading-the-response/s06-06-fix-the-content-zero"},{"id":"llm-apis/reading-the-response/s07-07-fix-the-stop-assumption"},{"id":"llm-apis/reading-the-response/s08-08-write-the-extractor"},{"id":"llm-apis/reading-the-response/s09-09-checkpoint"}]},{"slug":"the-model-picker","steps":[{"id":"llm-apis/the-model-picker/s01-01-why-model-choice-is-a-product-decision"},{"id":"llm-apis/the-model-picker/s02-02-haiku-territory"},{"id":"llm-apis/the-model-picker/s03-03-which-model-here"},{"id":"llm-apis/the-model-picker/s04-04-sonnet-territory"},{"id":"llm-apis/the-model-picker/s05-05-opus-and-the-thinking-budget"},{"id":"llm-apis/the-model-picker/s06-06-pick-the-model"},{"id":"llm-apis/the-model-picker/s07-07-write-the-model-routing-memo"},{"id":"llm-apis/the-model-picker/s08-08-checkpoint"}]}]},{"slug":"structured-output","lessons":[{"slug":"getting-json-back","steps":[{"id":"structured-output/getting-json-back/s01-01-intro"},{"id":"structured-output/getting-json-back/s02-02-why-validate"},{"id":"structured-output/getting-json-back/s03-03-pydantic-models"},{"id":"structured-output/getting-json-back/s04-04-predict-the-validated"},{"id":"structured-output/getting-json-back/s05-05-fill-the-field-type"},{"id":"structured-output/getting-json-back/s06-06-fix-the-required-field"},{"id":"structured-output/getting-json-back/s07-07-fix-the-type-mismatch"},{"id":"structured-output/getting-json-back/s08-08-write-parse-extraction"},{"id":"structured-output/getting-json-back/s09-09-checkpoint"}]},{"slug":"schemas-eat-prompts","steps":[{"id":"structured-output/schemas-eat-prompts/s01-01-the-trust-boundary"},{"id":"structured-output/schemas-eat-prompts/s02-02-three-shipped-breakages"},{"id":"structured-output/schemas-eat-prompts/s03-03-which-boundary-broke"},{"id":"structured-output/schemas-eat-prompts/s04-04-schema-as-prompt-replacement"},{"id":"structured-output/schemas-eat-prompts/s05-05-where-validation-belongs"},{"id":"structured-output/schemas-eat-prompts/s06-06-the-pydantic-migration-as-war-story"},{"id":"structured-output/schemas-eat-prompts/s07-07-write-the-boundary-audit"},{"id":"structured-output/schemas-eat-prompts/s08-08-checkpoint"}]},{"slug":"dataset-schema-validation","steps":[{"id":"structured-output/dataset-schema-validation/s01-01-validate-at-the-boundary"},{"id":"structured-output/dataset-schema-validation/s02-02-the-silent-drop"},{"id":"structured-output/dataset-schema-validation/s03-03-predict-the-missing-fields"},{"id":"structured-output/dataset-schema-validation/s04-04-fill-the-membership-test"},{"id":"structured-output/dataset-schema-validation/s05-05-fix-the-range-check"},{"id":"structured-output/dataset-schema-validation/s06-06-write-validate-batch"},{"id":"structured-output/dataset-schema-validation/s07-07-claims-with-receipts"},{"id":"structured-output/dataset-schema-validation/s08-08-checkpoint"}]}]},{"slug":"mcp","lessons":[{"slug":"what-mcp-is","steps":[{"id":"mcp/what-mcp-is/s01-01-intro"},{"id":"mcp/what-mcp-is/s02-02-which-side-is-the-server"},{"id":"mcp/what-mcp-is/s03-03-the-tool-list"},{"id":"mcp/what-mcp-is/s04-04-predict-the-tool-result"},{"id":"mcp/what-mcp-is/s05-05-fill-the-tool-name"},{"id":"mcp/what-mcp-is/s06-06-fix-the-missing-argument"},{"id":"mcp/what-mcp-is/s07-07-fix-the-result-shape"},{"id":"mcp/what-mcp-is/s08-08-write-call-tool"},{"id":"mcp/what-mcp-is/s09-09-checkpoint"}]},{"slug":"writing-a-tiny-mcp-server","steps":[{"id":"mcp/writing-a-tiny-mcp-server/s01-01-intro"},{"id":"mcp/writing-a-tiny-mcp-server/s02-02-which-message-is-call-tool"},{"id":"mcp/writing-a-tiny-mcp-server/s03-03-the-tool-registry"},{"id":"mcp/writing-a-tiny-mcp-server/s04-04-predict-the-list-tools-output"},{"id":"mcp/writing-a-tiny-mcp-server/s05-05-fill-the-input-schema"},{"id":"mcp/writing-a-tiny-mcp-server/s06-06-fix-the-missing-error-response"},{"id":"mcp/writing-a-tiny-mcp-server/s07-07-fix-the-validation-gap"},{"id":"mcp/writing-a-tiny-mcp-server/s08-08-write-the-call-tool-dispatcher"},{"id":"mcp/writing-a-tiny-mcp-server/s09-09-checkpoint"}]},{"slug":"why-mcp-won","steps":[{"id":"mcp/why-mcp-won/s01-01-the-graveyard-of-tool-protocols"},{"id":"mcp/why-mcp-won/s02-02-what-openai-plugins-got-wrong"},{"id":"mcp/why-mcp-won/s03-03-which-failure-mode"},{"id":"mcp/why-mcp-won/s04-04-the-mcp-bet"},{"id":"mcp/why-mcp-won/s05-05-the-network-effect"},{"id":"mcp/why-mcp-won/s06-06-spot-the-mcp-wedge"},{"id":"mcp/why-mcp-won/s07-07-write-the-protocol-postmortem"},{"id":"mcp/why-mcp-won/s08-08-checkpoint"}]}]},{"slug":"agent-loops","lessons":[{"slug":"the-loop","steps":[{"id":"agent-loops/the-loop/s01-01-intro"},{"id":"agent-loops/the-loop/s02-02-which-stop-reason"},{"id":"agent-loops/the-loop/s03-03-tool-result-shape"},{"id":"agent-loops/the-loop/s04-04-predict-the-next-call"},{"id":"agent-loops/the-loop/s05-05-fill-the-stop-condition"},{"id":"agent-loops/the-loop/s06-06-fix-the-tool-id"},{"id":"agent-loops/the-loop/s07-07-fix-the-loop-exit"},{"id":"agent-loops/the-loop/s08-08-write-the-loop"},{"id":"agent-loops/the-loop/s09-09-checkpoint"}]},{"slug":"multi-step-tools","steps":[{"id":"agent-loops/multi-step-tools/s01-01-intro"},{"id":"agent-loops/multi-step-tools/s02-02-which-tool-runs"},{"id":"agent-loops/multi-step-tools/s03-03-tool-registry"},{"id":"agent-loops/multi-step-tools/s04-04-predict-the-chain"},{"id":"agent-loops/multi-step-tools/s05-05-fill-the-dispatch"},{"id":"agent-loops/multi-step-tools/s06-06-fix-the-missing-tool"},{"id":"agent-loops/multi-step-tools/s07-07-fix-the-parallel-ids"},{"id":"agent-loops/multi-step-tools/s08-08-write-multi-tool-agent"},{"id":"agent-loops/multi-step-tools/s09-09-checkpoint"}]},{"slug":"routing","steps":[{"id":"agent-loops/routing/s01-01-intro"},{"id":"agent-loops/routing/s02-02-when-to-route"},{"id":"agent-loops/routing/s03-03-classifier-output"},{"id":"agent-loops/routing/s04-04-predict-the-route"},{"id":"agent-loops/routing/s05-05-fill-the-dispatch"},{"id":"agent-loops/routing/s06-06-fix-the-fallback"},{"id":"agent-loops/routing/s07-07-fix-the-loose-classifier"},{"id":"agent-loops/routing/s08-08-write-the-router"},{"id":"agent-loops/routing/s09-09-checkpoint"}]},{"slug":"evaluator-optimizer","steps":[{"id":"agent-loops/evaluator-optimizer/s01-01-intro"},{"id":"agent-loops/evaluator-optimizer/s02-02-when-it-earns-its-keep"},{"id":"agent-loops/evaluator-optimizer/s03-03-judge-output-shape"},{"id":"agent-loops/evaluator-optimizer/s04-04-predict-the-iterations"},{"id":"agent-loops/evaluator-optimizer/s05-05-fill-the-exit"},{"id":"agent-loops/evaluator-optimizer/s06-06-fix-the-judge-parsing"},{"id":"agent-loops/evaluator-optimizer/s07-07-fix-the-missing-cap"},{"id":"agent-loops/evaluator-optimizer/s08-08-write-the-evaluator-optimizer"},{"id":"agent-loops/evaluator-optimizer/s09-09-checkpoint"}]},{"slug":"why-every-framework-is-thirty-lines","steps":[{"id":"agent-loops/why-every-framework-is-thirty-lines/s01-01-the-frameworks-converged"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s02-02-langgraph-walkthrough"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s03-03-the-work-is-a-graph"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s04-04-which-framework-fits"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s05-05-vercel-ai-sdk-walkthrough"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s06-06-the-five-canonical-patterns"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s07-07-when-to-stay-framework-free"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s08-08-write-the-tradeoff-memo"},{"id":"agent-loops/why-every-framework-is-thirty-lines/s09-09-checkpoint"}]},{"slug":"the-industry-map","steps":[{"id":"agent-loops/the-industry-map/s01-01-the-five-layers"},{"id":"agent-loops/the-industry-map/s02-02-which-layer-failed"},{"id":"agent-loops/the-industry-map/s03-03-the-shape-of-the-stack"}]}]},{"slug":"git-and-github","lessons":[{"slug":"the-three-states","steps":[{"id":"git-and-github/the-three-states/s01-01-intro"},{"id":"git-and-github/the-three-states/s02-02-which-command-stages"},{"id":"git-and-github/the-three-states/s03-03-status-and-diff"},{"id":"git-and-github/the-three-states/s04-04-predict-the-status"},{"id":"git-and-github/the-three-states/s05-05-fill-the-command"},{"id":"git-and-github/the-three-states/s06-06-fix-the-staged-files"},{"id":"git-and-github/the-three-states/s07-07-fix-the-pr-command"},{"id":"git-and-github/the-three-states/s08-08-write-status-summary"},{"id":"git-and-github/the-three-states/s09-09-checkpoint"}]},{"slug":"three-git-disasters-ai-shipped","steps":[{"id":"git-and-github/three-git-disasters-ai-shipped/s01-01-why-postmortems-teach-better"},{"id":"git-and-github/three-git-disasters-ai-shipped/s02-02-uber-2016-credentials-in-repo"},{"id":"git-and-github/three-git-disasters-ai-shipped/s03-03-which-control-would-have-caught-it"},{"id":"git-and-github/three-git-disasters-ai-shipped/s04-04-the-samsung-paste"},{"id":"git-and-github/three-git-disasters-ai-shipped/s05-05-the-cursor-git-add-dot-default"},{"id":"git-and-github/three-git-disasters-ai-shipped/s06-06-spot-the-failure-mode"},{"id":"git-and-github/three-git-disasters-ai-shipped/s07-07-write-your-pre-commit-checklist"},{"id":"git-and-github/three-git-disasters-ai-shipped/s08-08-checkpoint"}]},{"slug":"github-actions-for-model-checks","steps":[{"id":"git-and-github/github-actions-for-model-checks/s01-01-intro"},{"id":"git-and-github/github-actions-for-model-checks/s02-02-green-but-wrong"},{"id":"git-and-github/github-actions-for-model-checks/s03-03-predict-the-run"},{"id":"git-and-github/github-actions-for-model-checks/s04-04-fill-the-trigger"},{"id":"git-and-github/github-actions-for-model-checks/s05-05-fix-the-eval-gate"},{"id":"git-and-github/github-actions-for-model-checks/s06-06-write-run-job"},{"id":"git-and-github/github-actions-for-model-checks/s07-07-checkpoint"}]}]},{"slug":"secrets-and-env","lessons":[{"slug":"keeping-keys-safe","steps":[{"id":"secrets-and-env/keeping-keys-safe/s01-01-intro"},{"id":"secrets-and-env/keeping-keys-safe/s02-02-which-line-leaks"},{"id":"secrets-and-env/keeping-keys-safe/s03-03-loading-env-vars"},{"id":"secrets-and-env/keeping-keys-safe/s04-04-predict-the-key"},{"id":"secrets-and-env/keeping-keys-safe/s05-05-fill-the-getenv"},{"id":"secrets-and-env/keeping-keys-safe/s06-06-fix-the-hardcoded-key"},{"id":"secrets-and-env/keeping-keys-safe/s07-07-fix-the-missing-default"},{"id":"secrets-and-env/keeping-keys-safe/s08-08-write-load-config"},{"id":"secrets-and-env/keeping-keys-safe/s09-09-checkpoint"}]},{"slug":"ml-service-secrets","steps":[{"id":"secrets-and-env/ml-service-secrets/s01-01-intro"},{"id":"secrets-and-env/ml-service-secrets/s02-02-which-log-line-leaks"},{"id":"secrets-and-env/ml-service-secrets/s03-03-predict-the-scan"},{"id":"secrets-and-env/ml-service-secrets/s04-04-fill-the-lookup"},{"id":"secrets-and-env/ml-service-secrets/s05-05-fix-the-logged-key"},{"id":"secrets-and-env/ml-service-secrets/s06-06-write-scan-logs"},{"id":"secrets-and-env/ml-service-secrets/s07-07-checkpoint"}]}]},{"slug":"prompting","lessons":[{"slug":"builder-briefs-before-prompts","steps":[{"id":"prompting/builder-briefs-before-prompts/s01-01-intro"}]},{"slug":"the-prompt-craft","steps":[{"id":"prompting/the-prompt-craft/s01-01-intro"},{"id":"prompting/the-prompt-craft/s02-02-which-prompt-is-better"},{"id":"prompting/the-prompt-craft/s03-03-context-rot"},{"id":"prompting/the-prompt-craft/s04-04-predict-the-output"},{"id":"prompting/the-prompt-craft/s05-05-fill-the-constraint"},{"id":"prompting/the-prompt-craft/s06-06-fix-the-vague-prompt"},{"id":"prompting/the-prompt-craft/s07-07-fix-the-bloated-context"},{"id":"prompting/the-prompt-craft/s08-08-write-the-prompt"},{"id":"prompting/the-prompt-craft/s09-09-checkpoint"}]},{"slug":"few-shot-and-reasoning","steps":[{"id":"prompting/few-shot-and-reasoning/s01-01-intro"},{"id":"prompting/few-shot-and-reasoning/s02-02-when-few-shot-helps"},{"id":"prompting/few-shot-and-reasoning/s03-03-the-cot-trap"},{"id":"prompting/few-shot-and-reasoning/s04-04-predict-the-format-lock"},{"id":"prompting/few-shot-and-reasoning/s05-05-fill-the-example-format"},{"id":"prompting/few-shot-and-reasoning/s06-06-fix-the-cot-on-reasoning"},{"id":"prompting/few-shot-and-reasoning/s07-07-fix-the-format-mismatch"},{"id":"prompting/few-shot-and-reasoning/s08-08-write-the-prompt-builder"},{"id":"prompting/few-shot-and-reasoning/s09-09-checkpoint"}]},{"slug":"agent-config-files","steps":[{"id":"prompting/agent-config-files/s01-01-intro"},{"id":"prompting/agent-config-files/s02-02-which-belongs-in-the-file"},{"id":"prompting/agent-config-files/s03-03-anatomy-of-a-good-file"},{"id":"prompting/agent-config-files/s04-04-predict-which-rule-fires"},{"id":"prompting/agent-config-files/s05-05-fill-the-rule-block"},{"id":"prompting/agent-config-files/s06-06-fix-the-too-vague"},{"id":"prompting/agent-config-files/s07-07-fix-the-stale-pointer"},{"id":"prompting/agent-config-files/s08-08-write-a-claude-md"},{"id":"prompting/agent-config-files/s09-09-checkpoint"}]},{"slug":"what-aged-in-prompting","steps":[{"id":"prompting/what-aged-in-prompting/s01-01-prompting-as-a-moving-target"},{"id":"prompting/what-aged-in-prompting/s02-02-cot-then-vs-now"},{"id":"prompting/what-aged-in-prompting/s03-03-which-still-works"},{"id":"prompting/what-aged-in-prompting/s04-04-the-system-prompt-arms-race"},{"id":"prompting/what-aged-in-prompting/s05-05-spot-the-stale-technique"},{"id":"prompting/what-aged-in-prompting/s06-06-the-prompt-engineer-debate"},{"id":"prompting/what-aged-in-prompting/s07-07-write-the-cheatsheet-update"},{"id":"prompting/what-aged-in-prompting/s08-08-checkpoint"}]}]},{"slug":"agent-traces","lessons":[{"slug":"reading-the-trace","steps":[{"id":"agent-traces/reading-the-trace/s01-01-intro"},{"id":"agent-traces/reading-the-trace/s02-02-which-turn-failed"},{"id":"agent-traces/reading-the-trace/s03-03-the-stop-reasons"},{"id":"agent-traces/reading-the-trace/s04-04-predict-the-stop"},{"id":"agent-traces/reading-the-trace/s05-05-fill-the-tool-name"},{"id":"agent-traces/reading-the-trace/s06-06-fix-the-loop"},{"id":"agent-traces/reading-the-trace/s07-07-fix-the-arg-shape"},{"id":"agent-traces/reading-the-trace/s08-08-write-the-summarizer"},{"id":"agent-traces/reading-the-trace/s09-09-checkpoint"}]},{"slug":"trace-driven-debugging","steps":[{"id":"agent-traces/trace-driven-debugging/s01-01-intro"},{"id":"agent-traces/trace-driven-debugging/s02-02-which-pattern-is-this"},{"id":"agent-traces/trace-driven-debugging/s03-03-the-loop-signature"},{"id":"agent-traces/trace-driven-debugging/s04-04-predict-the-call-count"},{"id":"agent-traces/trace-driven-debugging/s05-05-fill-the-detector"},{"id":"agent-traces/trace-driven-debugging/s06-06-fix-the-tool-routing"},{"id":"agent-traces/trace-driven-debugging/s07-07-fix-the-prompt-bloat"},{"id":"agent-traces/trace-driven-debugging/s08-08-write-the-trace-summary"},{"id":"agent-traces/trace-driven-debugging/s09-09-checkpoint"}]},{"slug":"prediction-ids-and-inference-logs","steps":[{"id":"agent-traces/prediction-ids-and-inference-logs/s01-01-intro"},{"id":"agent-traces/prediction-ids-and-inference-logs/s02-02-pick-the-log-line"},{"id":"agent-traces/prediction-ids-and-inference-logs/s03-03-predict-the-ids"},{"id":"agent-traces/prediction-ids-and-inference-logs/s04-04-fill-the-join-key"},{"id":"agent-traces/prediction-ids-and-inference-logs/s05-05-fix-the-join"},{"id":"agent-traces/prediction-ids-and-inference-logs/s06-06-write-the-logger"},{"id":"agent-traces/prediction-ids-and-inference-logs/s07-07-checkpoint"}]}]},{"slug":"evals","lessons":[{"slug":"check-before-you-trust","steps":[{"id":"evals/check-before-you-trust/s01-01-intro"}]},{"slug":"writing-evals","steps":[{"id":"evals/writing-evals/s01-01-intro"},{"id":"evals/writing-evals/s02-02-which-eval-passes"},{"id":"evals/writing-evals/s03-03-eval-patterns"},{"id":"evals/writing-evals/s04-04-predict-the-pass-rate"},{"id":"evals/writing-evals/s05-05-fill-the-assertion"},{"id":"evals/writing-evals/s06-06-fix-the-flaky-eval"},{"id":"evals/writing-evals/s07-07-fix-the-overstrict-eval"},{"id":"evals/writing-evals/s08-08-write-the-suite"},{"id":"evals/writing-evals/s09-09-checkpoint"}]},{"slug":"llm-as-judge","steps":[{"id":"evals/llm-as-judge/s01-01-intro"},{"id":"evals/llm-as-judge/s02-02-pairwise-or-rubric"},{"id":"evals/llm-as-judge/s03-03-the-four-biases"},{"id":"evals/llm-as-judge/s04-04-predict-the-bias"},{"id":"evals/llm-as-judge/s05-05-fill-the-rubric-call"},{"id":"evals/llm-as-judge/s06-06-fix-the-likert-trap"},{"id":"evals/llm-as-judge/s07-07-fix-the-position-bias"},{"id":"evals/llm-as-judge/s08-08-write-the-rubric-judge"},{"id":"evals/llm-as-judge/s09-09-checkpoint"}]},{"slug":"the-rise-of-evals-as-a-discipline","steps":[{"id":"evals/the-rise-of-evals-as-a-discipline/s01-01-the-prompt-engineering-era"},{"id":"evals/the-rise-of-evals-as-a-discipline/s02-02-the-eval-turn"},{"id":"evals/the-rise-of-evals-as-a-discipline/s03-03-which-company-survives-a-model-swap"},{"id":"evals/the-rise-of-evals-as-a-discipline/s04-04-case-studies"},{"id":"evals/the-rise-of-evals-as-a-discipline/s05-05-eval-first-vs-prompt-first"},{"id":"evals/the-rise-of-evals-as-a-discipline/s06-06-why-the-eval-engineer-eats-the-prompt-engineer"},{"id":"evals/the-rise-of-evals-as-a-discipline/s07-07-write-the-eval-readiness-audit"},{"id":"evals/the-rise-of-evals-as-a-discipline/s08-08-checkpoint"}]},{"slug":"model-validation-gates","steps":[{"id":"evals/model-validation-gates/s01-01-intro"},{"id":"evals/model-validation-gates/s02-02-spot-the-theater"},{"id":"evals/model-validation-gates/s03-03-predict-the-decision"},{"id":"evals/model-validation-gates/s04-04-fill-the-regression-check"},{"id":"evals/model-validation-gates/s05-05-fix-the-theater-gate"},{"id":"evals/model-validation-gates/s06-06-write-the-gate-runner"},{"id":"evals/model-validation-gates/s07-07-checkpoint"}]}]},{"slug":"context-and-retrieval","lessons":[{"slug":"chunking-that-respects-structure","steps":[{"id":"context-and-retrieval/chunking-that-respects-structure/s01-01-intro"},{"id":"context-and-retrieval/chunking-that-respects-structure/s02-02-which-content-shape-breaks"},{"id":"context-and-retrieval/chunking-that-respects-structure/s03-03-the-recursive-splitter"},{"id":"context-and-retrieval/chunking-that-respects-structure/s04-04-predict-the-shred"},{"id":"context-and-retrieval/chunking-that-respects-structure/s05-05-fill-the-separator-list"},{"id":"context-and-retrieval/chunking-that-respects-structure/s06-06-fix-the-mid-sentence-cut"},{"id":"context-and-retrieval/chunking-that-respects-structure/s07-07-fix-the-missing-overlap"},{"id":"context-and-retrieval/chunking-that-respects-structure/s08-08-write-the-recursive-splitter"},{"id":"context-and-retrieval/chunking-that-respects-structure/s09-09-checkpoint"}]},{"slug":"embedding-that-fits-the-budget","steps":[{"id":"context-and-retrieval/embedding-that-fits-the-budget/s01-01-intro"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s02-02-which-model-fits"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s03-03-the-vector-space"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s04-04-predict-the-closest"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s05-05-fill-the-cosine-formula"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s06-06-fix-the-wrong-embedding-model"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s07-07-fix-the-missing-normalization"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s08-08-write-the-similarity-ranker"},{"id":"context-and-retrieval/embedding-that-fits-the-budget/s09-09-checkpoint"}]},{"slug":"retrieval-that-finds-the-right-thing","steps":[{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s01-01-intro"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s02-02-which-result-is-good"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s03-03-the-distance-threshold"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s04-04-predict-the-top-k"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s05-05-fill-the-cutoff"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s06-06-fix-the-no-threshold"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s07-07-fix-the-dup-results"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s08-08-write-the-ranker"},{"id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s09-09-checkpoint"}]},{"slug":"rag-vs-long-context-vs-fine-tune","steps":[{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s01-01-the-three-way-fork"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s02-02-rag-when-you-need-freshness"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s03-03-which-fork-for-this-scenario"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s04-04-long-context-when-the-corpus-fits"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s05-05-fine-tune-when-style-or-format-is-the-product"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s06-06-which-fork-dies-first"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s07-07-graph-vector-or-both"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s08-08-the-rubric"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s09-09-write-the-fork-decision"},{"id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s10-10-checkpoint"}]},{"slug":"retrieval-metrics-and-vector-db-shape","steps":[{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s01-01-the-eval-set"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s02-02-filter-or-search-first"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s03-03-predict-recall-precision"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s04-04-fill-the-reciprocal"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s05-05-fix-the-late-filter"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s06-06-write-recall-at-k"},{"id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s07-07-checkpoint"}]}]},{"slug":"production-tradeoffs","lessons":[{"slug":"prompt-caching-correctly","steps":[{"id":"production-tradeoffs/prompt-caching-correctly/s01-01-intro"},{"id":"production-tradeoffs/prompt-caching-correctly/s02-02-which-piece-caches"},{"id":"production-tradeoffs/prompt-caching-correctly/s03-03-the-cache-breakpoints"},{"id":"production-tradeoffs/prompt-caching-correctly/s04-04-predict-the-savings"},{"id":"production-tradeoffs/prompt-caching-correctly/s05-05-fill-the-cache-control"},{"id":"production-tradeoffs/prompt-caching-correctly/s06-06-fix-the-cache-buster"},{"id":"production-tradeoffs/prompt-caching-correctly/s07-07-fix-the-ttl-default"},{"id":"production-tradeoffs/prompt-caching-correctly/s08-08-write-the-cost-estimator"},{"id":"production-tradeoffs/prompt-caching-correctly/s09-09-checkpoint"}]},{"slug":"read-the-token-bill","steps":[{"id":"production-tradeoffs/read-the-token-bill/s01-01-intro"},{"id":"production-tradeoffs/read-the-token-bill/s02-02-which-prompt-costs-more"},{"id":"production-tradeoffs/read-the-token-bill/s03-03-the-shape-of-a-bill"},{"id":"production-tradeoffs/read-the-token-bill/s04-04-predict-the-monthly-spend"},{"id":"production-tradeoffs/read-the-token-bill/s05-05-fill-the-cost-formula"},{"id":"production-tradeoffs/read-the-token-bill/s06-06-fix-the-pricing-ratio-error"},{"id":"production-tradeoffs/read-the-token-bill/s07-07-fix-the-units-error"},{"id":"production-tradeoffs/read-the-token-bill/s08-08-write-the-cost-estimator"},{"id":"production-tradeoffs/read-the-token-bill/s09-09-checkpoint"}]},{"slug":"the-model-price-war","steps":[{"id":"production-tradeoffs/the-model-price-war/s01-01-the-price-curve"},{"id":"production-tradeoffs/the-model-price-war/s02-02-three-bets-and-how-they-aged"},{"id":"production-tradeoffs/the-model-price-war/s03-03-which-startup-survived"},{"id":"production-tradeoffs/the-model-price-war/s04-04-the-incumbent-trap"},{"id":"production-tradeoffs/the-model-price-war/s05-05-features-now-viable"},{"id":"production-tradeoffs/the-model-price-war/s06-06-pricing-for-a-war-you-cant-predict"},{"id":"production-tradeoffs/the-model-price-war/s07-07-write-the-cost-model"},{"id":"production-tradeoffs/the-model-price-war/s08-08-checkpoint"}]},{"slug":"batch-realtime-costs","steps":[{"id":"production-tradeoffs/batch-realtime-costs/s01-01-who-is-waiting"},{"id":"production-tradeoffs/batch-realtime-costs/s02-02-which-lane"},{"id":"production-tradeoffs/batch-realtime-costs/s03-03-predict-the-bill"},{"id":"production-tradeoffs/batch-realtime-costs/s04-04-fill-the-miss-rate"},{"id":"production-tradeoffs/batch-realtime-costs/s05-05-fix-the-free-discount"},{"id":"production-tradeoffs/batch-realtime-costs/s06-06-write-monthly-bill"},{"id":"production-tradeoffs/batch-realtime-costs/s07-07-checkpoint"}]}]},{"slug":"debugging-output","lessons":[{"slug":"read-the-trace-not-the-chat","steps":[{"id":"debugging-output/read-the-trace-not-the-chat/s01-01-intro"},{"id":"debugging-output/read-the-trace-not-the-chat/s02-02-where-to-look-first"},{"id":"debugging-output/read-the-trace-not-the-chat/s03-03-the-four-failure-classes"},{"id":"debugging-output/read-the-trace-not-the-chat/s04-04-predict-the-bad-turn"},{"id":"debugging-output/read-the-trace-not-the-chat/s05-05-fill-the-find-bad-turn"},{"id":"debugging-output/read-the-trace-not-the-chat/s06-06-fix-the-fix-the-symptom"},{"id":"debugging-output/read-the-trace-not-the-chat/s07-07-fix-the-blame-the-model"},{"id":"debugging-output/read-the-trace-not-the-chat/s08-08-write-the-trace-summary"},{"id":"debugging-output/read-the-trace-not-the-chat/s09-09-checkpoint"}]},{"slug":"the-four-breakage-classes","steps":[{"id":"debugging-output/the-four-breakage-classes/s01-01-intro"},{"id":"debugging-output/the-four-breakage-classes/s02-02-classify-the-refund-bug"},{"id":"debugging-output/the-four-breakage-classes/s03-03-the-trace-anatomy"},{"id":"debugging-output/the-four-breakage-classes/s04-04-predict-the-class"},{"id":"debugging-output/the-four-breakage-classes/s05-05-fill-the-class-name"},{"id":"debugging-output/the-four-breakage-classes/s06-06-fix-the-misclassification"},{"id":"debugging-output/the-four-breakage-classes/s07-07-fix-the-parse-mangle"},{"id":"debugging-output/the-four-breakage-classes/s08-08-write-the-classifier"},{"id":"debugging-output/the-four-breakage-classes/s09-09-checkpoint"}]},{"slug":"five-real-postmortems","steps":[{"id":"debugging-output/five-real-postmortems/s01-01-the-postmortem-template"},{"id":"debugging-output/five-real-postmortems/s02-02-air-canada-refund-lawsuit"},{"id":"debugging-output/five-real-postmortems/s03-03-dpd-and-nyc-mycity"},{"id":"debugging-output/five-real-postmortems/s04-04-classify-the-five"},{"id":"debugging-output/five-real-postmortems/s05-05-recruiter-json-mangle"},{"id":"debugging-output/five-real-postmortems/s06-06-glean-style-retrieval-miss"},{"id":"debugging-output/five-real-postmortems/s07-07-which-fix-catches-the-most"},{"id":"debugging-output/five-real-postmortems/s08-08-write-the-postmortem"},{"id":"debugging-output/five-real-postmortems/s09-09-checkpoint"}]},{"slug":"prediction-failure-triage","steps":[{"id":"debugging-output/prediction-failure-triage/s01-01-reproduce-classify-route"},{"id":"debugging-output/prediction-failure-triage/s02-02-first-move"},{"id":"debugging-output/prediction-failure-triage/s03-03-predict-the-route"},{"id":"debugging-output/prediction-failure-triage/s04-04-fill-the-owners"},{"id":"debugging-output/prediction-failure-triage/s05-05-fix-the-paraphrase-repro"},{"id":"debugging-output/prediction-failure-triage/s06-06-write-the-router"},{"id":"debugging-output/prediction-failure-triage/s07-07-checkpoint"}]}]},{"slug":"capstone","lessons":[{"slug":"pick-a-project-that-ships","steps":[{"id":"capstone/pick-a-project-that-ships/s01-01-the-builders-graveyard"},{"id":"capstone/pick-a-project-that-ships/s02-02-which-project-survives-contact-with-reality"},{"id":"capstone/pick-a-project-that-ships/s03-03-the-three-shapes-that-ship"},{"id":"capstone/pick-a-project-that-ships/s04-04-the-wedge-checklist"},{"id":"capstone/pick-a-project-that-ships/s05-05-spot-the-missing-wedge"},{"id":"capstone/pick-a-project-that-ships/s06-06-scope-down-the-capstone"},{"id":"capstone/pick-a-project-that-ships/s07-07-checkpoint"}]},{"slug":"build-a-cli-agent","steps":[{"id":"capstone/build-a-cli-agent/s01-01-intro"},{"id":"capstone/build-a-cli-agent/s02-02-which-piece-is-missing"},{"id":"capstone/build-a-cli-agent/s03-03-the-tool-loop"},{"id":"capstone/build-a-cli-agent/s04-04-predict-the-final-answer"},{"id":"capstone/build-a-cli-agent/s05-05-fill-the-stop-check"},{"id":"capstone/build-a-cli-agent/s06-06-fix-the-tool-dispatch"},{"id":"capstone/build-a-cli-agent/s07-07-fix-the-message-append"},{"id":"capstone/build-a-cli-agent/s08-08-the-logging-pattern"},{"id":"capstone/build-a-cli-agent/s09-09-write-the-tool-runner"},{"id":"capstone/build-a-cli-agent/s10-10-fix-the-loop-exit"},{"id":"capstone/build-a-cli-agent/s11-11-write-the-agent"},{"id":"capstone/build-a-cli-agent/s12-12-checkpoint"}]},{"slug":"wire-the-real-model","steps":[{"id":"capstone/wire-the-real-model/s01-01-intro"},{"id":"capstone/wire-the-real-model/s02-02-where-the-key-lives"},{"id":"capstone/wire-the-real-model/s03-03-the-sdk-response-shape"},{"id":"capstone/wire-the-real-model/s04-04-predict-the-block-type"},{"id":"capstone/wire-the-real-model/s05-05-fill-the-key-load"},{"id":"capstone/wire-the-real-model/s06-06-fix-the-hardcoded-key"},{"id":"capstone/wire-the-real-model/s07-07-fix-the-block-access"},{"id":"capstone/wire-the-real-model/s08-08-write-the-shape-adapter"},{"id":"capstone/wire-the-real-model/s09-09-checkpoint"}]},{"slug":"validate-tool-inputs","steps":[{"id":"capstone/validate-tool-inputs/s01-01-intro"},{"id":"capstone/validate-tool-inputs/s02-02-which-input-needs-validation"},{"id":"capstone/validate-tool-inputs/s03-03-the-validator-surface"},{"id":"capstone/validate-tool-inputs/s04-04-predict-the-error"},{"id":"capstone/validate-tool-inputs/s05-05-fill-the-validate-call"},{"id":"capstone/validate-tool-inputs/s06-06-fix-the-untyped-tool"},{"id":"capstone/validate-tool-inputs/s07-07-fix-the-error-leak"},{"id":"capstone/validate-tool-inputs/s08-08-write-the-validator"},{"id":"capstone/validate-tool-inputs/s09-09-checkpoint"}]},{"slug":"add-evals-and-traces","steps":[{"id":"capstone/add-evals-and-traces/s01-01-intro"},{"id":"capstone/add-evals-and-traces/s02-02-which-eval-fits"},{"id":"capstone/add-evals-and-traces/s03-03-the-trace-shape"},{"id":"capstone/add-evals-and-traces/s04-04-predict-the-pass-rate"},{"id":"capstone/add-evals-and-traces/s05-05-fill-the-trace-append"},{"id":"capstone/add-evals-and-traces/s06-06-fix-the-trace-mutation"},{"id":"capstone/add-evals-and-traces/s07-07-fix-the-tautological-eval"},{"id":"capstone/add-evals-and-traces/s08-08-write-the-eval-runner"},{"id":"capstone/add-evals-and-traces/s09-09-checkpoint"}]},{"slug":"wire-an-mcp-tool","steps":[{"id":"capstone/wire-an-mcp-tool/s01-01-intro"},{"id":"capstone/wire-an-mcp-tool/s02-02-when-to-use-mcp"},{"id":"capstone/wire-an-mcp-tool/s03-03-the-mcp-protocol"},{"id":"capstone/wire-an-mcp-tool/s04-04-predict-the-tool-list"},{"id":"capstone/wire-an-mcp-tool/s05-05-fill-the-tools-call"},{"id":"capstone/wire-an-mcp-tool/s06-06-fix-the-result-shape"},{"id":"capstone/wire-an-mcp-tool/s07-07-fix-the-iserror-flag"},{"id":"capstone/wire-an-mcp-tool/s08-08-write-the-mcp-bridge"},{"id":"capstone/wire-an-mcp-tool/s09-09-checkpoint"}]},{"slug":"portfolio-ml-system-option","steps":[{"id":"capstone/portfolio-ml-system-option/s01-01-the-other-capstone"},{"id":"capstone/portfolio-ml-system-option/s02-02-what-reviewers-open"},{"id":"capstone/portfolio-ml-system-option/s03-03-predict-the-gate"},{"id":"capstone/portfolio-ml-system-option/s04-04-fill-the-gate"},{"id":"capstone/portfolio-ml-system-option/s05-05-fix-the-train-accuracy-gate"},{"id":"capstone/portfolio-ml-system-option/s06-06-write-the-scope-check"},{"id":"capstone/portfolio-ml-system-option/s07-07-checkpoint"}]}]},{"slug":"agent-harnesses","lessons":[{"slug":"project-workspaces-not-product-names","steps":[{"id":"agent-harnesses/project-workspaces-not-product-names/s01-01-intro"}]},{"slug":"what-a-harness-is","steps":[{"id":"agent-harnesses/what-a-harness-is/s01-01-intro"},{"id":"agent-harnesses/what-a-harness-is/s02-02-which-is-a-harness"},{"id":"agent-harnesses/what-a-harness-is/s03-03-the-four-layers"},{"id":"agent-harnesses/what-a-harness-is/s04-04-predict-the-layer"},{"id":"agent-harnesses/what-a-harness-is/s05-05-fill-the-dispatch"},{"id":"agent-harnesses/what-a-harness-is/s06-06-fix-the-no-retry"},{"id":"agent-harnesses/what-a-harness-is/s07-07-fix-the-hardcoded-registry"},{"id":"agent-harnesses/what-a-harness-is/s08-08-write-mini-harness"},{"id":"agent-harnesses/what-a-harness-is/s09-09-checkpoint"}]},{"slug":"architecting-an-ai-native-workflow","steps":[{"id":"agent-harnesses/architecting-an-ai-native-workflow/s01-01-intro"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s02-02-which-workflow-fits"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s03-03-mapping-the-workflow"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s04-04-predict-the-readiness-score"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s05-05-fill-the-required-fields"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s06-06-fix-the-implicit-policy"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s07-07-fix-the-wrong-metric"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s08-08-write-the-workflow-validator"},{"id":"agent-harnesses/architecting-an-ai-native-workflow/s09-09-checkpoint"}]},{"slug":"five-industries-walked-through","steps":[{"id":"agent-harnesses/five-industries-walked-through/s01-01-the-case-study-method"},{"id":"agent-harnesses/five-industries-walked-through/s02-02-home-services"},{"id":"agent-harnesses/five-industries-walked-through/s03-03-which-task-belongs-to-the-agent"},{"id":"agent-harnesses/five-industries-walked-through/s04-04-insurance-brokerage"},{"id":"agent-harnesses/five-industries-walked-through/s05-05-recruiting-firm"},{"id":"agent-harnesses/five-industries-walked-through/s06-06-which-industry-wedges-fastest"},{"id":"agent-harnesses/five-industries-walked-through/s07-07-the-incumbent-trap"},{"id":"agent-harnesses/five-industries-walked-through/s08-08-write-the-process-debt-audit"},{"id":"agent-harnesses/five-industries-walked-through/s09-09-checkpoint"}]},{"slug":"incident-runbook-from-the-repo","steps":[{"id":"agent-harnesses/incident-runbook-from-the-repo/s01-01-intro"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s02-02-which-is-a-runbook"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s03-03-detect-decide-writeback"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s04-04-predict-the-decision"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s05-05-fill-the-decision-rule"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s06-06-fix-the-wiki-page"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s07-07-fix-the-unnamed-human"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s08-08-write-the-runbook"},{"id":"agent-harnesses/incident-runbook-from-the-repo/s09-09-checkpoint"}]}]},{"slug":"ai-image-generation","lessons":[{"slug":"the-image-model-landscape","steps":[{"id":"ai-image-generation/the-image-model-landscape/s01-01-the-six-families"},{"id":"ai-image-generation/the-image-model-landscape/s02-02-flux-vs-midjourney"},{"id":"ai-image-generation/the-image-model-landscape/s03-03-pick-the-model"},{"id":"ai-image-generation/the-image-model-landscape/s04-04-nano-banana-and-the-batch-economics"},{"id":"ai-image-generation/the-image-model-landscape/s05-05-text-in-image"},{"id":"ai-image-generation/the-image-model-landscape/s06-06-which-model-for-text"},{"id":"ai-image-generation/the-image-model-landscape/s07-07-the-decision-tree"},{"id":"ai-image-generation/the-image-model-landscape/s08-08-write-pick-image-model"},{"id":"ai-image-generation/the-image-model-landscape/s09-09-checkpoint"}]},{"slug":"prompting-for-real-output","steps":[{"id":"ai-image-generation/prompting-for-real-output/s01-01-why-the-girl-with-red-hair-fails"},{"id":"ai-image-generation/prompting-for-real-output/s02-02-the-eight-knobs"},{"id":"ai-image-generation/prompting-for-real-output/s03-03-fix-the-vague-prompt"},{"id":"ai-image-generation/prompting-for-real-output/s04-04-controlnet-and-reference"},{"id":"ai-image-generation/prompting-for-real-output/s05-05-which-knob-is-missing"},{"id":"ai-image-generation/prompting-for-real-output/s06-06-failure-modes"},{"id":"ai-image-generation/prompting-for-real-output/s07-07-write-score-image-prompt"},{"id":"ai-image-generation/prompting-for-real-output/s08-08-checkpoint"}]},{"slug":"the-image-pipeline","steps":[{"id":"ai-image-generation/the-image-pipeline/s01-01-the-six-stages"},{"id":"ai-image-generation/the-image-pipeline/s02-02-batch-and-filter"},{"id":"ai-image-generation/the-image-pipeline/s03-03-which-stage-is-broken"},{"id":"ai-image-generation/the-image-pipeline/s04-04-format-and-platform"},{"id":"ai-image-generation/the-image-pipeline/s05-05-fill-the-pipeline"},{"id":"ai-image-generation/the-image-pipeline/s06-06-write-cost-for-batch"},{"id":"ai-image-generation/the-image-pipeline/s07-07-checkpoint"}]}]},{"slug":"ai-video-generation","lessons":[{"slug":"the-video-model-lineup","steps":[{"id":"ai-video-generation/the-video-model-lineup/s01-01-the-landscape"},{"id":"ai-video-generation/the-video-model-lineup/s02-02-sora-and-veo"},{"id":"ai-video-generation/the-video-model-lineup/s03-03-higgsfield-and-camera-control"},{"id":"ai-video-generation/the-video-model-lineup/s04-04-the-second-wave"},{"id":"ai-video-generation/the-video-model-lineup/s05-05-text-vs-image-vs-video"},{"id":"ai-video-generation/the-video-model-lineup/s06-06-which-lane"},{"id":"ai-video-generation/the-video-model-lineup/s07-07-pick-the-model"},{"id":"ai-video-generation/the-video-model-lineup/s08-08-write-pick-video-model"},{"id":"ai-video-generation/the-video-model-lineup/s09-09-checkpoint"}]},{"slug":"camera-control-and-motion","steps":[{"id":"ai-video-generation/camera-control-and-motion/s01-01-the-static-camera-problem"},{"id":"ai-video-generation/camera-control-and-motion/s02-02-shot-vocabulary"},{"id":"ai-video-generation/camera-control-and-motion/s03-03-which-shot-is-which"},{"id":"ai-video-generation/camera-control-and-motion/s04-04-camera-moves"},{"id":"ai-video-generation/camera-control-and-motion/s05-05-the-i2v-keyframe-pattern"},{"id":"ai-video-generation/camera-control-and-motion/s06-06-spot-the-generic-prompt"},{"id":"ai-video-generation/camera-control-and-motion/s07-07-write-audit-shot-list"},{"id":"ai-video-generation/camera-control-and-motion/s08-08-checkpoint"}]},{"slug":"the-cost-math","steps":[{"id":"ai-video-generation/the-cost-math/s01-01-the-unit-economics"},{"id":"ai-video-generation/the-cost-math/s02-02-the-retake-rate-reality"},{"id":"ai-video-generation/the-cost-math/s03-03-which-tier-for-this-budget"},{"id":"ai-video-generation/the-cost-math/s04-04-the-60-second-promo"},{"id":"ai-video-generation/the-cost-math/s05-05-write-cost-per-minute"},{"id":"ai-video-generation/the-cost-math/s06-06-checkpoint"}]}]},{"slug":"programmatic-design","lessons":[{"slug":"why-programmatic","steps":[{"id":"programmatic-design/why-programmatic/s01-01-intro"},{"id":"programmatic-design/why-programmatic/s02-02-the-three-jobs"},{"id":"programmatic-design/why-programmatic/s03-03-which-task-is-code"},{"id":"programmatic-design/why-programmatic/s04-04-parametric-vs-one-shot"},{"id":"programmatic-design/why-programmatic/s05-05-the-classifier"},{"id":"programmatic-design/why-programmatic/s06-06-write-classify-video-task"},{"id":"programmatic-design/why-programmatic/s07-07-checkpoint"}]},{"slug":"hyperframes-and-remotion","steps":[{"id":"programmatic-design/hyperframes-and-remotion/s01-01-the-two-tools"},{"id":"programmatic-design/hyperframes-and-remotion/s02-02-hyperframes-tour"},{"id":"programmatic-design/hyperframes-and-remotion/s03-03-hyperframes-cli"},{"id":"programmatic-design/hyperframes-and-remotion/s04-04-remotion-tour"},{"id":"programmatic-design/hyperframes-and-remotion/s05-05-the-decision-tree"},{"id":"programmatic-design/hyperframes-and-remotion/s06-06-pick-the-tool"},{"id":"programmatic-design/hyperframes-and-remotion/s07-07-claude-design-html"},{"id":"programmatic-design/hyperframes-and-remotion/s08-08-write-pick-tool"},{"id":"programmatic-design/hyperframes-and-remotion/s09-09-checkpoint"}]},{"slug":"the-ai-native-design-pipeline","steps":[{"id":"programmatic-design/the-ai-native-design-pipeline/s01-01-the-seven-steps"},{"id":"programmatic-design/the-ai-native-design-pipeline/s02-02-where-ai-fits-each-step"},{"id":"programmatic-design/the-ai-native-design-pipeline/s03-03-which-model-which-step"},{"id":"programmatic-design/the-ai-native-design-pipeline/s04-04-website-to-hyperframes"},{"id":"programmatic-design/the-ai-native-design-pipeline/s05-05-the-cost-model"},{"id":"programmatic-design/the-ai-native-design-pipeline/s06-06-score-the-pipeline"},{"id":"programmatic-design/the-ai-native-design-pipeline/s07-07-write-score-pipeline"},{"id":"programmatic-design/the-ai-native-design-pipeline/s08-08-checkpoint"}]}]},{"slug":"harness-engineering","lessons":[{"slug":"the-harness-engineering-mindset","steps":[{"id":"harness-engineering/the-harness-engineering-mindset/s01-01-the-model-is-only-one-input"},{"id":"harness-engineering/the-harness-engineering-mindset/s02-02-the-six-pieces-of-a-harness"},{"id":"harness-engineering/the-harness-engineering-mindset/s03-03-which-piece-is-missing"},{"id":"harness-engineering/the-harness-engineering-mindset/s04-04-the-skill-issue-reframe"},{"id":"harness-engineering/the-harness-engineering-mindset/s05-05-pick-the-fix"},{"id":"harness-engineering/the-harness-engineering-mindset/s06-06-the-harness-gap"},{"id":"harness-engineering/the-harness-engineering-mindset/s07-07-write-the-harness-inventory"},{"id":"harness-engineering/the-harness-engineering-mindset/s08-08-checkpoint"}]},{"slug":"the-ratchet","steps":[{"id":"harness-engineering/the-ratchet/s01-01-failures-are-signals-not-flukes"},{"id":"harness-engineering/the-ratchet/s02-02-every-line-traces-to-a-failure"},{"id":"harness-engineering/the-ratchet/s03-03-which-rule-earned-its-line"},{"id":"harness-engineering/the-ratchet/s04-04-working-backwards-from-behavior"},{"id":"harness-engineering/the-ratchet/s05-05-the-ratchet-log"},{"id":"harness-engineering/the-ratchet/s06-06-spot-the-cruft"},{"id":"harness-engineering/the-ratchet/s07-07-write-the-ratchet-update"},{"id":"harness-engineering/the-ratchet/s08-08-audit-the-ratchet"},{"id":"harness-engineering/the-ratchet/s09-09-checkpoint"}]},{"slug":"context-engineering","steps":[{"id":"harness-engineering/context-engineering/s01-01-the-context-rot"},{"id":"harness-engineering/context-engineering/s02-02-compaction"},{"id":"harness-engineering/context-engineering/s03-03-tool-call-offloading"},{"id":"harness-engineering/context-engineering/s04-04-progressive-disclosure"},{"id":"harness-engineering/context-engineering/s05-05-which-technique-fits"},{"id":"harness-engineering/context-engineering/s06-06-the-memory-hierarchy"},{"id":"harness-engineering/context-engineering/s07-07-classify-the-data"},{"id":"harness-engineering/context-engineering/s08-08-write-the-context-budget"},{"id":"harness-engineering/context-engineering/s09-09-checkpoint"}]},{"slug":"long-horizon-execution","steps":[{"id":"harness-engineering/long-horizon-execution/s01-01-the-40-step-problem"},{"id":"harness-engineering/long-horizon-execution/s02-02-loops-intercepting-early-stops"},{"id":"harness-engineering/long-horizon-execution/s03-03-planning-as-a-separate-step"},{"id":"harness-engineering/long-horizon-execution/s04-04-splits-generation-vs-evaluation"},{"id":"harness-engineering/long-horizon-execution/s05-05-which-pattern-fixes-the-failure"},{"id":"harness-engineering/long-horizon-execution/s06-06-hooks-as-the-enforcement-layer"},{"id":"harness-engineering/long-horizon-execution/s07-07-the-type-check-backpressure-pattern"},{"id":"harness-engineering/long-horizon-execution/s08-08-write-the-hook-router"},{"id":"harness-engineering/long-horizon-execution/s09-09-write-the-completion-guard"},{"id":"harness-engineering/long-horizon-execution/s10-10-checkpoint"}]},{"slug":"the-haas-shift","steps":[{"id":"harness-engineering/the-haas-shift/s01-01-llm-apis-to-harness-apis"},{"id":"harness-engineering/the-haas-shift/s02-02-the-convergence"},{"id":"harness-engineering/the-haas-shift/s03-03-which-haas-fits"},{"id":"harness-engineering/the-haas-shift/s04-04-harnesses-dont-shrink-they-move"},{"id":"harness-engineering/the-haas-shift/s05-05-the-overfitting-feedback-loop"},{"id":"harness-engineering/the-haas-shift/s06-06-mcp-security-as-prompt-injection"},{"id":"harness-engineering/the-haas-shift/s07-07-which-component-is-debt"},{"id":"harness-engineering/the-haas-shift/s08-08-write-the-build-vs-buy"},{"id":"harness-engineering/the-haas-shift/s09-09-checkpoint"}]},{"slug":"pipeline-boundaries","steps":[{"id":"harness-engineering/pipeline-boundaries/s01-01-the-boundary-is-an-artifact"},{"id":"harness-engineering/pipeline-boundaries/s02-02-spot-the-risky-handoff"},{"id":"harness-engineering/pipeline-boundaries/s03-03-predict-the-gate"},{"id":"harness-engineering/pipeline-boundaries/s04-04-fill-the-gate"},{"id":"harness-engineering/pipeline-boundaries/s05-05-fix-the-overtrust"},{"id":"harness-engineering/pipeline-boundaries/s06-06-write-the-advance-check"},{"id":"harness-engineering/pipeline-boundaries/s07-07-checkpoint"}]}]},{"slug":"intro-to-terminal","lessons":[{"slug":"what-the-terminal-is","steps":[{"id":"intro-to-terminal/what-the-terminal-is/s01-01-not-a-cockpit"},{"id":"intro-to-terminal/what-the-terminal-is/s02-02-what-it-actually-is"},{"id":"intro-to-terminal/what-the-terminal-is/s03-03-open-it"},{"id":"intro-to-terminal/what-the-terminal-is/s04-04-which-app"},{"id":"intro-to-terminal/what-the-terminal-is/s05-05-the-prompt"},{"id":"intro-to-terminal/what-the-terminal-is/s06-06-what-a-command-is"},{"id":"intro-to-terminal/what-the-terminal-is/s07-07-checkpoint"}]},{"slug":"moving-around","steps":[{"id":"intro-to-terminal/moving-around/s01-01-the-filesystem-is-a-tree"},{"id":"intro-to-terminal/moving-around/s02-02-pwd-and-ls"},{"id":"intro-to-terminal/moving-around/s03-03-where-am-i"},{"id":"intro-to-terminal/moving-around/s04-04-read-this-listing"},{"id":"intro-to-terminal/moving-around/s05-05-cd"},{"id":"intro-to-terminal/moving-around/s06-06-follow-the-moves"},{"id":"intro-to-terminal/moving-around/s07-07-pick-the-command"},{"id":"intro-to-terminal/moving-around/s08-08-checkpoint"}]},{"slug":"making-things","steps":[{"id":"intro-to-terminal/making-things/s01-01-mkdir-and-touch"},{"id":"intro-to-terminal/making-things/s02-02-folder-or-file"},{"id":"intro-to-terminal/making-things/s03-03-cat-and-echo"},{"id":"intro-to-terminal/making-things/s04-04-reading-a-file"},{"id":"intro-to-terminal/making-things/s05-05-rm-carefully"},{"id":"intro-to-terminal/making-things/s06-06-the-no-undo-rule"},{"id":"intro-to-terminal/making-things/s07-07-string-it-together"},{"id":"intro-to-terminal/making-things/s08-08-checkpoint"}]}]},{"slug":"intro-to-claude-cli","lessons":[{"slug":"what-the-claude-cli-is","steps":[{"id":"intro-to-claude-cli/what-the-claude-cli-is/s01-01-the-glass-is-gone"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s02-02-chat-vs-cli"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s03-03-which-tool"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s04-04-the-agent-loop"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s05-05-read-the-loop"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s06-06-permission-and-limits"},{"id":"intro-to-claude-cli/what-the-claude-cli-is/s07-07-checkpoint"}]},{"slug":"install-and-sign-in","steps":[{"id":"intro-to-claude-cli/install-and-sign-in/s01-01-two-ways-to-install"},{"id":"intro-to-claude-cli/install-and-sign-in/s02-02-run-the-installer"},{"id":"intro-to-claude-cli/install-and-sign-in/s03-03-install-check"},{"id":"intro-to-claude-cli/install-and-sign-in/s04-04-sign-in"},{"id":"intro-to-claude-cli/install-and-sign-in/s05-05-what-sign-in-did"},{"id":"intro-to-claude-cli/install-and-sign-in/s06-06-auth-check"},{"id":"intro-to-claude-cli/install-and-sign-in/s07-07-account-not-api-key"},{"id":"intro-to-claude-cli/install-and-sign-in/s08-08-checkpoint"}]},{"slug":"your-first-session","steps":[{"id":"intro-to-claude-cli/your-first-session/s01-01-start-in-a-folder"},{"id":"intro-to-claude-cli/your-first-session/s02-02-your-first-instruction"},{"id":"intro-to-claude-cli/your-first-session/s03-03-watch-the-loop"},{"id":"intro-to-claude-cli/your-first-session/s04-04-read-what-it-did"},{"id":"intro-to-claude-cli/your-first-session/s05-05-one-shot-or-interactive"},{"id":"intro-to-claude-cli/your-first-session/s06-06-chat-or-cli-decision"},{"id":"intro-to-claude-cli/your-first-session/s07-07-where-this-goes"},{"id":"intro-to-claude-cli/your-first-session/s08-08-checkpoint"}]}]},{"slug":"intro-to-codex-cli","lessons":[{"slug":"another-tool-same-idea","steps":[{"id":"intro-to-codex-cli/another-tool-same-idea/s01-01-why-a-second-tool"},{"id":"intro-to-codex-cli/another-tool-same-idea/s02-02-what-carries-over"},{"id":"intro-to-codex-cli/another-tool-same-idea/s03-03-what-carries-over-check"},{"id":"intro-to-codex-cli/another-tool-same-idea/s04-04-what-is-different"},{"id":"intro-to-codex-cli/another-tool-same-idea/s05-05-spot-the-difference"},{"id":"intro-to-codex-cli/another-tool-same-idea/s06-06-when-to-reach-for-which"},{"id":"intro-to-codex-cli/another-tool-same-idea/s07-07-checkpoint"}]},{"slug":"install-and-sign-in","steps":[{"id":"intro-to-codex-cli/install-and-sign-in/s01-01-the-install-paths"},{"id":"intro-to-codex-cli/install-and-sign-in/s02-02-installing-it"},{"id":"intro-to-codex-cli/install-and-sign-in/s03-03-verify-it"},{"id":"intro-to-codex-cli/install-and-sign-in/s04-04-install-check"},{"id":"intro-to-codex-cli/install-and-sign-in/s05-05-sign-in"},{"id":"intro-to-codex-cli/install-and-sign-in/s06-06-account-vs-api-key"},{"id":"intro-to-codex-cli/install-and-sign-in/s07-07-auth-check"},{"id":"intro-to-codex-cli/install-and-sign-in/s08-08-checkpoint"}]},{"slug":"your-first-session","steps":[{"id":"intro-to-codex-cli/your-first-session/s01-01-start-and-instruct"},{"id":"intro-to-codex-cli/your-first-session/s02-02-the-loop-again"},{"id":"intro-to-codex-cli/your-first-session/s03-03-read-what-it-did"},{"id":"intro-to-codex-cli/your-first-session/s04-04-usage-patterns"},{"id":"intro-to-codex-cli/your-first-session/s05-05-usage-scenario"},{"id":"intro-to-codex-cli/your-first-session/s06-06-running-both"},{"id":"intro-to-codex-cli/your-first-session/s07-07-where-this-goes"},{"id":"intro-to-codex-cli/your-first-session/s08-08-checkpoint"}]}]},{"slug":"team-skills","lessons":[{"slug":"what-a-skill-is","steps":[{"id":"team-skills/what-a-skill-is/s01-01-the-re-explaining-problem"},{"id":"team-skills/what-a-skill-is/s02-02-a-skill-defined"},{"id":"team-skills/what-a-skill-is/s03-03-a-real-skill"},{"id":"team-skills/what-a-skill-is/s04-04-what-a-skill-is-check"},{"id":"team-skills/what-a-skill-is/s05-05-skill-vs-prompt-vs-agent"},{"id":"team-skills/what-a-skill-is/s06-06-skill-or-agent"},{"id":"team-skills/what-a-skill-is/s07-07-checkpoint"}]},{"slug":"skills-as-a-team-thing","steps":[{"id":"team-skills/skills-as-a-team-thing/s01-01-the-team-problem"},{"id":"team-skills/skills-as-a-team-thing/s02-02-org-wide-skills"},{"id":"team-skills/skills-as-a-team-thing/s03-03-sharing-and-the-directory"},{"id":"team-skills/skills-as-a-team-thing/s04-04-deployment-check"},{"id":"team-skills/skills-as-a-team-thing/s05-05-collaborating-through-skills"},{"id":"team-skills/skills-as-a-team-thing/s06-06-sharing-scenario"},{"id":"team-skills/skills-as-a-team-thing/s07-07-checkpoint"}]},{"slug":"role-playbooks","steps":[{"id":"team-skills/role-playbooks/s01-01-hr-skills"},{"id":"team-skills/role-playbooks/s02-02-legal-skills"},{"id":"team-skills/role-playbooks/s03-03-operations-skills"},{"id":"team-skills/role-playbooks/s04-04-match-the-skill"},{"id":"team-skills/role-playbooks/s05-05-skill-or-one-off"},{"id":"team-skills/role-playbooks/s06-06-skill-or-one-off-check"},{"id":"team-skills/role-playbooks/s07-07-checkpoint"}]},{"slug":"governance-and-trust","steps":[{"id":"team-skills/governance-and-trust/s01-01-a-skill-can-do-real-things"},{"id":"team-skills/governance-and-trust/s02-02-treat-it-like-software"},{"id":"team-skills/governance-and-trust/s03-03-evaluate-before-you-deploy"},{"id":"team-skills/governance-and-trust/s04-04-access-control"},{"id":"team-skills/governance-and-trust/s05-05-governance-scenario"},{"id":"team-skills/governance-and-trust/s06-06-a-rollout-checklist"},{"id":"team-skills/governance-and-trust/s07-07-checkpoint"}]}]},{"slug":"dataframes-numpy-pandas","lessons":[{"slug":"ml-package-map","steps":[{"id":"dataframes-numpy-pandas/ml-package-map/s01-01-intro"},{"id":"dataframes-numpy-pandas/ml-package-map/s02-02-match-the-tool"},{"id":"dataframes-numpy-pandas/ml-package-map/s03-03-predict-sys-modules"},{"id":"dataframes-numpy-pandas/ml-package-map/s04-04-fill-the-registry-key"},{"id":"dataframes-numpy-pandas/ml-package-map/s05-05-fix-the-swapped-map"},{"id":"dataframes-numpy-pandas/ml-package-map/s06-06-write-find-unused"},{"id":"dataframes-numpy-pandas/ml-package-map/s07-07-checkpoint"}]},{"slug":"array-and-series-shapes","steps":[{"id":"dataframes-numpy-pandas/array-and-series-shapes/s01-01-intro"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s02-02-choose-the-risk"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s03-03-predict-the-checks"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s04-04-fill-the-return"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s05-05-fix-the-overtrust"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s06-06-write-count-ready"},{"id":"dataframes-numpy-pandas/array-and-series-shapes/s07-07-checkpoint"}]},{"slug":"dataframe-selection-and-cleaning","steps":[{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s01-01-intro"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s02-02-choose-the-risk"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s03-03-predict-the-checks"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s04-04-fill-the-return"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s05-05-fix-the-overtrust"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s06-06-write-count-ready"},{"id":"dataframes-numpy-pandas/dataframe-selection-and-cleaning/s07-07-checkpoint"}]},{"slug":"missing-values-and-types","steps":[{"id":"dataframes-numpy-pandas/missing-values-and-types/s01-01-intro"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s02-02-choose-the-risk"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s03-03-predict-the-checks"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s04-04-fill-the-return"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s05-05-fix-the-overtrust"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s06-06-write-count-ready"},{"id":"dataframes-numpy-pandas/missing-values-and-types/s07-07-checkpoint"}]},{"slug":"groupby-joins-and-features","steps":[{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s01-01-intro"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s02-02-choose-the-risk"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s03-03-predict-the-checks"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s04-04-fill-the-return"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s05-05-fix-the-overtrust"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s06-06-write-count-ready"},{"id":"dataframes-numpy-pandas/groupby-joins-and-features/s07-07-checkpoint"}]},{"slug":"mission-api-to-dataframe","steps":[{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s01-01-intro"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s02-02-choose-the-risk"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s03-03-predict-the-checks"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s04-04-fill-the-return"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s05-05-fix-the-overtrust"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s06-06-write-count-ready"},{"id":"dataframes-numpy-pandas/mission-api-to-dataframe/s07-07-checkpoint"}]}]},{"slug":"sql-for-ml-datasets","lessons":[{"slug":"select-filter-aggregate","steps":[{"id":"sql-for-ml-datasets/select-filter-aggregate/s01-01-intro"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s02-02-choose-the-risk"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s03-03-predict-the-checks"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s04-04-fill-the-return"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s05-05-fix-the-overtrust"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s06-06-write-count-ready"},{"id":"sql-for-ml-datasets/select-filter-aggregate/s07-07-checkpoint"}]},{"slug":"joins-and-label-windows","steps":[{"id":"sql-for-ml-datasets/joins-and-label-windows/s01-01-intro"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s02-02-choose-the-risk"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s03-03-predict-the-checks"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s04-04-fill-the-return"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s05-05-fix-the-overtrust"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s06-06-write-count-ready"},{"id":"sql-for-ml-datasets/joins-and-label-windows/s07-07-checkpoint"}]},{"slug":"sql-quality-checks","steps":[{"id":"sql-for-ml-datasets/sql-quality-checks/s01-01-intro"},{"id":"sql-for-ml-datasets/sql-quality-checks/s02-02-choose-the-risk"},{"id":"sql-for-ml-datasets/sql-quality-checks/s03-03-predict-the-checks"},{"id":"sql-for-ml-datasets/sql-quality-checks/s04-04-fill-the-return"},{"id":"sql-for-ml-datasets/sql-quality-checks/s05-05-fix-the-overtrust"},{"id":"sql-for-ml-datasets/sql-quality-checks/s06-06-write-count-ready"},{"id":"sql-for-ml-datasets/sql-quality-checks/s07-07-checkpoint"}]},{"slug":"mission-sql-feature-query-lab","steps":[{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s01-01-intro"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s02-02-choose-the-risk"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s03-03-predict-the-checks"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s04-04-fill-the-return"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s05-05-fix-the-overtrust"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s06-06-write-count-ready"},{"id":"sql-for-ml-datasets/mission-sql-feature-query-lab/s07-07-checkpoint"}]}]},{"slug":"dataset-pipelines","lessons":[{"slug":"csv-jsonl-parquet-tradeoffs","steps":[{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s01-01-intro"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s02-02-choose-the-risk"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s03-03-predict-the-checks"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s04-04-fill-the-return"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s05-05-fix-the-overtrust"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s06-06-write-count-ready"},{"id":"dataset-pipelines/csv-jsonl-parquet-tradeoffs/s07-07-checkpoint"}]},{"slug":"pagination-retries-checkpoints","steps":[{"id":"dataset-pipelines/pagination-retries-checkpoints/s01-01-intro"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s02-02-choose-the-risk"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s03-03-predict-the-checks"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s04-04-fill-the-return"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s05-05-fix-the-overtrust"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s06-06-write-count-ready"},{"id":"dataset-pipelines/pagination-retries-checkpoints/s07-07-checkpoint"}]},{"slug":"schema-drift-and-contracts","steps":[{"id":"dataset-pipelines/schema-drift-and-contracts/s01-01-intro"},{"id":"dataset-pipelines/schema-drift-and-contracts/s02-02-choose-the-risk"},{"id":"dataset-pipelines/schema-drift-and-contracts/s03-03-predict-the-checks"},{"id":"dataset-pipelines/schema-drift-and-contracts/s04-04-fill-the-return"},{"id":"dataset-pipelines/schema-drift-and-contracts/s05-05-fix-the-overtrust"},{"id":"dataset-pipelines/schema-drift-and-contracts/s06-06-write-count-ready"},{"id":"dataset-pipelines/schema-drift-and-contracts/s07-07-checkpoint"}]},{"slug":"pipeline-orchestration-shape","steps":[{"id":"dataset-pipelines/pipeline-orchestration-shape/s01-01-intro"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s02-02-choose-the-risk"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s03-03-predict-the-checks"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s04-04-fill-the-return"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s05-05-fix-the-overtrust"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s06-06-write-count-ready"},{"id":"dataset-pipelines/pipeline-orchestration-shape/s07-07-checkpoint"}]},{"slug":"mission-api-to-dataset-pipeline","steps":[{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s01-01-intro"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s02-02-choose-the-risk"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s03-03-predict-the-checks"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s04-04-fill-the-return"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s05-05-fix-the-overtrust"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s06-06-write-count-ready"},{"id":"dataset-pipelines/mission-api-to-dataset-pipeline/s07-07-checkpoint"}]}]},{"slug":"ml-math-and-stats","lessons":[{"slug":"vectors-matrices-dot-products","steps":[{"id":"ml-math-and-stats/vectors-matrices-dot-products/s01-01-intro"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s02-02-choose-the-risk"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s03-03-predict-the-checks"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s04-04-fill-the-return"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s05-05-fix-the-overtrust"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s06-06-write-count-ready"},{"id":"ml-math-and-stats/vectors-matrices-dot-products/s07-07-checkpoint"}]},{"slug":"probability-and-baselines","steps":[{"id":"ml-math-and-stats/probability-and-baselines/s01-01-intro"},{"id":"ml-math-and-stats/probability-and-baselines/s02-02-choose-the-risk"},{"id":"ml-math-and-stats/probability-and-baselines/s03-03-predict-the-checks"},{"id":"ml-math-and-stats/probability-and-baselines/s04-04-fill-the-return"},{"id":"ml-math-and-stats/probability-and-baselines/s05-05-fix-the-overtrust"},{"id":"ml-math-and-stats/probability-and-baselines/s06-06-write-count-ready"},{"id":"ml-math-and-stats/probability-and-baselines/s07-07-checkpoint"}]},{"slug":"distributions-sampling-variance","steps":[{"id":"ml-math-and-stats/distributions-sampling-variance/s01-01-intro"},{"id":"ml-math-and-stats/distributions-sampling-variance/s02-02-choose-the-risk"},{"id":"ml-math-and-stats/distributions-sampling-variance/s03-03-predict-the-checks"},{"id":"ml-math-and-stats/distributions-sampling-variance/s04-04-fill-the-return"},{"id":"ml-math-and-stats/distributions-sampling-variance/s05-05-fix-the-overtrust"},{"id":"ml-math-and-stats/distributions-sampling-variance/s06-06-write-variance"},{"id":"ml-math-and-stats/distributions-sampling-variance/s07-07-checkpoint"}]},{"slug":"bias-variance-and-leakage","steps":[{"id":"ml-math-and-stats/bias-variance-and-leakage/s01-01-intro"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s02-02-choose-the-risk"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s03-03-predict-the-checks"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s04-04-fill-the-return"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s05-05-fix-the-overtrust"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s06-06-write-diagnose"},{"id":"ml-math-and-stats/bias-variance-and-leakage/s07-07-checkpoint"}]},{"slug":"mission-statistical-sanity-check","steps":[{"id":"ml-math-and-stats/mission-statistical-sanity-check/s01-01-intro"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s02-02-choose-the-risk"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s03-03-predict-the-checks"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s04-04-fill-the-return"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s05-05-fix-the-overtrust"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s06-06-write-count-ready"},{"id":"ml-math-and-stats/mission-statistical-sanity-check/s07-07-checkpoint"}]}]},{"slug":"supervised-learning-workflows","lessons":[{"slug":"labels-features-splits","steps":[{"id":"supervised-learning-workflows/labels-features-splits/s01-01-intro"},{"id":"supervised-learning-workflows/labels-features-splits/s02-02-choose-the-risk"},{"id":"supervised-learning-workflows/labels-features-splits/s03-03-predict-the-checks"},{"id":"supervised-learning-workflows/labels-features-splits/s04-04-fill-the-return"},{"id":"supervised-learning-workflows/labels-features-splits/s05-05-fix-the-overtrust"},{"id":"supervised-learning-workflows/labels-features-splits/s06-06-write-split-audit"},{"id":"supervised-learning-workflows/labels-features-splits/s07-07-checkpoint"}]},{"slug":"baselines-before-fancy-models","steps":[{"id":"supervised-learning-workflows/baselines-before-fancy-models/s01-01-intro"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s02-02-choose-the-risk"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s03-03-predict-the-checks"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s04-04-fill-the-return"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s05-05-fix-the-overtrust"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s06-06-write-baseline-comparison"},{"id":"supervised-learning-workflows/baselines-before-fancy-models/s07-07-checkpoint"}]},{"slug":"training-and-prediction-loop","steps":[{"id":"supervised-learning-workflows/training-and-prediction-loop/s01-01-intro"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s02-02-choose-the-risk"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s03-03-predict-the-checks"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s04-04-fill-the-return"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s05-05-fix-the-overtrust"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s06-06-write-train"},{"id":"supervised-learning-workflows/training-and-prediction-loop/s07-07-checkpoint"}]},{"slug":"overfitting-and-regularization","steps":[{"id":"supervised-learning-workflows/overfitting-and-regularization/s01-01-intro"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s02-02-choose-the-risk"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s03-03-predict-the-checks"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s04-04-fill-the-return"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s05-05-fix-the-overtrust"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s06-06-write-overfit-audit"},{"id":"supervised-learning-workflows/overfitting-and-regularization/s07-07-checkpoint"}]},{"slug":"mission-baseline-model-showdown","steps":[{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s01-01-intro"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s02-02-choose-the-risk"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s03-03-predict-the-checks"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s04-04-fill-the-return"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s05-05-fix-the-overtrust"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s06-06-write-classifier-brief"},{"id":"supervised-learning-workflows/mission-baseline-model-showdown/s07-07-checkpoint"}]}]},{"slug":"unsupervised-learning-and-embeddings","lessons":[{"slug":"clustering-without-labels","steps":[{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s01-01-intro"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s02-02-choose-the-risk"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s03-03-predict-the-checks"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s04-04-fill-the-return"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s05-05-fix-the-overtrust"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s06-06-write-cluster-summary"},{"id":"unsupervised-learning-and-embeddings/clustering-without-labels/s07-07-checkpoint"}]},{"slug":"dimensionality-and-distance","steps":[{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s01-01-intro"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s02-02-choose-the-risk"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s03-03-predict-the-checks"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s04-04-fill-the-return"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s05-05-fix-the-overtrust"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s06-06-write-euclidean"},{"id":"unsupervised-learning-and-embeddings/dimensionality-and-distance/s07-07-checkpoint"}]},{"slug":"embeddings-as-features","steps":[{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s01-01-intro"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s02-02-choose-the-risk"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s03-03-predict-the-checks"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s04-04-fill-the-return"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s05-05-fix-the-overtrust"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s06-06-write-retrieval-receipt"},{"id":"unsupervised-learning-and-embeddings/embeddings-as-features/s07-07-checkpoint"}]},{"slug":"mission-tiny-recommender","steps":[{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s01-01-intro"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s02-02-choose-the-risk"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s03-03-predict-the-checks"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s04-04-fill-the-return"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s05-05-fix-the-overtrust"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s06-06-write-recommendation-card"},{"id":"unsupervised-learning-and-embeddings/mission-tiny-recommender/s07-07-checkpoint"}]}]},{"slug":"metrics-and-error-analysis","lessons":[{"slug":"confusion-matrix","steps":[{"id":"metrics-and-error-analysis/confusion-matrix/s01-01-intro"},{"id":"metrics-and-error-analysis/confusion-matrix/s02-02-choose-the-risk"},{"id":"metrics-and-error-analysis/confusion-matrix/s03-03-predict-the-checks"},{"id":"metrics-and-error-analysis/confusion-matrix/s04-04-fill-the-return"},{"id":"metrics-and-error-analysis/confusion-matrix/s05-05-fix-the-overtrust"},{"id":"metrics-and-error-analysis/confusion-matrix/s06-06-write-count-ready"},{"id":"metrics-and-error-analysis/confusion-matrix/s07-07-checkpoint"}]},{"slug":"precision-recall-thresholds","steps":[{"id":"metrics-and-error-analysis/precision-recall-thresholds/s01-01-intro"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s02-02-choose-the-risk"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s03-03-predict-the-checks"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s04-04-fill-the-return"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s05-05-fix-the-overtrust"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s06-06-write-count-ready"},{"id":"metrics-and-error-analysis/precision-recall-thresholds/s07-07-checkpoint"}]},{"slug":"slice-based-error-analysis","steps":[{"id":"metrics-and-error-analysis/slice-based-error-analysis/s01-01-intro"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s02-02-choose-the-risk"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s03-03-predict-the-checks"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s04-04-fill-the-return"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s05-05-fix-the-overtrust"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s06-06-write-count-ready"},{"id":"metrics-and-error-analysis/slice-based-error-analysis/s07-07-checkpoint"}]},{"slug":"regression-metrics-and-residuals","steps":[{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s01-01-intro"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s02-02-choose-the-risk"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s03-03-predict-the-checks"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s04-04-fill-the-return"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s05-05-fix-the-overtrust"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s06-06-write-count-ready"},{"id":"metrics-and-error-analysis/regression-metrics-and-residuals/s07-07-checkpoint"}]},{"slug":"mission-confusion-matrix-triage","steps":[{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s01-01-intro"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s02-02-choose-the-risk"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s03-03-predict-the-checks"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s04-04-fill-the-return"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s05-05-fix-the-overtrust"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s06-06-write-count-ready"},{"id":"metrics-and-error-analysis/mission-confusion-matrix-triage/s07-07-checkpoint"}]}]},{"slug":"pytorch-tensors-and-autograd","lessons":[{"slug":"tensors-and-shapes","steps":[{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s01-01-intro"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s02-02-choose-the-risk"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s03-03-predict-the-checks"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s04-04-fill-the-return"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s05-05-fix-the-overtrust"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s06-06-write-count-ready"},{"id":"pytorch-tensors-and-autograd/tensors-and-shapes/s07-07-checkpoint"}]},{"slug":"broadcasting-and-vectorization","steps":[{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s01-01-intro"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s02-02-choose-the-risk"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s03-03-predict-the-checks"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s04-04-fill-the-return"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s05-05-fix-the-overtrust"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s06-06-write-count-ready"},{"id":"pytorch-tensors-and-autograd/broadcasting-and-vectorization/s07-07-checkpoint"}]},{"slug":"autograd-mental-model","steps":[{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s01-01-intro"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s02-02-choose-the-risk"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s03-03-predict-the-checks"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s04-04-fill-the-return"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s05-05-fix-the-overtrust"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s06-06-write-count-ready"},{"id":"pytorch-tensors-and-autograd/autograd-mental-model/s07-07-checkpoint"}]},{"slug":"mission-tensor-playground","steps":[{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s01-01-intro"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s02-02-choose-the-risk"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s03-03-predict-the-checks"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s04-04-fill-the-return"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s05-05-fix-the-overtrust"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s06-06-write-count-ready"},{"id":"pytorch-tensors-and-autograd/mission-tensor-playground/s07-07-checkpoint"}]}]},{"slug":"training-loops-and-optimizers","lessons":[{"slug":"loss-forward-backward-step","steps":[{"id":"training-loops-and-optimizers/loss-forward-backward-step/s01-01-intro"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s02-02-choose-the-risk"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s03-03-predict-the-checks"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s04-04-fill-the-return"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s05-05-fix-the-overtrust"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s06-06-write-count-ready"},{"id":"training-loops-and-optimizers/loss-forward-backward-step/s07-07-checkpoint"}]},{"slug":"gradient-descent-by-hand","steps":[{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s01-01-intro"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s02-02-choose-the-risk"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s03-03-predict-the-checks"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s04-04-fill-the-return"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s05-05-fix-the-overtrust"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s06-06-write-count-ready"},{"id":"training-loops-and-optimizers/gradient-descent-by-hand/s07-07-checkpoint"}]},{"slug":"optimizers-and-schedulers","steps":[{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s01-01-intro"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s02-02-choose-the-risk"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s03-03-predict-the-checks"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s04-04-fill-the-return"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s05-05-fix-the-overtrust"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s06-06-write-count-ready"},{"id":"training-loops-and-optimizers/optimizers-and-schedulers/s07-07-checkpoint"}]},{"slug":"checkpoints-and-reproducibility","steps":[{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s01-01-intro"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s02-02-choose-the-risk"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s03-03-predict-the-checks"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s04-04-fill-the-return"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s05-05-fix-the-overtrust"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s06-06-write-count-ready"},{"id":"training-loops-and-optimizers/checkpoints-and-reproducibility/s07-07-checkpoint"}]},{"slug":"mission-overfit-then-recover","steps":[{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s01-01-intro"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s02-02-choose-the-risk"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s03-03-predict-the-checks"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s04-04-fill-the-return"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s05-05-fix-the-overtrust"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s06-06-write-count-ready"},{"id":"training-loops-and-optimizers/mission-overfit-then-recover/s07-07-checkpoint"}]}]},{"slug":"deep-learning-architectures","lessons":[{"slug":"cnns-and-local-patterns","steps":[{"id":"deep-learning-architectures/cnns-and-local-patterns/s01-01-intro"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s02-02-choose-the-risk"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s03-03-predict-the-checks"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s04-04-fill-the-return"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s05-05-fix-the-overtrust"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s06-06-write-count-ready"},{"id":"deep-learning-architectures/cnns-and-local-patterns/s07-07-checkpoint"}]},{"slug":"tokenizers-and-context-budget","steps":[{"id":"deep-learning-architectures/tokenizers-and-context-budget/s01-01-intro"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s02-02-choose-the-risk"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s03-03-predict-the-checks"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s04-04-fill-the-return"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s05-05-fix-the-overtrust"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s06-06-write-count-ready"},{"id":"deep-learning-architectures/tokenizers-and-context-budget/s07-07-checkpoint"}]},{"slug":"attention-and-transformer-blocks","steps":[{"id":"deep-learning-architectures/attention-and-transformer-blocks/s01-01-intro"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s02-02-choose-the-risk"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s03-03-predict-the-checks"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s04-04-fill-the-return"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s05-05-fix-the-overtrust"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s06-06-write-count-ready"},{"id":"deep-learning-architectures/attention-and-transformer-blocks/s07-07-checkpoint"}]},{"slug":"decoding-kv-cache-quantization","steps":[{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s01-01-intro"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s02-02-choose-the-risk"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s03-03-predict-the-checks"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s04-04-fill-the-return"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s05-05-fix-the-overtrust"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s06-06-write-count-ready"},{"id":"deep-learning-architectures/decoding-kv-cache-quantization/s07-07-checkpoint"}]},{"slug":"mission-architecture-tradeoff-note","steps":[{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s01-01-intro"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s02-02-choose-the-risk"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s03-03-predict-the-checks"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s04-04-fill-the-return"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s05-05-fix-the-overtrust"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s06-06-write-count-ready"},{"id":"deep-learning-architectures/mission-architecture-tradeoff-note/s07-07-checkpoint"}]}]},{"slug":"feature-experiments-registries","lessons":[{"slug":"feature-pipeline-contracts","steps":[{"id":"feature-experiments-registries/feature-pipeline-contracts/s01-01-intro"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s02-02-choose-the-risk"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s03-03-predict-the-checks"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s04-04-fill-the-return"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s05-05-fix-the-overtrust"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s06-06-write-count-ready"},{"id":"feature-experiments-registries/feature-pipeline-contracts/s07-07-checkpoint"}]},{"slug":"train-inference-skew","steps":[{"id":"feature-experiments-registries/train-inference-skew/s01-01-intro"},{"id":"feature-experiments-registries/train-inference-skew/s02-02-choose-the-risk"},{"id":"feature-experiments-registries/train-inference-skew/s03-03-predict-the-checks"},{"id":"feature-experiments-registries/train-inference-skew/s04-04-fill-the-return"},{"id":"feature-experiments-registries/train-inference-skew/s05-05-fix-the-overtrust"},{"id":"feature-experiments-registries/train-inference-skew/s06-06-write-count-ready"},{"id":"feature-experiments-registries/train-inference-skew/s07-07-checkpoint"}]},{"slug":"experiment-tracker-lite","steps":[{"id":"feature-experiments-registries/experiment-tracker-lite/s01-01-intro"},{"id":"feature-experiments-registries/experiment-tracker-lite/s02-02-choose-the-risk"},{"id":"feature-experiments-registries/experiment-tracker-lite/s03-03-predict-the-checks"},{"id":"feature-experiments-registries/experiment-tracker-lite/s04-04-fill-the-return"},{"id":"feature-experiments-registries/experiment-tracker-lite/s05-05-fix-the-overtrust"},{"id":"feature-experiments-registries/experiment-tracker-lite/s06-06-write-count-ready"},{"id":"feature-experiments-registries/experiment-tracker-lite/s07-07-checkpoint"}]},{"slug":"model-registry-shape","steps":[{"id":"feature-experiments-registries/model-registry-shape/s01-01-intro"},{"id":"feature-experiments-registries/model-registry-shape/s02-02-choose-the-risk"},{"id":"feature-experiments-registries/model-registry-shape/s03-03-predict-the-checks"},{"id":"feature-experiments-registries/model-registry-shape/s04-04-fill-the-return"},{"id":"feature-experiments-registries/model-registry-shape/s05-05-fix-the-overtrust"},{"id":"feature-experiments-registries/model-registry-shape/s06-06-write-count-ready"},{"id":"feature-experiments-registries/model-registry-shape/s07-07-checkpoint"}]},{"slug":"mission-feature-pipeline-and-tracker","steps":[{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s01-01-intro"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s02-02-choose-the-risk"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s03-03-predict-the-checks"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s04-04-fill-the-return"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s05-05-fix-the-overtrust"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s06-06-write-count-ready"},{"id":"feature-experiments-registries/mission-feature-pipeline-and-tracker/s07-07-checkpoint"}]}]},{"slug":"model-serving-and-mlops","lessons":[{"slug":"fastapi-inference-shape","steps":[{"id":"model-serving-and-mlops/fastapi-inference-shape/s01-01-intro"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s06-06-write-count-ready"},{"id":"model-serving-and-mlops/fastapi-inference-shape/s07-07-checkpoint"}]},{"slug":"batch-vs-realtime-serving","steps":[{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s01-01-intro"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s06-06-write-count-ready"},{"id":"model-serving-and-mlops/batch-vs-realtime-serving/s07-07-checkpoint"}]},{"slug":"docker-runtime-and-config","steps":[{"id":"model-serving-and-mlops/docker-runtime-and-config/s01-01-intro"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s06-06-write-count-ready"},{"id":"model-serving-and-mlops/docker-runtime-and-config/s07-07-checkpoint"}]},{"slug":"github-actions-validation-gates","steps":[{"id":"model-serving-and-mlops/github-actions-validation-gates/s01-01-intro"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s06-06-write-failing-checks"},{"id":"model-serving-and-mlops/github-actions-validation-gates/s07-07-checkpoint"}]},{"slug":"continuous-training-and-rollout","steps":[{"id":"model-serving-and-mlops/continuous-training-and-rollout/s01-01-intro"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s06-06-write-count-ready"},{"id":"model-serving-and-mlops/continuous-training-and-rollout/s07-07-checkpoint"}]},{"slug":"mission-fastapi-model-server","steps":[{"id":"model-serving-and-mlops/mission-fastapi-model-server/s01-01-intro"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s02-02-choose-the-risk"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s03-03-predict-the-checks"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s04-04-fill-the-return"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s05-05-fix-the-overtrust"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s06-06-write-count-ready"},{"id":"model-serving-and-mlops/mission-fastapi-model-server/s07-07-checkpoint"}]}]},{"slug":"monitoring-cloud-portfolio","lessons":[{"slug":"structured-logs-and-alerts","steps":[{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s01-01-intro"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s06-06-write-count-ready"},{"id":"monitoring-cloud-portfolio/structured-logs-and-alerts/s07-07-checkpoint"}]},{"slug":"data-and-concept-drift","steps":[{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s01-01-intro"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s06-06-write-count-ready"},{"id":"monitoring-cloud-portfolio/data-and-concept-drift/s07-07-checkpoint"}]},{"slug":"retraining-triggers","steps":[{"id":"monitoring-cloud-portfolio/retraining-triggers/s01-01-intro"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s06-06-write-count-ready"},{"id":"monitoring-cloud-portfolio/retraining-triggers/s07-07-checkpoint"}]},{"slug":"cloud-gpu-kubernetes-cost","steps":[{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s01-01-intro"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s06-06-write-count-ready"},{"id":"monitoring-cloud-portfolio/cloud-gpu-kubernetes-cost/s07-07-checkpoint"}]},{"slug":"portfolio-architecture-docs","steps":[{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s01-01-intro"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s06-06-write-count-ready"},{"id":"monitoring-cloud-portfolio/portfolio-architecture-docs/s07-07-checkpoint"}]},{"slug":"mission-final-portfolio-ml-system","steps":[{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s01-01-intro"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s02-02-choose-the-risk"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s03-03-predict-the-checks"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s04-04-fill-the-return"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s05-05-fix-the-overtrust"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s06-06-write-decide"},{"id":"monitoring-cloud-portfolio/mission-final-portfolio-ml-system/s07-07-checkpoint"}]}]},{"slug":"graphic-design-studio","lessons":[{"slug":"prompting-like-an-art-director","steps":[{"id":"graphic-design-studio/prompting-like-an-art-director/s01-01-the-prompt-is-a-creative-brief"},{"id":"graphic-design-studio/prompting-like-an-art-director/s02-02-style-language"},{"id":"graphic-design-studio/prompting-like-an-art-director/s03-03-which-tool-for-the-headline"},{"id":"graphic-design-studio/prompting-like-an-art-director/s04-04-predict-the-scaffold"},{"id":"graphic-design-studio/prompting-like-an-art-director/s05-05-negative-space-and-the-iteration-loop"},{"id":"graphic-design-studio/prompting-like-an-art-director/s06-06-fix-the-mood-word-prompt"},{"id":"graphic-design-studio/prompting-like-an-art-director/s07-07-write-prompt-linter"},{"id":"graphic-design-studio/prompting-like-an-art-director/s08-08-checkpoint"}]},{"slug":"the-design-kit","steps":[{"id":"graphic-design-studio/the-design-kit/s01-01-the-brand-bible-as-data"},{"id":"graphic-design-studio/the-design-kit/s02-02-fill-the-palette-line"},{"id":"graphic-design-studio/the-design-kit/s03-03-banned-looks"},{"id":"graphic-design-studio/the-design-kit/s04-04-predict-the-negatives"},{"id":"graphic-design-studio/the-design-kit/s05-05-which-field-failed"},{"id":"graphic-design-studio/the-design-kit/s06-06-fix-the-crossed-kit"},{"id":"graphic-design-studio/the-design-kit/s07-07-write-contrast-check"},{"id":"graphic-design-studio/the-design-kit/s08-08-tokens-for-the-interface"},{"id":"graphic-design-studio/the-design-kit/s09-09-write-token-audit"},{"id":"graphic-design-studio/the-design-kit/s10-10-checkpoint"}]},{"slug":"fifty-assets-one-brand","steps":[{"id":"graphic-design-studio/fifty-assets-one-brand/s01-01-the-consistency-problem"},{"id":"graphic-design-studio/fifty-assets-one-brand/s02-02-seeds-the-same-dice-twice"},{"id":"graphic-design-studio/fifty-assets-one-brand/s03-03-why-does-the-mascot-drift"},{"id":"graphic-design-studio/fifty-assets-one-brand/s04-04-predict-the-batch-plan"},{"id":"graphic-design-studio/fifty-assets-one-brand/s05-05-the-qa-filter"},{"id":"graphic-design-studio/fifty-assets-one-brand/s06-06-defend-the-pick"},{"id":"graphic-design-studio/fifty-assets-one-brand/s07-07-fix-the-qa-threshold"},{"id":"graphic-design-studio/fifty-assets-one-brand/s08-08-write-palette-distance"},{"id":"graphic-design-studio/fifty-assets-one-brand/s09-09-checkpoint"}]}]},{"slug":"campaign-studio","lessons":[{"slug":"voice-pack-as-data","steps":[{"id":"campaign-studio/voice-pack-as-data/s01-01-the-campaign-and-the-gate"},{"id":"campaign-studio/voice-pack-as-data/s02-02-lint-the-draft"},{"id":"campaign-studio/voice-pack-as-data/s03-03-predict-the-violations"},{"id":"campaign-studio/voice-pack-as-data/s04-04-testable-or-mood"},{"id":"campaign-studio/voice-pack-as-data/s05-05-fill-the-linter"},{"id":"campaign-studio/voice-pack-as-data/s06-06-variants-as-records"},{"id":"campaign-studio/voice-pack-as-data/s07-07-write-lint-copy"},{"id":"campaign-studio/voice-pack-as-data/s08-08-checkpoint"}]},{"slug":"batch-plan-and-calendar","steps":[{"id":"campaign-studio/batch-plan-and-calendar/s01-01-slots-before-prompts"},{"id":"campaign-studio/batch-plan-and-calendar/s02-02-which-field-is-missing"},{"id":"campaign-studio/batch-plan-and-calendar/s03-03-write-batch-plan"},{"id":"campaign-studio/batch-plan-and-calendar/s04-04-the-calendar-is-data"},{"id":"campaign-studio/batch-plan-and-calendar/s05-05-predict-the-queue"},{"id":"campaign-studio/batch-plan-and-calendar/s06-06-fix-the-gate"},{"id":"campaign-studio/batch-plan-and-calendar/s07-07-checkpoint"}]},{"slug":"the-numbers-pass","steps":[{"id":"campaign-studio/the-numbers-pass/s01-01-a-report-is-a-draft-over-data"},{"id":"campaign-studio/the-numbers-pass/s02-02-predict-the-verdict"},{"id":"campaign-studio/the-numbers-pass/s03-03-write-verify-numbers"},{"id":"campaign-studio/the-numbers-pass/s04-04-denominators-and-the-ftc"},{"id":"campaign-studio/the-numbers-pass/s05-05-what-may-not-ship"},{"id":"campaign-studio/the-numbers-pass/s06-06-fill-the-policy-router"},{"id":"campaign-studio/the-numbers-pass/s07-07-checkpoint"}]}]},{"slug":"support-studio","lessons":[{"slug":"the-policy-pack","steps":[{"id":"support-studio/the-policy-pack/s01-01-the-reply-that-binds-the-company"},{"id":"support-studio/the-policy-pack/s02-02-the-pack-as-data"},{"id":"support-studio/the-policy-pack/s03-03-spot-the-banned-promise"},{"id":"support-studio/the-policy-pack/s04-04-predict-the-lint"},{"id":"support-studio/the-policy-pack/s05-05-fix-the-stale-citation"},{"id":"support-studio/the-policy-pack/s06-06-the-review-loop-evidence"},{"id":"support-studio/the-policy-pack/s07-07-write-lint-draft"},{"id":"support-studio/the-policy-pack/s08-08-checkpoint"}]},{"slug":"golden-evals-earn-autonomy","steps":[{"id":"support-studio/golden-evals-earn-autonomy/s01-01-the-klarna-arc"},{"id":"support-studio/golden-evals-earn-autonomy/s02-02-a-macro-is-three-things"},{"id":"support-studio/golden-evals-earn-autonomy/s03-03-inside-the-lane"},{"id":"support-studio/golden-evals-earn-autonomy/s04-04-predict-the-eval-run"},{"id":"support-studio/golden-evals-earn-autonomy/s05-05-fill-the-promotion-rule"},{"id":"support-studio/golden-evals-earn-autonomy/s06-06-write-run-golden-evals"},{"id":"support-studio/golden-evals-earn-autonomy/s07-07-checkpoint"}]},{"slug":"the-escalation-boundary","steps":[{"id":"support-studio/the-escalation-boundary/s01-01-drafting-is-not-spending"},{"id":"support-studio/the-escalation-boundary/s02-02-triggers-as-conditions"},{"id":"support-studio/the-escalation-boundary/s03-03-route-this-ticket"},{"id":"support-studio/the-escalation-boundary/s04-04-predict-the-routing"},{"id":"support-studio/the-escalation-boundary/s05-05-fix-the-refund-hole"},{"id":"support-studio/the-escalation-boundary/s06-06-disclosure-and-the-kill-switch"},{"id":"support-studio/the-escalation-boundary/s07-07-write-route-ticket"},{"id":"support-studio/the-escalation-boundary/s08-08-checkpoint"}]}]},{"slug":"copy-studio","lessons":[{"slug":"the-voice-fingerprint","steps":[{"id":"copy-studio/the-voice-fingerprint/s01-01-the-client-you-inherit"},{"id":"copy-studio/the-voice-fingerprint/s02-02-the-fingerprint"},{"id":"copy-studio/the-voice-fingerprint/s03-03-predict-the-ceiling"},{"id":"copy-studio/the-voice-fingerprint/s04-04-the-samples-say-otherwise"},{"id":"copy-studio/the-voice-fingerprint/s05-05-fill-derive-the-rules"},{"id":"copy-studio/the-voice-fingerprint/s06-06-the-pack-with-provenance"},{"id":"copy-studio/the-voice-fingerprint/s07-07-write-fingerprint"},{"id":"copy-studio/the-voice-fingerprint/s08-08-checkpoint"}]},{"slug":"variants-that-hold-the-line","steps":[{"id":"copy-studio/variants-that-hold-the-line/s01-01-twelve-by-thursday"},{"id":"copy-studio/variants-that-hold-the-line/s02-02-the-drift-problem"},{"id":"copy-studio/variants-that-hold-the-line/s03-03-predict-the-drift-report"},{"id":"copy-studio/variants-that-hold-the-line/s04-04-which-subject-line-ships"},{"id":"copy-studio/variants-that-hold-the-line/s05-05-fix-the-channel-check"},{"id":"copy-studio/variants-that-hold-the-line/s06-06-the-kill-log"},{"id":"copy-studio/variants-that-hold-the-line/s07-07-write-voice-gate"},{"id":"copy-studio/variants-that-hold-the-line/s08-08-checkpoint"}]},{"slug":"the-claim-check","steps":[{"id":"copy-studio/the-claim-check/s01-01-forty-one-of-seventy-seven"},{"id":"copy-studio/the-claim-check/s02-02-what-counts-as-checkable"},{"id":"copy-studio/the-claim-check/s03-03-predict-the-claim-list"},{"id":"copy-studio/the-claim-check/s04-04-what-closes-a-row"},{"id":"copy-studio/the-claim-check/s05-05-fill-the-ship-decision"},{"id":"copy-studio/the-claim-check/s06-06-the-corrections-log"},{"id":"copy-studio/the-claim-check/s07-07-write-ship-gate"},{"id":"copy-studio/the-claim-check/s08-08-checkpoint"}]}]},{"slug":"analytics-studio","lessons":[{"slug":"the-cleaning-log","steps":[{"id":"analytics-studio/the-cleaning-log/s01-01-where-did-those-rows-go"},{"id":"analytics-studio/the-cleaning-log/s02-02-the-log-as-data"},{"id":"analytics-studio/the-cleaning-log/s03-03-refuse-the-silent-step"},{"id":"analytics-studio/the-cleaning-log/s04-04-predict-the-receipts"},{"id":"analytics-studio/the-cleaning-log/s05-05-fill-the-null-decision"},{"id":"analytics-studio/the-cleaning-log/s06-06-fix-the-vanishing-rows"},{"id":"analytics-studio/the-cleaning-log/s07-07-write-the-null-ledger"},{"id":"analytics-studio/the-cleaning-log/s08-08-checkpoint"}]},{"slug":"every-number-has-an-address","steps":[{"id":"analytics-studio/every-number-has-an-address/s01-01-the-plausible-wrong-number"},{"id":"analytics-studio/every-number-has-an-address/s02-02-sample-first-then-full"},{"id":"analytics-studio/every-number-has-an-address/s03-03-which-check-catches-it"},{"id":"analytics-studio/every-number-has-an-address/s04-04-predict-the-fan-out"},{"id":"analytics-studio/every-number-has-an-address/s05-05-fill-the-trace"},{"id":"analytics-studio/every-number-has-an-address/s06-06-write-audit-the-memo"},{"id":"analytics-studio/every-number-has-an-address/s07-07-checkpoint"}]},{"slug":"challenge-then-rerun","steps":[{"id":"analytics-studio/challenge-then-rerun/s01-01-the-confident-recommendation"},{"id":"analytics-studio/challenge-then-rerun/s02-02-the-triad-as-a-checklist"},{"id":"analytics-studio/challenge-then-rerun/s03-03-name-the-leg"},{"id":"analytics-studio/challenge-then-rerun/s04-04-predict-the-rerun"},{"id":"analytics-studio/challenge-then-rerun/s05-05-fix-the-drifting-memo"},{"id":"analytics-studio/challenge-then-rerun/s06-06-write-score-the-challenge"},{"id":"analytics-studio/challenge-then-rerun/s07-07-checkpoint"}]}]},{"slug":"delivery-studio","lessons":[{"slug":"the-spec-is-the-build","steps":[{"id":"delivery-studio/the-spec-is-the-build/s01-01-the-clarifying-question-is-gone"},{"id":"delivery-studio/the-spec-is-the-build/s02-02-the-ticket-as-data"},{"id":"delivery-studio/the-spec-is-the-build/s03-03-spot-the-untestable-check"},{"id":"delivery-studio/the-spec-is-the-build/s04-04-predict-the-spec-lint"},{"id":"delivery-studio/the-spec-is-the-build/s05-05-fill-the-scope-fence"},{"id":"delivery-studio/the-spec-is-the-build/s06-06-inside-or-outside-the-frontier"},{"id":"delivery-studio/the-spec-is-the-build/s07-07-write-lint-spec"},{"id":"delivery-studio/the-spec-is-the-build/s08-08-checkpoint"}]},{"slug":"acceptance-before-the-build","steps":[{"id":"delivery-studio/acceptance-before-the-build/s01-01-the-ninety-seven-thousand-dollar-skip"},{"id":"delivery-studio/acceptance-before-the-build/s02-02-evidence-against-the-checks"},{"id":"delivery-studio/acceptance-before-the-build/s03-03-which-check-catches-it"},{"id":"delivery-studio/acceptance-before-the-build/s04-04-fix-the-vibes-review"},{"id":"delivery-studio/acceptance-before-the-build/s05-05-predict-the-review-run"},{"id":"delivery-studio/acceptance-before-the-build/s06-06-rework-is-a-list-not-a-feeling"},{"id":"delivery-studio/acceptance-before-the-build/s07-07-write-review-deliverable"},{"id":"delivery-studio/acceptance-before-the-build/s08-08-checkpoint"}]},{"slug":"status-from-artifacts","steps":[{"id":"delivery-studio/status-from-artifacts/s01-01-watermelon-status"},{"id":"delivery-studio/status-from-artifacts/s02-02-derive-the-rag"},{"id":"delivery-studio/status-from-artifacts/s03-03-spot-the-artifact"},{"id":"delivery-studio/status-from-artifacts/s04-04-fill-the-worst-of"},{"id":"delivery-studio/status-from-artifacts/s05-05-predict-the-rollup"},{"id":"delivery-studio/status-from-artifacts/s06-06-the-not-ready-call"},{"id":"delivery-studio/status-from-artifacts/s07-07-write-readiness-gate"},{"id":"delivery-studio/status-from-artifacts/s08-08-checkpoint"}]},{"slug":"the-hours-saved-receipt","steps":[{"id":"delivery-studio/the-hours-saved-receipt/s01-01-the-token-chart-is-not-a-receipt"},{"id":"delivery-studio/the-hours-saved-receipt/s02-02-the-receipt-as-data"},{"id":"delivery-studio/the-hours-saved-receipt/s03-03-which-line-survives-staff-meeting"},{"id":"delivery-studio/the-hours-saved-receipt/s04-04-fix-the-vanity-chart"},{"id":"delivery-studio/the-hours-saved-receipt/s05-05-predict-the-receipt-run"},{"id":"delivery-studio/the-hours-saved-receipt/s06-06-only-accepted-ships-count"},{"id":"delivery-studio/the-hours-saved-receipt/s07-07-write-hours-receipt"},{"id":"delivery-studio/the-hours-saved-receipt/s08-08-checkpoint"}]}]},{"slug":"people-studio","lessons":[{"slug":"the-data-firewall","steps":[{"id":"people-studio/the-data-firewall/s01-01-the-rule-that-cannot-live-in-memory"},{"id":"people-studio/the-data-firewall/s02-02-the-firewall-as-data"},{"id":"people-studio/the-data-firewall/s03-03-which-payload-passes"},{"id":"people-studio/the-data-firewall/s04-04-predict-the-scan"},{"id":"people-studio/the-data-firewall/s05-05-fix-the-leaky-gate"},{"id":"people-studio/the-data-firewall/s06-06-employee-relations-takes-nothing"},{"id":"people-studio/the-data-firewall/s07-07-write-prompt-gate"},{"id":"people-studio/the-data-firewall/s08-08-checkpoint"}]},{"slug":"briefs-with-receipts","steps":[{"id":"people-studio/briefs-with-receipts/s01-01-the-most-automated-desk"},{"id":"people-studio/briefs-with-receipts/s02-02-a-claim-with-a-receipt"},{"id":"people-studio/briefs-with-receipts/s03-03-sourced-or-inference"},{"id":"people-studio/briefs-with-receipts/s04-04-predict-the-receipt-audit"},{"id":"people-studio/briefs-with-receipts/s05-05-fill-the-lookup"},{"id":"people-studio/briefs-with-receipts/s06-06-the-screening-line"},{"id":"people-studio/briefs-with-receipts/s07-07-write-audit-brief"},{"id":"people-studio/briefs-with-receipts/s08-08-checkpoint"}]},{"slug":"the-paper-trail","steps":[{"id":"people-studio/the-paper-trail/s01-01-policy-is-legal-surface"},{"id":"people-studio/the-paper-trail/s02-02-the-boundary-as-a-state-machine"},{"id":"people-studio/the-paper-trail/s03-03-who-signs-off"},{"id":"people-studio/the-paper-trail/s04-04-predict-the-version-log"},{"id":"people-studio/the-paper-trail/s05-05-fix-the-shortcut"},{"id":"people-studio/the-paper-trail/s06-06-the-evidence-trail"},{"id":"people-studio/the-paper-trail/s07-07-write-decision-record"},{"id":"people-studio/the-paper-trail/s08-08-checkpoint"}]}]},{"slug":"ops-studio","lessons":[{"slug":"the-sop-is-data","steps":[{"id":"ops-studio/the-sop-is-data/s01-01-the-two-weeks-notice-problem"},{"id":"ops-studio/the-sop-is-data/s02-02-the-sop-as-data"},{"id":"ops-studio/the-sop-is-data/s03-03-spot-the-invented-step"},{"id":"ops-studio/the-sop-is-data/s04-04-fill-the-sop-lint"},{"id":"ops-studio/the-sop-is-data/s05-05-predict-the-diff"},{"id":"ops-studio/the-sop-is-data/s06-06-the-diff-is-the-change-control"},{"id":"ops-studio/the-sop-is-data/s07-07-write-lint-sop"},{"id":"ops-studio/the-sop-is-data/s08-08-checkpoint"}]},{"slug":"auto-run-or-named-approval","steps":[{"id":"ops-studio/auto-run-or-named-approval/s01-01-the-checklist-that-lied"},{"id":"ops-studio/auto-run-or-named-approval/s02-02-the-schedule-is-data"},{"id":"ops-studio/auto-run-or-named-approval/s03-03-which-item-may-auto-run"},{"id":"ops-studio/auto-run-or-named-approval/s04-04-fix-the-silent-auto-complete"},{"id":"ops-studio/auto-run-or-named-approval/s05-05-predict-the-exception-queue"},{"id":"ops-studio/auto-run-or-named-approval/s06-06-exceptions-are-a-queue-not-a-shrug"},{"id":"ops-studio/auto-run-or-named-approval/s07-07-write-run-checklist"},{"id":"ops-studio/auto-run-or-named-approval/s08-08-checkpoint"}]},{"slug":"numbers-with-receipts","steps":[{"id":"ops-studio/numbers-with-receipts/s01-01-the-number-that-moves-money"},{"id":"ops-studio/numbers-with-receipts/s02-02-recompute-dont-trust"},{"id":"ops-studio/numbers-with-receipts/s03-03-which-claim-has-no-receipt"},{"id":"ops-studio/numbers-with-receipts/s04-04-predict-the-verification"},{"id":"ops-studio/numbers-with-receipts/s05-05-fill-the-receipt-check"},{"id":"ops-studio/numbers-with-receipts/s06-06-the-anomaly-is-a-question"},{"id":"ops-studio/numbers-with-receipts/s07-07-write-verify-report"},{"id":"ops-studio/numbers-with-receipts/s08-08-checkpoint"}]},{"slug":"the-pii-proxy","steps":[{"id":"ops-studio/the-pii-proxy/s01-01-the-ticket-that-left-the-building"},{"id":"ops-studio/the-pii-proxy/s02-02-mask-then-unmask"},{"id":"ops-studio/the-pii-proxy/s03-03-which-payload-may-leave"},{"id":"ops-studio/the-pii-proxy/s04-04-fill-the-leftover-check"},{"id":"ops-studio/the-pii-proxy/s05-05-predict-the-two-sides"},{"id":"ops-studio/the-pii-proxy/s06-06-unmask-is-a-named-human-act"},{"id":"ops-studio/the-pii-proxy/s07-07-write-the-proxy"},{"id":"ops-studio/the-pii-proxy/s08-08-checkpoint"}]}]},{"slug":"legal-studio","lessons":[{"slug":"the-cite-check-gate","steps":[{"id":"legal-studio/the-cite-check-gate/s01-01-the-brief-that-reads-perfectly"},{"id":"legal-studio/the-cite-check-gate/s02-02-three-verdicts-as-data"},{"id":"legal-studio/the-cite-check-gate/s03-03-which-cite-needs-no-check"},{"id":"legal-studio/the-cite-check-gate/s04-04-predict-the-verification-run"},{"id":"legal-studio/the-cite-check-gate/s05-05-fix-the-verifier-that-asks-again"},{"id":"legal-studio/the-cite-check-gate/s06-06-write-verify-authorities"},{"id":"legal-studio/the-cite-check-gate/s07-07-checkpoint"}]},{"slug":"clause-extraction-with-pinpoints","steps":[{"id":"legal-studio/clause-extraction-with-pinpoints/s01-01-a-claim-without-a-pinpoint-is-a-rumor"},{"id":"legal-studio/clause-extraction-with-pinpoints/s02-02-the-playbook-as-data"},{"id":"legal-studio/clause-extraction-with-pinpoints/s03-03-which-row-is-unusable"},{"id":"legal-studio/clause-extraction-with-pinpoints/s04-04-predict-the-triage"},{"id":"legal-studio/clause-extraction-with-pinpoints/s05-05-fill-the-walk-away-route"},{"id":"legal-studio/clause-extraction-with-pinpoints/s06-06-write-triage-extraction"},{"id":"legal-studio/clause-extraction-with-pinpoints/s07-07-checkpoint"}]},{"slug":"privilege-rules-and-the-review-log","steps":[{"id":"legal-studio/privilege-rules-and-the-review-log/s01-01-privilege-is-a-routing-question"},{"id":"legal-studio/privilege-rules-and-the-review-log/s02-02-the-routing-table"},{"id":"legal-studio/privilege-rules-and-the-review-log/s03-03-which-request-is-the-waiver-risk"},{"id":"legal-studio/privilege-rules-and-the-review-log/s04-04-predict-the-router"},{"id":"legal-studio/privilege-rules-and-the-review-log/s05-05-fix-the-privilege-hole"},{"id":"legal-studio/privilege-rules-and-the-review-log/s06-06-the-log-that-survives-a-motion"},{"id":"legal-studio/privilege-rules-and-the-review-log/s07-07-write-log-ready"},{"id":"legal-studio/privilege-rules-and-the-review-log/s08-08-checkpoint"}]}]},{"slug":"agent-studio","lessons":[{"slug":"the-tool-contract","steps":[{"id":"agent-studio/the-tool-contract/s01-01-the-feature-you-ship-today"},{"id":"agent-studio/the-tool-contract/s02-02-the-schema-is-the-blast-radius"},{"id":"agent-studio/the-tool-contract/s03-03-spot-the-loose-tool"},{"id":"agent-studio/the-tool-contract/s04-04-predict-the-validation"},{"id":"agent-studio/the-tool-contract/s05-05-fix-the-yolo-rerun"},{"id":"agent-studio/the-tool-contract/s06-06-write-validate-call"},{"id":"agent-studio/the-tool-contract/s07-07-checkpoint"}]},{"slug":"the-loop-with-a-meter","steps":[{"id":"agent-studio/the-loop-with-a-meter/s01-01-a-loop-that-cannot-stop-is-an-invoice"},{"id":"agent-studio/the-loop-with-a-meter/s02-02-caps-are-part-of-the-loop"},{"id":"agent-studio/the-loop-with-a-meter/s03-03-why-did-it-stop"},{"id":"agent-studio/the-loop-with-a-meter/s04-04-predict-the-stop-reason"},{"id":"agent-studio/the-loop-with-a-meter/s05-05-fill-the-budget-check"},{"id":"agent-studio/the-loop-with-a-meter/s06-06-the-trace-is-the-receipt"},{"id":"agent-studio/the-loop-with-a-meter/s07-07-write-run-loop"},{"id":"agent-studio/the-loop-with-a-meter/s08-08-checkpoint"}]},{"slug":"evals-before-it-works","steps":[{"id":"agent-studio/evals-before-it-works/s01-01-green-before-it-runs"},{"id":"agent-studio/evals-before-it-works/s02-02-a-golden-case-is-a-past-incident"},{"id":"agent-studio/evals-before-it-works/s03-03-which-case-catches-it"},{"id":"agent-studio/evals-before-it-works/s04-04-predict-the-eval-run"},{"id":"agent-studio/evals-before-it-works/s05-05-fix-the-router-to-green"},{"id":"agent-studio/evals-before-it-works/s06-06-the-ratchet-entry"},{"id":"agent-studio/evals-before-it-works/s07-07-write-lint-the-ratchet"},{"id":"agent-studio/evals-before-it-works/s08-08-checkpoint"}]}]}]},"detail":{"number":22,"slug":"context-and-retrieval","title":"context and retrieval — feeding the model real data","blurb":"rag without the overengineering. chunking, embeddings, vector search, and the small set of patterns that make a model answer from your data instead of its training set.","overview":"$27","status":"live","lessons":[{"slug":"chunking-that-respects-structure","title":"Chunking that respects structure — don't shred your own documents","estMinutes":12,"prerequisites":["orientation"],"status":"live","steps":[{"type":"read","id":"context-and-retrieval/chunking-that-respects-structure/s01-01-intro","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":120,"concept":"structure-aware-chunking","body":"$28","cta":"Got it","code":"# naive: cut every N characters. fast, simple, destroys structure.\ndef chunk_by_chars(text, size=80):\n return [text[i:i + size] for i in range(0, len(text), size)]\n\ndoc = \"\"\"# Setup\n\nInstall the package with pip.\n\n```python\nimport requests\nresponse = requests.get(url)\n```\n\nThen read the response.\"\"\"\n\nfor i, chunk in enumerate(chunk_by_chars(doc, 80), start=1):\n print(f\"--- chunk {i} ---\")\n print(chunk)\n","runnable":true},{"type":"mc","id":"context-and-retrieval/chunking-that-respects-structure/s02-02-which-content-shape-breaks","xp":2,"hint":[{"level":1,"body":"Structured content (markdown code blocks, JSON braces) loses meaning when sliced mid-structure. Free-form prose tolerates random cuts much better — every word boundary is roughly equivalent.","cost":0}],"personalize":false,"phase":"warmup","estSeconds":55,"concept":"chunk-shapes-most-vulnerable","prompt":"Naive char-count chunking cuts every N characters regardless of\nstructure. Which TWO shapes ABOVE are most likely to be SHREDDED —\nmeaning a single chunk loses semantic integrity (a code block torn\nin half, a JSON object's brace orphaned)?\n","options":[{"id":"a_and_c","label":"A (markdown with code blocks) and C (JSON-like config)"},{"id":"b_and_d","label":"B (plain prose) and D (short FAQ)","explain":"Plain prose recovers gracefully — adjacent chunks hold the same surrounding sentences. Short FAQs fit in one chunk so chunking barely runs."},{"id":"a_and_d","label":"A and D","explain":"D rarely exceeds chunk size. The FAQ either fits whole or fits per-Q&A."},{"id":"b_and_c","label":"B and C","explain":"B's prose handles char-cuts surprisingly well. C is the bigger problem, but the prose isn't."}],"answerIds":["a_and_c"],"shuffle":false,"code":"# four content shapes. which break worst under naive char-count chunking?\nshapes = {\n \"A\": \"A markdown doc with code blocks, headings, and bullet lists.\",\n \"B\": \"Plain prose articles with no special syntax (e.g., a novel).\",\n \"C\": \"JSON-like config dumps with nested braces and indentation.\",\n \"D\": \"A 5-line FAQ where each question and answer fit in one paragraph.\",\n}\nfor k, v in shapes.items():\n print(f\"{k}: {v}\")\n","runnable":true},{"type":"read","id":"context-and-retrieval/chunking-that-respects-structure/s03-03-the-recursive-splitter","xp":2,"hint":[],"personalize":false,"phase":"build","estSeconds":100,"concept":"recursive-separator-fallback","body":"$29","cta":"Got it","code":"$2a","runnable":true},{"type":"predict","id":"context-and-retrieval/chunking-that-respects-structure/s04-04-predict-the-shred","xp":3,"hint":[{"level":1,"body":"The doc is 57 chars. Cut every 20 chars → 3 chunks. The first chunk is the first 20 chars (`\"# Title\\n\\nFirst parag\"`) — the cut lands mid-word because the splitter ignores structure.","cost":0},{"level":2,"body":"Three chunks. Each cuts mid-word at a hard 20-character boundary. This is exactly the bug structure-aware chunking fixes.","cost":0}],"personalize":false,"phase":"build","estSeconds":80,"concept":"naive-vs-structured-chunk-count","code":"# naive char-count chunking on a small markdown doc.\ndoc = \"# Title\\n\\nFirst paragraph.\\n\\nSecond paragraph here.\\n\\nThird.\"\n\ndef chunk_by_chars(text, size):\n return [text[i:i + size] for i in range(0, len(text), size)]\n\nchunks = chunk_by_chars(doc, 20)\nprint(f\"chunk count: {len(chunks)}\")\nfor c in chunks:\n print(repr(c))\n","prompt":"Read the editor. The doc is 60 characters total. Splitting by 20\ncharacters produces how many chunks, and what does the first\nchunk look like?\n\nWhat's the chunk count printed on line 1?\n","grader":{"kind":"stdout-equality","expected":"chunk count: 3\n'# Title\\n\\nFirst parag'\n'raph.\\n\\nSecond paragr'\n'aph here.\\n\\nThird.'\n","normalize":"collapse-trailing-newline"}},{"type":"fill","id":"context-and-retrieval/chunking-that-respects-structure/s05-05-fill-the-separator-list","xp":3,"hint":[{"level":1,"body":"Four entries, in priority order: paragraph (`\"\\n\\n\"`), sentence (`\". \"`), word (`\" \"`), char (`\"\"`). The empty string at the end is the safety net.","cost":0},{"level":2,"body":"`[\"\\n\\n\", \". \", \" \", \"\"]` — paragraph, sentence, word, char. Most-semantic first, hard-cut last.","cost":0}],"personalize":false,"phase":"build","estSeconds":60,"concept":"separator-priority-list","prompt":"The recursive splitter takes a list of separators in PRIORITY\norder — try the most semantic one first, fall back through less\nsemantic ones, end with a hard char cut. Fill in the list for\ngeneral English text: paragraph break, then sentence end, then\nword, then character.\n\nExpected output:\n```\n['\\n\\n', '. ', ' ', '']\n```\n","code":"# the standard English separator priority.\nseparators = ___\nprint(separators)\n","blanks":[{"id":"separator_list","accept":["[\"\\n\\n\", \". \", \" \", \"\"]","['\\n\\n', '. ', ' ', '']"],"caseSensitive":true,"normalize":"trim"}]},{"type":"fix","id":"context-and-retrieval/chunking-that-respects-structure/s06-06-fix-the-mid-sentence-cut","xp":4,"hint":[{"level":1,"body":"Replace the hard cut with `text.split(\". \")` — but that drops the period from each piece. Re-attach the period to all parts except the last with a list comprehension.","cost":0},{"level":2,"body":"```python\nparts = text.split(\". \")\nchunks = [p + \".\" for p in parts[:-1]] + [parts[-1]]\n```\n","cost":0}],"personalize":false,"phase":"build","estSeconds":95,"concept":"split-on-sentence-not-char","brokenCode":"text = \"Acme was founded in 1958. The company makes running shoes.\"\n\n# bug: hard character cut at position 30 — lands mid-sentence.\ncut_at = 30\nchunks = [text[:cut_at].strip(), text[cut_at:].strip()]\n\nprint(chunks)\n","prompt":"The naive splitter is cutting mid-sentence. Two sentences fit in\none budget but the cut lands in the middle of the second. The user\nlater asks \"what does the second sentence say?\" — retrieval gets\nhalf of it.\n\nFix the split to break on `\". \"` (sentence boundary) instead of a\nhard character cut. The two resulting chunks should each contain\nexactly one complete sentence.\n\nExpected output:\n```\n['Acme was founded in 1958.', 'The company makes running shoes.']\n```\n","grader":{"kind":"stdout-equality","expected":"['Acme was founded in 1958.', 'The company makes running shoes.']","normalize":"collapse-trailing-newline"},"bugLines":[4,5],"revealAfter":5},{"type":"fix","id":"context-and-retrieval/chunking-that-respects-structure/s07-07-fix-the-missing-overlap","xp":4,"hint":[{"level":1,"body":"Take the last `overlap` characters of `chunks[0]` and prepend them to `chunks[1]`. Leave `chunks[0]` itself unchanged.","cost":0},{"level":2,"body":"```python\nresult = [chunks[0], chunks[0][-overlap:] + \" \" + chunks[1]]\n```\n\nNote the space between the overlap and chunk[1] body — a literal space joins the carry-over to the new content.\n","cost":0}],"personalize":false,"phase":"build","estSeconds":105,"concept":"chunk-overlap-recall","brokenCode":"chunks = [\n \"Acme was founded in 1958.\",\n \"The company makes running shoes.\",\n]\n\noverlap = 10\n# bug: no overlap. each chunk is isolated; cross-chunk facts get lost.\nresult = chunks\n\nprint(result)\n","prompt":"Sentences are split cleanly into chunks, but the chunks have NO\noverlap. The user asks \"Acme makes WHAT — and when was it\nfounded?\" — that fact spans both chunks. Retrieval might pull\nonly chunk 0 (\"founded in 1958\") OR only chunk 1 (\"makes running\nshoes\"), missing half the answer.\n\nFix the chunks to add 10 characters of overlap from the END of\nchunk 0 INTO chunk 1, so retrievers that match either chunk see\nboth facts.\n\nExpected output:\n```\n['Acme was founded in 1958.', 'd in 1958. The company makes running shoes.']\n```\n","grader":{"kind":"stdout-equality","expected":"['Acme was founded in 1958.', 'd in 1958. The company makes running shoes.']","normalize":"collapse-trailing-newline"},"bugLines":[7],"revealAfter":4},{"type":"write","id":"context-and-retrieval/chunking-that-respects-structure/s08-08-write-the-recursive-splitter","xp":5,"hint":[{"level":1,"body":"Two halves: (1) base case + separator-loop with greedy accumulation, (2) recursion on chunks that are still too big with the next separator. Use `enumerate(separators)` to track which one you're on.","cost":0},{"level":2,"body":"Key trick: greedy accumulation. Start with `current = \"\"`. For each part, build `candidate = current + sep + part`. If candidate fits, current = candidate. If not, push current to chunks and start a new current = part.","cost":0}],"personalize":false,"phase":"build","estSeconds":200,"concept":"recursive-text-splitter","prompt":"Write `recursive_split(text, max_chars, separators)` that returns\na list of chunks, each ≤ `max_chars`. Logic:\n\n- **Base case**: if `len(text) <= max_chars`, return `[text]`.\n- Otherwise iterate `separators` in order. For the first separator\n that exists in `text`:\n - Split `text` on that separator into `parts`.\n - Reassemble parts greedily: accumulate parts into `current`\n (joined with the separator); when adding the next part would\n exceed `max_chars`, push `current` into `chunks` and start a\n new accumulator.\n - For any chunk still bigger than `max_chars`, recurse with\n `separators` advanced past the current one.\n - Return the result.\n- If no separator fits, fall through and hard-cut every\n `max_chars` characters.\n\nThe separators are `[\"\\n\\n\", \". \", \" \", \"\"]`. Use them in order.\n\nTwo cases run for you. Expected output:\n```\nparagraph case: 3 chunks\nlong-sentence case: 3 chunks\n```\n","starter":"# define recursive_split(text, max_chars, separators) below\n\ndoc1 = \"Para one short.\\n\\nPara two also short.\\n\\nPara three is here.\"\nresult1 = recursive_split(doc1, 25, [\"\\n\\n\", \". \", \" \", \"\"])\nprint(f\"paragraph case: {len(result1)} chunks\")\n\ndoc2 = \"Sentence one. Sentence two is much longer than the others.\"\nresult2 = recursive_split(doc2, 30, [\"\\n\\n\", \". \", \" \", \"\"])\nprint(f\"long-sentence case: {len(result2)} chunks\")\n","grader":{"kind":"stdout-equality","expected":"paragraph case: 3 chunks\nlong-sentence case: 3 chunks\n","normalize":"collapse-trailing-newline"},"solution":"$2b","hiddenTests":[]},{"type":"checkpoint","id":"context-and-retrieval/chunking-that-respects-structure/s09-09-checkpoint","xp":8,"hint":[],"personalize":false,"phase":"check","estSeconds":220,"concept":"chunker-with-overlap","prompt":"Final drill. Build a structure-aware chunker WITH overlap. Write\n`chunk_document(text, max_chars, overlap, separators)` that:\n\n- First runs the recursive split (algorithm from step 8) to get\n chunks ≤ `max_chars`.\n- Then adds overlap: for each chunk after the first, prepend the\n LAST `overlap` characters of the previous chunk.\n- Returns the overlap-augmented chunk list.\n\nThree cases run for you. Expected output:\n```\n3 chunks, lengths: [22, 30, 27]\nfirst chunk: 'First short paragraph.'\nsecond chunk starts with: 'ragraph.Second paragrap'\n```\n","starter":"$2c","grader":{"kind":"stdout-equality","expected":"3 chunks, lengths: [22, 30, 27]\nfirst chunk: 'First short paragraph.'\nsecond chunk starts with: 'ragraph.Second paragrap'\n","normalize":"collapse-trailing-newline"},"solution":"$2d","passThreshold":0.66}],"xpTotal":32},{"slug":"embedding-that-fits-the-budget","title":"Embedding that fits the budget — pick a model that matches your corpus","estMinutes":12,"prerequisites":["chunking-that-respects-structure"],"status":"live","steps":[{"type":"read","id":"context-and-retrieval/embedding-that-fits-the-budget/s01-01-intro","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":130,"concept":"embedding-cost-vs-quality","body":"$2e","cta":"Got it","code":"# an embedding is a list of numbers that captures meaning. similar\n# text → similar numbers. we'll fake the real API call here — pyodide\n# can't reach OpenAI, but the SHAPE is identical to the real thing.\nimport hashlib\n\ndef embed(text):\n # fake 8-dim embedding. deterministic (sha256, not the built-in\n # hash() which is randomized per run). real call returns 1536 or 3072 floats.\n return [int(hashlib.sha256((text + str(i)).encode()).hexdigest(), 16) % 1000 / 1000 for i in range(8)]\n\nfor phrase in [\"cancel my subscription\", \"how do I unsubscribe\", \"today's weather\"]:\n vec = embed(phrase)\n preview = [round(x, 3) for x in vec[:4]]\n print(f\"{phrase!r:35} -> {preview}... (8 dims)\")\n","runnable":true},{"type":"mc","id":"context-and-retrieval/embedding-that-fits-the-budget/s02-02-which-model-fits","xp":2,"hint":[{"level":1,"body":"Default to `text-embedding-3-small` unless you have a specific reason not to. The two reasons to upgrade: (1) specialized domain where accuracy is critical and budget allows (legal, medical, code), or (2) eval data showing small fails on your corpus. Tiny corpora and massive ones both default to small for different reasons (cost doesn't matter / cost matters a lot).","cost":0}],"personalize":false,"phase":"warmup","estSeconds":65,"concept":"pick-embedding-by-corpus","prompt":"Match each project to the right embedding choice. Remember the\ncost table: `text-embedding-3-small` is $0.02/1M tokens (default),\n`text-embedding-3-large` is $0.13/1M (6.5× more), local\nsentence-transformers is free but slower.\n\nWhich option correctly matches ALL FOUR projects?\n","options":[{"id":"correct","label":"A=large (accuracy), B=small (scale), C=small (default), D=small (multilingual works in small)"},{"id":"large_everything","label":"All four → text-embedding-3-large for best quality","explain":"6.5× the cost for marginal gains. C is 50 pages — large is overkill. D is 100M reviews — large would cost 6.5× more for no business reason. Default to small, upgrade only with eval evidence."},{"id":"small_everything","label":"All four → text-embedding-3-small to save money","explain":"A is legal contracts where misretrieval has real consequences. That's the one case where the eval set probably shows large outperforms. Don't blanket-cheap your way into a bad legal RAG."},{"id":"local_for_a","label":"A=local sentence-transformers (privacy), B=large, C=large, D=small","explain":"Backwards. A wants the highest accuracy and the budget is flexible. B and C don't need large. Local makes sense only when the legal team specifically demands air-gapped processing."}],"answerIds":["correct"],"shuffle":false,"code":"# four real RAG projects. each has a different corpus, budget, and\n# privacy profile. which embedding tier fits each one?\n\nprojects = {\n \"A\": \"10,000 internal legal contracts. accuracy critical. budget flexible.\",\n \"B\": \"@@pd-path:path.domain.scaleCorpus@@\",\n \"C\": \"50 marketing landing pages. tiny corpus. shipping tomorrow.\",\n \"D\": \"100,000,000 product reviews. multilingual. need cheapest workable option.\",\n}\nfor k, v in projects.items():\n print(f\"{k}: {v}\")\n","runnable":true},{"type":"read","id":"context-and-retrieval/embedding-that-fits-the-budget/s03-03-the-vector-space","xp":2,"hint":[],"personalize":false,"phase":"build","estSeconds":150,"concept":"cosine-similarity-intuition","body":"$2f","cta":"Got it","code":"# cosine similarity = \"do these two vectors point in the same direction?\"\n# 1.0 = identical direction. 0.0 = perpendicular. -1.0 = opposite.\n# for embeddings, you almost always see values between 0 and 1.\n\nimport math\n\ndef cosine(a, b):\n dot = sum(x * y for x, y in zip(a, b))\n norm_a = math.sqrt(sum(x * x for x in a))\n norm_b = math.sqrt(sum(y * y for y in b))\n return dot / (norm_a * norm_b)\n\n# three faked embeddings. think of these as \"where the meaning lives.\"\nquery = [0.9, 0.1, 0.0, 0.0] # \"reset password\"\n\ncandidates = {\n \"reset my password\": [0.85, 0.15, 0.05, 0.0], # near twin\n \"cancel my subscription\": [0.1, 0.9, 0.1, 0.0], # totally different\n \"I forgot my login credentials\": [0.7, 0.3, 0.1, 0.05], # related, not identical\n}\n\nfor text, vec in candidates.items():\n score = cosine(query, vec)\n print(f\"{score:.4f} {text}\")\n","runnable":true},{"type":"predict","id":"context-and-retrieval/embedding-that-fits-the-budget/s04-04-predict-the-closest","xp":3,"hint":[{"level":1,"body":"Cosine compares direction. `doc_a` points the same way as `query` (both heavy on axis 1). `doc_b` points perpendicular (heavy on axis 2). `doc_c` shares zero direction with the query (axes 3 and 4 only). So `doc_a` wins easily, `doc_b` is moderate, `doc_c` is exactly 0.0.","cost":0},{"level":2,"body":"Score 1: `(0.8*0.9 + 0.2*0.1) / (sqrt(0.68) * sqrt(0.82)) ≈ 0.9910`. Score 2: ~0.3482. Score 3: 0.0000 (the dot product is zero — `query` has zero on axes 3 and 4, `doc_c` has zero on axes 1 and 2). Closest: doc_a.","cost":0}],"personalize":false,"phase":"build","estSeconds":80,"concept":"closest-vector-by-cosine","code":"# three \"document\" embeddings and one \"query\" embedding.\n# predict which document is closest by cosine similarity.\n\nimport math\n\ndef cosine(a, b):\n dot = sum(x * y for x, y in zip(a, b))\n na = math.sqrt(sum(x * x for x in a))\n nb = math.sqrt(sum(y * y for y in b))\n return dot / (na * nb)\n\nvectors = {\n \"doc_a\": [0.9, 0.1, 0.0, 0.0], # \"reset password\"\n \"doc_b\": [0.1, 0.9, 0.0, 0.0], # \"cancel subscription\"\n \"doc_c\": [0.0, 0.0, 0.9, 0.1], # \"export data\"\n}\nquery = [0.8, 0.2, 0.0, 0.0] # \"forgot my password\"\n\nfor name, vec in vectors.items():\n print(f\"{name}: {cosine(query, vec):.4f}\")\n\nbest = max(vectors.items(), key=lambda kv: cosine(query, kv[1]))\nprint(f\"closest: {best[0]}\")\n","prompt":"Look at the four vectors in the editor. The query vector\n`[0.8, 0.2, 0.0, 0.0]` is dominated by the first dimension —\nweight 0.8 on axis 1, weak elsewhere.\n\nWhich doc shares that pattern? Predict the exact stdout: three\nsimilarity scores (4 decimal places each) and the name of the\nclosest doc.\n","grader":{"kind":"stdout-equality","expected":"doc_a: 0.9910\ndoc_b: 0.3482\ndoc_c: 0.0000\nclosest: doc_a\n","normalize":"collapse-trailing-newline"}},{"type":"fill","id":"context-and-retrieval/embedding-that-fits-the-budget/s05-05-fill-the-cosine-formula","xp":3,"hint":[{"level":1,"body":"The dot product is exactly the same pattern as `norm_a`'s inner sum, but you multiply `a[i] * b[i]` instead of `a[i] * a[i]`. The norm of `b` mirrors `norm_a` — just swap `a` for `b`.","cost":0},{"level":2,"body":"```python\ndot = sum(x * y for x, y in zip(a, b))\nnorm_b = math.sqrt(sum(x * x for x in b))\n```\n","cost":0}],"personalize":false,"phase":"build","estSeconds":70,"concept":"cosine-formula-components","prompt":"Cosine similarity has two pieces: the **dot product** of two\nvectors (numerator) divided by the **product of their norms**\n(denominator). Fill in the two expressions.\n\n- The dot product is the sum over `i` of `a[i] * b[i]`.\n- The norm of a vector is `sqrt(sum of squares)`.\n\nExpected output:\n```\n0.9910\n```\n","code":"import math\n\na = [0.8, 0.2, 0.0, 0.0]\nb = [0.9, 0.1, 0.0, 0.0]\n\ndot = ___\nnorm_a = math.sqrt(sum(x * x for x in a))\nnorm_b = ___\n\ncosine = dot / (norm_a * norm_b)\nprint(f\"{cosine:.4f}\")\n","blanks":[{"id":"dot_product","accept":["sum(x * y for x, y in zip(a, b))","sum(x*y for x, y in zip(a, b))","sum(x*y for x,y in zip(a,b))","sum(a[i] * b[i] for i in range(len(a)))","sum(a[i]*b[i] for i in range(len(a)))"],"caseSensitive":true,"normalize":"trim"},{"id":"norm_b","accept":["math.sqrt(sum(x * x for x in b))","math.sqrt(sum(x*x for x in b))","math.sqrt(sum(y * y for y in b))","math.sqrt(sum(y*y for y in b))"],"caseSensitive":true,"normalize":"trim"}]},{"type":"fix","id":"context-and-retrieval/embedding-that-fits-the-budget/s06-06-fix-the-wrong-embedding-model","xp":4,"hint":[{"level":1,"body":"Look at the doc embedding step — it uses `embed_a`. The query is using `embed_b`. The vectors aren't in the same space. Change the query line to use the same function as the docs.","cost":0},{"level":2,"body":"```python\nquery_vec = embed_a(\"reset my password\")\n```\nRule of thumb: route every embedding through ONE function in your codebase. If `embed_query()` and `embed_doc()` exist as separate functions, they're an accident waiting to happen — wrap them both around the same model call.\n","cost":0}],"personalize":false,"phase":"build","estSeconds":110,"concept":"same-model-for-both-sides","brokenCode":"$30","prompt":"This is the bug that ships to production most often: the query\ngets embedded with one model, the documents with another. Cosine\nsimilarity is meaningless across two different vector spaces — you\nget garbage retrieval.\n\nIn the editor, two stub \"models\" (`embed_a` and `embed_b`) mimic\nthe OpenAI vs Voyage situation: same input, completely different\noutput spaces. The docs were embedded with `embed_a`. The query\nwas accidentally embedded with `embed_b`.\n\nFix it so the query is embedded with the SAME model as the docs.\nThe user asks \"reset my password\" — the matching doc should\nreturn a cosine of 1.0.\n\nExpected output:\n```\n1.0000 reset my password\n0.0000 cancel my subscription\n0.0000 export my data\n```\n","grader":{"kind":"stdout-equality","expected":"1.0000 reset my password\n0.0000 cancel my subscription\n0.0000 export my data\n","normalize":"collapse-trailing-newline"},"bugLines":[30],"revealAfter":6},{"type":"fix","id":"context-and-retrieval/embedding-that-fits-the-budget/s07-07-fix-the-missing-normalization","xp":4,"hint":[{"level":1,"body":"Divide the dot product by `norm(a) * norm(b)`, where `norm(v) = math.sqrt(sum(x*x for x in v))`. Without that step, you're not computing cosine — you're computing raw dot product, which prefers bigger vectors regardless of direction.","cost":0},{"level":2,"body":"```python\ndef similarity(a, b):\n dot = sum(x * y for x, y in zip(a, b))\n na = math.sqrt(sum(x * x for x in a))\n nb = math.sqrt(sum(y * y for y in b))\n return dot / (na * nb)\n```\nTip: some embedding APIs return pre-normalized vectors (all of magnitude 1.0). With those, dot product IS cosine. But never assume — measure the norm of a few sample vectors first.\n","cost":0}],"personalize":false,"phase":"build","estSeconds":110,"concept":"normalize-before-cosine","brokenCode":"import math\n\n# bug: this is JUST the dot product. no normalization. distorts retrieval.\ndef similarity(a, b):\n return sum(x * y for x, y in zip(a, b))\n\nquery = [1.0, 0.0, 0.0]\ndocs = {\n \"doc_match\": [0.5, 0.0, 0.0], # exact direction, small magnitude\n \"doc_long\": [0.6, 0.6, 0.6], # wrong direction, big magnitude\n}\n\nscores = {name: similarity(query, vec) for name, vec in docs.items()}\nfor name, score in scores.items():\n print(f\"{name}: {score:.4f}\")\n\nwinner = max(scores.items(), key=lambda kv: kv[1])[0]\nprint(f\"winner: {winner}\")\n","prompt":"Someone wrote a \"fast similarity\" that's just the dot product —\nthey skipped the normalization step. It compiles, it runs, and it\nRETRIEVES THE WRONG DOCUMENT.\n\nWhy: dot product without normalization rewards big vectors. A\ndoc that points the wrong direction but has a bigger magnitude\nbeats a doc that points the right direction with a smaller\nmagnitude.\n\nIn the editor, `doc_match` is an exact-direction match for the\nquery (cosine = 1.0). `doc_long` points a totally different\ndirection but has a bigger magnitude. The broken function thinks\n`doc_long` is closer. Fix the function to divide by the norms.\n\nExpected output:\n```\ndoc_match: 1.0000\ndoc_long: 0.5774\nwinner: doc_match\n```\n","grader":{"kind":"stdout-equality","expected":"doc_match: 1.0000\ndoc_long: 0.5774\nwinner: doc_match\n","normalize":"collapse-trailing-newline"},"bugLines":[4,5],"revealAfter":6},{"type":"write","id":"context-and-retrieval/embedding-that-fits-the-budget/s08-08-write-the-similarity-ranker","xp":5,"hint":[{"level":1,"body":"Three steps: (1) compute cosine of `query_vec` against each entry in `doc_vecs`, keeping track of the index, (2) sort by descending score, (3) return just the indices. `enumerate(doc_vecs)` gives you `(i, vec)` pairs to start with.","cost":0},{"level":2,"body":"```python\nscored = [(i, cosine(query_vec, v)) for i, v in enumerate(doc_vecs)]\nscored.sort(key=lambda t: -t[1])\nreturn [i for i, _ in scored]\n```\nThe `key=lambda t: -t[1]` trick sorts descending without needing `reverse=True`. Equivalent.\n","cost":0}],"personalize":false,"phase":"build","estSeconds":200,"concept":"rank-by-cosine-similarity","prompt":"Write `rank_by_similarity(query_vec, doc_vecs)` that returns a\nlist of doc INDICES sorted by descending cosine similarity to the\nquery. The highest-scoring doc comes first.\n\n- Input: a query vector and a list of doc vectors (all the same dim).\n- Output: list of indices into `doc_vecs`, sorted best-first.\n- Use cosine similarity (dot product / product of norms).\n\nA query and four docs run for you. The query points heavily in\nthe first dimension. Doc 1 matches it closely. Doc 3 is the\nnext-closest. Docs 2 and 0 point elsewhere.\n\nExpected output:\n```\n[1, 3, 2, 0]\n```\n","starter":"import math\n\n# write rank_by_similarity(query_vec, doc_vecs) below\n\nquery = [0.9, 0.1, 0.0, 0.0]\ndoc_vecs = [\n [0.1, 0.9, 0.0, 0.1], # 0 — cancel subscription (off direction)\n [0.85, 0.15, 0.05, 0.0], # 1 — reset password (near twin of query)\n [0.2, 0.1, 0.8, 0.3], # 2 — enterprise pricing (off direction)\n [0.5, 0.5, 0.2, 0.4], # 3 — partial match\n]\nprint(rank_by_similarity(query, doc_vecs))\n","grader":{"kind":"stdout-equality","expected":"[1, 3, 2, 0]","normalize":"collapse-trailing-newline"},"solution":"import math\n\ndef rank_by_similarity(query_vec, doc_vecs):\n def cosine(a, b):\n dot = sum(x * y for x, y in zip(a, b))\n na = math.sqrt(sum(x * x for x in a))\n nb = math.sqrt(sum(y * y for y in b))\n return dot / (na * nb)\n\n scored = [(i, cosine(query_vec, v)) for i, v in enumerate(doc_vecs)]\n scored.sort(key=lambda t: -t[1])\n return [i for i, _ in scored]\n\nquery = [0.9, 0.1, 0.0, 0.0]\ndoc_vecs = [\n [0.1, 0.9, 0.0, 0.1],\n [0.85, 0.15, 0.05, 0.0],\n [0.2, 0.1, 0.8, 0.3],\n [0.5, 0.5, 0.2, 0.4],\n]\nprint(rank_by_similarity(query, doc_vecs))\n","hiddenTests":[]},{"type":"checkpoint","id":"context-and-retrieval/embedding-that-fits-the-budget/s09-09-checkpoint","xp":8,"hint":[],"personalize":false,"phase":"check","estSeconds":220,"concept":"embedding-pipeline-end-to-end","prompt":"Final drill. Wire it all together into a tiny FAQ search engine —\nthe same shape as a real RAG retrieval step, just with stub\nembeddings instead of an API call.\n\nBuild `top_k(query_vec, faqs, k)`:\n\n- `faqs` is a list of `(text, vec)` tuples — 5 of them in the\n starter.\n- `query_vec` is a pre-computed embedding for the user's question.\n- Compute cosine similarity between `query_vec` and each FAQ vec.\n- Return the `k` highest-scoring FAQs as `(text, score)` tuples,\n best first.\n\nThe starter calls your function with `k=2` to return the top-2\nclosest FAQs to the question \"I forgot my password.\"\n\nReal-world framing: this is the retrieval step of RAG. In\nproduction you'd swap the hand-picked vectors for real OpenAI\nembeddings, swap the list of 5 for thousands of chunks in a\nvector database, and feed the top-K results into the prompt you\nsend to the model.\n\nExpected output:\n```\nquery: 'I forgot my password'\ntop 2 matches:\n 0.9963 how do I reset my password?\n 0.2457 how do I upgrade my plan?\n```\n","starter":"import math\n\n# write top_k(query_vec, faqs, k) below\n\nfaqs = [\n (\"how do I reset my password?\", [0.9, 0.1, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0]),\n (\"how do I cancel my subscription?\", [0.1, 0.9, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0]),\n (\"how do I upgrade my plan?\", [0.1, 0.7, 0.2, 0.0, 0.0, 0.1, 0.0, 0.1]),\n (\"how do I export my data?\", [0.0, 0.0, 0.9, 0.1, 0.0, 0.0, 0.1, 0.0]),\n (\"what are your business hours?\", [0.0, 0.0, 0.0, 0.0, 0.9, 0.1, 0.0, 0.1]),\n]\n\nquery_text = \"I forgot my password\"\nquery_vec = [0.85, 0.1, 0.0, 0.0, 0.15, 0.0, 0.0, 0.05]\n\nresults = top_k(query_vec, faqs, 2)\n\nprint(f\"query: {query_text!r}\")\nprint(\"top 2 matches:\")\nfor text, score in results:\n print(f\" {score:.4f} {text}\")\n","grader":{"kind":"stdout-equality","expected":"query: 'I forgot my password'\ntop 2 matches:\n 0.9963 how do I reset my password?\n 0.2457 how do I upgrade my plan?\n","normalize":"collapse-trailing-newline"},"solution":"$31","passThreshold":0.66}],"xpTotal":32},{"slug":"retrieval-that-finds-the-right-thing","title":"Retrieval that finds the right thing — top-k, thresholds, and the rerank step everyone skips","estMinutes":14,"prerequisites":["embedding-that-fits-the-budget"],"status":"live","steps":[{"type":"read","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s01-01-intro","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":130,"concept":"top-k-is-not-enough","body":"$32","cta":"Got it","code":"# cosine scores for 5 candidate chunks against a single query.\nscored = [\n (\"chunk_42\", 0.91), # clearly relevant\n (\"chunk_07\", 0.87), # relevant\n (\"chunk_18\", 0.84), # related\n (\"chunk_31\", 0.39), # tangential\n (\"chunk_56\", 0.12), # completely unrelated\n]\n\n# naive top-k: take the highest 3 regardless of how low they go.\ntop_3 = sorted(scored, key=lambda x: x[1], reverse=True)[:3]\nfor c, s in top_3:\n print(f\"{c}: {s:.2f}\")\n","runnable":true},{"type":"mc","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s02-02-which-result-is-good","xp":2,"hint":[{"level":1,"body":"Three signals make a result set healthy: high scores, distinct chunks, and chunks from the obviously-relevant source. Look for the set that hits all three.","cost":0}],"personalize":false,"phase":"warmup","estSeconds":55,"concept":"when-top-k-is-misleading","prompt":"Four retrieval result sets. Which ONE is the only set you'd be\nwilling to send to the model as-is?\n","options":[{"id":"a","label":"A — three policy_doc chunks with scores 0.84+"},{"id":"b","label":"B — finance + HR chunks with scores 0.49 to 0.61","explain":"Scores below ~0.7 should make you suspicious; below 0.5 is noise on most embedding models. Sending finance docs as 'refund policy' is the hallucination setup."},{"id":"c","label":"C — policy_doc/p2 appears TWICE in the top-3","explain":"Duplicate chunks waste a slot in the prompt. The model gets less unique context, the bill is the same, the answer is no better. Dedupe before sending."},{"id":"d","label":"D — all three results are the exact same chunk","explain":"This means either your index has duplicates or your top-k sort is broken. Either way, you're sending one chunk's worth of signal pretending to be three."}],"answerIds":["a"],"shuffle":false,"code":"# four retrieval result sets for the query: \"refund policy?\"\nresults = {\n \"A\": [(\"policy_doc/p2\", 0.88), (\"policy_doc/p3\", 0.86), (\"policy_doc/p9\", 0.84)],\n \"B\": [(\"finance/q4_2024\", 0.61), (\"finance/q3_2024\", 0.55), (\"hr/onboarding\", 0.49)],\n \"C\": [(\"policy_doc/p2\", 0.91), (\"policy_doc/p2\", 0.91), (\"policy_doc/p4\", 0.87)],\n \"D\": [(\"policy_doc/p2\", 0.89), (\"policy_doc/p2\", 0.89), (\"policy_doc/p2\", 0.89)],\n}\nfor k, v in results.items():\n print(f\"{k}: {[s for _, s in v]}\")\n","runnable":true},{"type":"read","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s03-03-the-distance-threshold","xp":1,"hint":[],"personalize":false,"phase":"build","estSeconds":110,"concept":"cosine-threshold-cutoff","body":"$33","cta":"Got it","code":"# naive top-k vs. threshold-then-top-k.\nscored = [\n (\"chunk_42\", 0.91),\n (\"chunk_07\", 0.87),\n (\"chunk_18\", 0.84),\n (\"chunk_31\", 0.39),\n (\"chunk_56\", 0.12),\n]\n\nTHRESHOLD = 0.5\n\ndef naive_top_k(scored, k):\n return sorted(scored, key=lambda x: x[1], reverse=True)[:k]\n\ndef thresholded_top_k(scored, k, threshold):\n keep = [(c, s) for c, s in scored if s >= threshold]\n return sorted(keep, key=lambda x: x[1], reverse=True)[:k]\n\nprint(\"naive top-5: \", [c for c, _ in naive_top_k(scored, 5)])\nprint(\"with threshold: \", [c for c, _ in thresholded_top_k(scored, 5, THRESHOLD)])\n","runnable":true},{"type":"predict","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s04-04-predict-the-top-k","xp":3,"hint":[{"level":1,"body":"Three candidates clear 0.5: doc_a (0.92), doc_b (0.78), doc_c (0.55). doc_d (0.41) and doc_e (0.12) get filtered out. The top-3 cap never bites because only 3 candidates survived.","cost":0},{"level":2,"body":"Output is doc_a, doc_b, doc_c in descending score order, with two-decimal formatting.","cost":0}],"personalize":false,"phase":"build","estSeconds":80,"concept":"top-k-with-threshold","code":"scored = [\n (\"doc_a\", 0.92),\n (\"doc_b\", 0.78),\n (\"doc_c\", 0.55),\n (\"doc_d\", 0.41),\n (\"doc_e\", 0.12),\n]\n\nTHRESHOLD = 0.5\nK = 3\n\nkeep = [(c, s) for c, s in scored if s >= THRESHOLD]\ntop = sorted(keep, key=lambda x: x[1], reverse=True)[:K]\n\nfor c, s in top:\n print(f\"{c}: {s:.2f}\")\n","prompt":"Five candidates, threshold 0.5, asking for top-3. Predict the\nprinted output. (Hint: only three candidates clear the threshold,\nso the top-3 cap is generous.)\n\nWhat does the script print?\n","grader":{"kind":"stdout-equality","expected":"doc_a: 0.92\ndoc_b: 0.78\ndoc_c: 0.55\n","normalize":"collapse-trailing-newline"}},{"type":"fill","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s05-05-fill-the-cutoff","xp":3,"hint":[{"level":1,"body":"The `(c, s)` unpacking exposes `s` as the score. Compare `s` to `THRESHOLD` with `>=` (not `>` — you want exact-threshold matches to qualify).","cost":0},{"level":2,"body":"`s >= THRESHOLD`. That keeps 0.91, 0.62, and 0.51 (three). 0.48 and 0.10 drop out.","cost":0}],"personalize":false,"phase":"build","estSeconds":65,"concept":"threshold-filter-with-list-comp","prompt":"Fill the list comprehension so that only candidates with a score\nat least THRESHOLD survive. Use a single `>=` comparison.\n\nExpected output:\n```\n3 candidates above threshold\n```\n","code":"scored = [\n (\"a\", 0.91), (\"b\", 0.62), (\"c\", 0.48), (\"d\", 0.51), (\"e\", 0.10),\n]\nTHRESHOLD = 0.5\n\nkeep = [(c, s) for c, s in scored if ___]\n\nprint(f\"{len(keep)} candidates above threshold\")\n","blanks":[{"id":"filter_expr","accept":["s >= THRESHOLD","s>=THRESHOLD"],"caseSensitive":true,"normalize":"trim"}]},{"type":"fix","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s06-06-fix-the-no-threshold","xp":4,"hint":[{"level":1,"body":"Filter the list before sorting. `keep = [(c, s) for c, s in scored if s >= 0.5]`, then sort and slice.","cost":0},{"level":2,"body":"Replace the function body with two lines:\n```python\nkeep = [(c, s) for c, s in scored if s >= 0.5]\nreturn sorted(keep, key=lambda x: x[1], reverse=True)[:k]\n```\nWith this corpus, `keep` is empty, the sort returns [], and the slice is also [].\n","cost":0}],"personalize":false,"phase":"build","estSeconds":95,"concept":"threshold-before-slice","brokenCode":"scored = [\n (\"noise_1\", 0.21),\n (\"noise_2\", 0.18),\n (\"noise_3\", 0.15),\n (\"noise_4\", 0.09),\n]\n\ndef retrieve(scored, k=3):\n # bug: no threshold. returns garbage when nothing is relevant.\n return sorted(scored, key=lambda x: x[1], reverse=True)[:k]\n\nprint([c for c, _ in retrieve(scored, k=3)])\n","prompt":"AI wrote `retrieve(scored, k=3)` that takes the top-3 candidates\nregardless of how low their scores go. The test query has zero\nrelevant chunks in the corpus, so the top-3 returns three garbage\nchunks the model would treat as real.\n\nAdd a threshold filter (>= 0.5) BEFORE the slice. The function\nshould return an empty list when nothing clears the bar.\n\nExpected output:\n```\n[]\n```\n","grader":{"kind":"stdout-equality","expected":"[]","normalize":"collapse-trailing-newline"},"bugLines":[10],"revealAfter":4},{"type":"fix","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s07-07-fix-the-dup-results","xp":4,"hint":[{"level":1,"body":"Sort by score first, then walk the sorted list keeping a `seen` set. Skip any chunk_id already in `seen`.","cost":0},{"level":2,"body":"```python\nranked = sorted(scored, key=lambda x: x[1], reverse=True)\nseen = set()\nout = []\nfor c, s in ranked:\n if c not in seen:\n seen.add(c)\n out.append((c, s))\n if len(out) == k:\n break\nreturn out\n```\n","cost":0}],"personalize":false,"phase":"build","estSeconds":100,"concept":"dedupe-before-top-k","brokenCode":"scored = [\n (\"policy/p2\", 0.91),\n (\"policy/p2\", 0.89), # same chunk, slightly lower score\n (\"policy/p4\", 0.85),\n (\"policy/p7\", 0.72),\n (\"policy/p9\", 0.65),\n]\n\ndef retrieve(scored, k=3):\n # bug: no dedupe. policy/p2 takes two slots out of three.\n return sorted(scored, key=lambda x: x[1], reverse=True)[:k]\n\nprint([c for c, _ in retrieve(scored, k=3)])\n","prompt":"Two documents indexed the same paragraph (it was copy-pasted\nacross docs). The retriever returns the same chunk_id twice in\nthe top-3, wasting a slot. The fix is to dedupe by chunk_id\nBEFORE taking the top-k. Keep the highest-scoring occurrence.\n\nExpected output:\n```\n['policy/p2', 'policy/p4', 'policy/p7']\n```\n","grader":{"kind":"stdout-equality","expected":"['policy/p2', 'policy/p4', 'policy/p7']","normalize":"collapse-trailing-newline"},"bugLines":[11],"revealAfter":4},{"type":"write","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s08-08-write-the-ranker","xp":5,"hint":[{"level":1,"body":"Three steps in order: (1) filter by threshold, (2) sort by score descending, (3) walk the sorted list with a `seen` set to dedupe, stopping at k results.","cost":0},{"level":2,"body":"Body sketch: `filtered = [...]`, `ranked = sorted(...)`, init `seen = set()` and `out = []`, then a for-loop that skips dupes, appends, and breaks at `len(out) == k`.","cost":0}],"personalize":false,"phase":"build","estSeconds":200,"concept":"threshold-dedupe-top-k","prompt":"Write `retrieve(scored, k, threshold)` that returns the top-k\ncandidates, filtered by `score >= threshold`, deduped by\nchunk_id (keep the highest-scoring occurrence). Order the\noutput by descending score.\n\nTwo cases run. Expected output:\n```\n['policy/p2', 'policy/p4', 'policy/p7']\n[]\n```\n","starter":"good = [\n (\"policy/p2\", 0.91),\n (\"policy/p2\", 0.89), # dup, lower score\n (\"policy/p4\", 0.85),\n (\"policy/p7\", 0.72),\n (\"noise\", 0.31), # below threshold\n]\n\nnoisy = [\n (\"garbage_1\", 0.22),\n (\"garbage_2\", 0.18),\n (\"garbage_3\", 0.15),\n]\n\n# define retrieve(scored, k, threshold) below\n\nprint([c for c, _ in retrieve(good, k=3, threshold=0.5)])\nprint([c for c, _ in retrieve(noisy, k=3, threshold=0.5)])\n","grader":{"kind":"stdout-equality","expected":"['policy/p2', 'policy/p4', 'policy/p7']\n[]\n","normalize":"collapse-trailing-newline"},"solution":"good = [\n (\"policy/p2\", 0.91),\n (\"policy/p2\", 0.89),\n (\"policy/p4\", 0.85),\n (\"policy/p7\", 0.72),\n (\"noise\", 0.31),\n]\n\nnoisy = [\n (\"garbage_1\", 0.22),\n (\"garbage_2\", 0.18),\n (\"garbage_3\", 0.15),\n]\n\ndef retrieve(scored, k, threshold):\n filtered = [(c, s) for c, s in scored if s >= threshold]\n ranked = sorted(filtered, key=lambda x: x[1], reverse=True)\n seen = set()\n out = []\n for c, s in ranked:\n if c in seen:\n continue\n seen.add(c)\n out.append((c, s))\n if len(out) == k:\n break\n return out\n\nprint([c for c, _ in retrieve(good, k=3, threshold=0.5)])\nprint([c for c, _ in retrieve(noisy, k=3, threshold=0.5)])\n","hiddenTests":[]},{"type":"checkpoint","id":"context-and-retrieval/retrieval-that-finds-the-right-thing/s09-09-checkpoint","xp":8,"hint":[],"personalize":false,"phase":"check","estSeconds":220,"concept":"full-retrieval-pipeline","prompt":"Final drill. Build `rag_retrieve(query_vec, indexed, k, threshold)`\nthat:\n\n- Takes `indexed` as a list of `(chunk_id, doc_vec)` pairs.\n- Computes cosine similarity between `query_vec` and each `doc_vec`\n (use `cosine` provided in the starter).\n- Filters by `score >= threshold`.\n- Dedupes by `chunk_id`, keeping the highest-scoring occurrence.\n- Returns the top-k chunk_ids in descending score order.\n\nThree cases run. Expected output:\n```\n['policy/refund_terms', 'policy/refund_eligibility']\n[]\n['policy/refund_terms']\n```\n","starter":"$34","grader":{"kind":"stdout-equality","expected":"['policy/refund_terms', 'policy/refund_eligibility']\n[]\n['policy/refund_terms']\n","normalize":"collapse-trailing-newline"},"solution":"$35","passThreshold":0.66}],"xpTotal":31},{"slug":"rag-vs-long-context-vs-fine-tune","title":"RAG vs long context vs fine-tune — the decision that's killed more AI startups than any model swap","estMinutes":22,"prerequisites":["retrieval-that-finds-the-right-thing"],"status":"live","steps":[{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s01-01-the-three-way-fork","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":200,"concept":"the-three-way-fork","body":"$36","cta":"Got it","runnable":true},{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s02-02-rag-when-you-need-freshness","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":200,"concept":"rag-when-fresh","body":"$37","cta":"Got it","runnable":true},{"type":"mc","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s03-03-which-fork-for-this-scenario","xp":2,"hint":[{"level":1,"body":"Map each scenario to its dominant axis. A: freshness (changes 3x/week). B: stable corpus per session (one contract). C: capability gap (house style). D: stable corpus, small enough to fit in context. Then pick the fork that wins on that axis.","cost":0}],"personalize":false,"phase":"build","estSeconds":60,"concept":"pick-the-fork-for-scenario","prompt":"Four product scenarios. Each one has a clear best fork (RAG,\nlong-context, or fine-tune). Per the five-axis tradeoffs from the\nreading, which assignment is correct?\n","options":[{"id":"a_rag_b_lc_c_ft_d_lc","label":"A=RAG, B=long-context, C=fine-tune, D=long-context"},{"id":"a_rag_b_rag_c_rag_d_rag","label":"A=RAG, B=RAG, C=RAG, D=RAG","explain":"RAG isn't a universal answer. A handbook that never changes doesn't need a vector store. A contract you load once per session fits in context. A style problem isn't a freshness problem."},{"id":"a_ft_b_rag_c_lc_d_ft","label":"A=fine-tune, B=RAG, C=long-context, D=fine-tune","explain":"Fine-tune on docs that ship 3x/week means retraining 3x/week — the worst possible match. And fine-tuning a handbook locks the policy into weights, no citations, no updates."},{"id":"a_lc_b_ft_c_rag_d_ft","label":"A=long-context, B=fine-tune, C=RAG, D=fine-tune","explain":"Long-context on a daily-changing doc set means re-stuffing the prompt every release with no cache hits. Fine-tuning a 400-page one-off contract is absurd — it'll never be queried again."}],"answerIds":["a_rag_b_lc_c_ft_d_lc"],"shuffle":false,"code":"# four product scenarios. each picks ONE fork. which goes with which?\nscenarios = [\n (\"A\", \"Customer support bot reading docs that ship 3x/week\"),\n (\"B\", \"Legal-review bot loading one 400-page contract per session\"),\n (\"C\", \"Code review bot trained to match a company's house style\"),\n (\"D\", \"Internal handbook chatbot for a 180-page policy doc that hasn't changed in 18 months\"),\n]\nfor k, v in scenarios:\n print(f\"{k}: {v}\")\n","runnable":true},{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s04-04-long-context-when-the-corpus-fits","xp":1,"hint":[],"personalize":false,"phase":"build","estSeconds":200,"concept":"long-context-when-fits","body":"$38","cta":"Got it","runnable":true},{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s05-05-fine-tune-when-style-or-format-is-the-product","xp":1,"hint":[],"personalize":false,"phase":"build","estSeconds":220,"concept":"fine-tune-when-style","body":"$39","cta":"Got it","runnable":true},{"type":"mc","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s06-06-which-fork-dies-first","xp":2,"hint":[{"level":1,"body":"Fine-tune's main wedge was style/format that base models couldn't match. As base models get better at in-context style transfer and few-shot prompting, that gap closes. Plus fine-tune-as-a-service democratizes the moat. Harvey's defense becomes thin when GPT-6 + 100 example prompts produces 90% of Harvey's quality. RAG and long-context have structural moats fine-tune doesn't.","cost":0}],"personalize":false,"phase":"build","estSeconds":70,"concept":"which-fork-dies-first","prompt":"Look ahead to 2027. Context windows are 10M tokens commodity.\nFine-tune-as-a-service is cheap. Vector store infra is\ncommoditized. Given those three trends, WHICH fork has the worst\nlong-term odds as a standalone architecture?\n","options":[{"id":"rag","label":"RAG dies first","explain":"RAG still owns big-corpus + high-freshness, both of which compound as enterprises generate more data. Long-context encroached on the small-corpus territory but the big-corpus story is structurally untouchable. RAG is squeezed, not dying."},{"id":"long_context","label":"Long-context dies first","explain":"Long-context is gaining territory in 2027, not losing it. 10M-token windows and longer cache TTLs make it dominant for the corpus shapes it already won. Wrong direction."},{"id":"fine_tune","label":"Fine-tune dies first"},{"id":"all_three","label":"All three die, replaced by something new","explain":"Tempting but lazy. The three forks describe physics-level options for getting knowledge into a model: pull at query time, stuff in context, bake in weights. Those don't disappear; they just shift in relative dominance."}],"answerIds":["fine_tune"],"shuffle":false,"code":"$3a","runnable":true},{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s07-07-graph-vector-or-both","xp":1,"hint":[],"personalize":false,"phase":"build","estSeconds":150,"concept":"retrieval-shape-graph-vs-vector","body":"$3b","cta":"Got it","code":"edges = [\n {\"from\": \"acme\", \"rel\": \"owned_by\", \"to\": \"globex\"},\n {\"from\": \"globex\", \"rel\": \"based_in\", \"to\": \"berlin\"},\n]\n\ndef follow(start, rel):\n for e in edges:\n if e[\"from\"] == start and e[\"rel\"] == rel:\n return e[\"to\"]\n\nparent = follow(\"acme\", \"owned_by\")\nprint(\"acme's parent:\", parent)\nprint(\"parent based in:\", follow(parent, \"based_in\"))\n","runnable":true},{"type":"read","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s08-08-the-rubric","xp":1,"hint":[],"personalize":false,"phase":"build","estSeconds":200,"concept":"the-fork-rubric","body":"$3c","cta":"Got it","runnable":true},{"type":"write","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s09-09-write-the-fork-decision","xp":5,"hint":[{"level":1,"body":"Six if-branches, first match wins. Use `.get()` with sane defaults so missing fields don't crash. Order matters — the size-exceeds-2M check has to be first because it overrides everything else (RAG by physics).","cost":0},{"level":2,"body":"Glean: corpus is 50B tokens, way over 2M, so rule 1 fires immediately and returns rag. Harvey: corpus is 1.5M tokens (under 2M), so rule 1 doesn't fire. Rule 2 checks style_critical (True) AND update_frequency_days <= 7 (3 days qualifies) → both true, return hybrid. The order of rules matters: hybrid has to come before plain fine-tune, otherwise Harvey would return fine_tune instead.","cost":0}],"personalize":false,"phase":"build","estSeconds":200,"concept":"pick-fork-decision-function","prompt":"$3d","starter":"glean = {\n \"name\": \"Glean (enterprise search)\",\n \"corpus_size_tokens\": 50_000_000_000,\n \"update_frequency_days\": 1,\n \"style_critical\": False,\n \"cost_per_call_max\": 0.05,\n \"latency_budget_ms\": 2000,\n}\n\nharvey = {\n \"name\": \"Harvey (legal AI)\",\n \"corpus_size_tokens\": 1_500_000,\n \"update_frequency_days\": 3,\n \"style_critical\": True,\n \"cost_per_call_max\": 0.50,\n \"latency_budget_ms\": 5000,\n}\n\n# define pick_fork(spec) below\n\nprint(f\"{glean['name'] + ':':33}{pick_fork(glean)}\")\nprint(f\"{harvey['name'] + ':':33}{pick_fork(harvey)}\")\n","grader":{"kind":"stdout-equality","expected":"Glean (enterprise search): rag\nHarvey (legal AI): hybrid\n","normalize":"collapse-trailing-newline"},"solution":"glean = {\n \"name\": \"Glean (enterprise search)\",\n \"corpus_size_tokens\": 50_000_000_000,\n \"update_frequency_days\": 1,\n \"style_critical\": False,\n \"cost_per_call_max\": 0.05,\n \"latency_budget_ms\": 2000,\n}\n\nharvey = {\n \"name\": \"Harvey (legal AI)\",\n \"corpus_size_tokens\": 1_500_000,\n \"update_frequency_days\": 3,\n \"style_critical\": True,\n \"cost_per_call_max\": 0.50,\n \"latency_budget_ms\": 5000,\n}\n\ndef pick_fork(spec):\n if spec.get(\"corpus_size_tokens\", 0) > 2_000_000:\n return \"rag\"\n if spec.get(\"style_critical\", False) and spec.get(\"update_frequency_days\", 9999) <= 7:\n return \"hybrid\"\n if spec.get(\"style_critical\", False):\n return \"fine_tune\"\n if spec.get(\"update_frequency_days\", 9999) <= 7:\n return \"rag\"\n if spec.get(\"corpus_size_tokens\", 0) <= 2_000_000:\n return \"long_context\"\n return \"rag\"\n\nprint(f\"{glean['name'] + ':':33}{pick_fork(glean)}\")\nprint(f\"{harvey['name'] + ':':33}{pick_fork(harvey)}\")\n","hiddenTests":[]},{"type":"checkpoint","id":"context-and-retrieval/rag-vs-long-context-vs-fine-tune/s10-10-checkpoint","xp":8,"hint":[],"personalize":false,"phase":"check","estSeconds":240,"concept":"evaluate-fork-against-product","prompt":"$3e","starter":"harvey = {\n \"name\": \"Harvey\",\n \"corpus_size_tokens\": 1_500_000,\n \"update_frequency_days\": 3,\n \"style_critical\": True,\n \"cost_per_call_max\": 0.50,\n \"latency_budget_ms\": 5000,\n}\n\nglean = {\n \"name\": \"Glean\",\n \"corpus_size_tokens\": 50_000_000_000,\n \"update_frequency_days\": 1,\n \"style_critical\": False,\n \"cost_per_call_max\": 0.05,\n \"latency_budget_ms\": 2000,\n}\n\nhandbook_bot = {\n \"name\": \"Handbook bot\",\n \"corpus_size_tokens\": 80_000,\n \"update_frequency_days\": 365,\n \"style_critical\": False,\n \"cost_per_call_max\": 0.05,\n \"latency_budget_ms\": 2000,\n}\n\n# define evaluate_strategy(product, fork) below\n\nprint(f\"Harvey + fine_tune: {evaluate_strategy(harvey, 'fine_tune')}\")\nprint(f\"Glean + long_context: {evaluate_strategy(glean, 'long_context')}\")\nprint(f\"Handbook bot + rag: {evaluate_strategy(handbook_bot, 'rag')}\")\n","grader":{"kind":"stdout-equality","expected":"Harvey + fine_tune: (False, 'freshness too high for fine-tune')\nGlean + long_context: (False, 'corpus too large for long-context window')\nHandbook bot + rag: (False, 'RAG overkill for small stable corpus')\n","normalize":"collapse-trailing-newline"},"solution":"$3f","passThreshold":0.66}],"xpTotal":23},{"slug":"retrieval-metrics-and-vector-db-shape","title":"Retrieval metrics and vector-DB shape — recall@k, MRR, and why the filter runs before the search","estMinutes":15,"prerequisites":["retrieval-that-finds-the-right-thing"],"status":"live","steps":[{"type":"read","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s01-01-the-eval-set","xp":1,"hint":[],"personalize":false,"phase":"warmup","estSeconds":200,"concept":"retrieval-eval-set","body":"$40","cta":"Got it","code":"# one eval query: what you retrieved vs what a human marked relevant.\nretrieved = [\"p7\", \"p2\", \"p9\"] # your top-3, in rank order\nrelevant = {\"p2\", \"p4\"} # gold labels for this query\n\nhits = [cid for cid in retrieved if cid in relevant]\nprint(\"retrieved:\", retrieved)\nprint(\"hits:\", hits)\nprint(f\"recall@3: {len(hits) / len(relevant):.2f}\")\nprint(f\"precision@3: {len(hits) / len(retrieved):.2f}\")\n","runnable":true},{"type":"mc","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s02-02-filter-or-search-first","xp":2,"hint":[{"level":1,"body":"The bug is an ordering bug: the slice happens before the filter, so the filter can only remove — never recover. Ask which operation has to run first so the top-k is drawn from rows that were eligible to begin with.","cost":0}],"personalize":false,"phase":"build","estSeconds":70,"concept":"prefilter-vs-postfilter","prompt":"Acme asks for its refund policy. The pipeline ranks ALL rows,\nslices the top-2, then filters by tenant — and acme sees an empty\nresult, even though `acme/refunds` sits in the index at a healthy\n0.81. What's the right fix?\n","options":[{"id":"a","label":"Raise k until acme's documents eventually survive the slice.","explain":"k=50 might work today and fail tomorrow when globex uploads more docs. You'd be tuning a knob to paper over a structural ordering bug — and a big k makes every query slower and the top-k noisier."},{"id":"b","label":"Filter rows to `tenant == \"acme\"` first, then rank and slice within that subset."},{"id":"c","label":"Drop the tenant filter — the globex chunks scored higher, so they're better answers.","explain":"They're better matches for the *words*, and a data leak for the *business*. Showing globex's documents to acme is a security incident wearing a good similarity score."},{"id":"d","label":"Lower the similarity threshold so more candidates get through.","explain":"The threshold guards against noise, not against wrong-tenant rows. globex/refunds at 0.93 clears any sane threshold — it's still the wrong tenant's document."}],"answerIds":["b"],"shuffle":false,"code":"# global top-2 for \"refund policy\", asked by tenant acme.\nscored = [\n (\"globex/refunds\", 0.93, \"globex\"),\n (\"globex/pricing\", 0.88, \"globex\"),\n (\"acme/refunds\", 0.81, \"acme\"),\n (\"acme/sla\", 0.79, \"acme\"),\n]\ntop_2 = scored[:2]\nacme_results = [cid for cid, _, t in top_2 if t == \"acme\"]\nprint(\"acme sees:\", acme_results)\n","runnable":true},{"type":"predict","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s03-03-predict-recall-precision","xp":3,"hint":[{"level":1,"body":"The slice keeps p1, p2, p9. Two of those are in the gold set — p4 was retrieved but at rank 5, so recall@3 never sees it.","cost":0},{"level":2,"body":"hits = 2. Recall line: `recall@3: 2/4 = 0.50`. Precision line: `precision@3: 2/3 = 0.67`. Both use the f-string's `:.2f` formatting.","cost":0}],"personalize":false,"phase":"build","estSeconds":90,"concept":"recall-precision-by-hand","code":"relevant = {\"p2\", \"p4\", \"p6\", \"p9\"} # 4 gold chunks for this query\nretrieved = [\"p1\", \"p2\", \"p9\", \"p5\", \"p4\"]\nK = 3\n\ntop_k = retrieved[:K]\nhits = sum(1 for cid in top_k if cid in relevant)\nprint(f\"recall@{K}: {hits}/{len(relevant)} = {hits / len(relevant):.2f}\")\nprint(f\"precision@{K}: {hits}/{K} = {hits / K:.2f}\")\n","prompt":"Four gold chunks, five retrieved, but only the top-3 counts.\nRecall divides hits by the size of the gold set; precision\ndivides the same hits by K. Walk the top-3, count the hits, and\npredict both printed lines exactly.\n","grader":{"kind":"stdout-equality","expected":"recall@3: 2/4 = 0.50\nprecision@3: 2/3 = 0.67\n","normalize":"collapse-trailing-newline"}},{"type":"fill","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s04-04-fill-the-reciprocal","xp":3,"hint":[{"level":1,"body":"\"Reciprocal rank\" is literal: one divided by the rank. Python 3's `/` already gives you a float, so `1 / 3` is 0.333, not zero.","cost":0},{"level":2,"body":"`1 / rank`. The sum is 1 + 0.333 + 0.5 = 1.833; divided by 3 queries that's 0.611, printed as 0.61.","cost":0}],"personalize":false,"phase":"build","estSeconds":75,"concept":"mean-reciprocal-rank","prompt":"MRR rewards putting the right chunk FIRST. Each query contributes\nthe reciprocal of the rank where its first relevant hit appeared:\nrank 1 → 1.0, rank 2 → 0.5, rank 3 → 0.33. Average those and\nyou have MRR.\n\nFill the blank so each query adds its reciprocal rank.\n\nExpected output:\n```\nMRR: 0.61\n```\n","code":"# rank of the FIRST relevant hit for each of three eval queries (1-based).\nfirst_hit_ranks = [1, 3, 2]\n\ntotal = 0.0\nfor rank in first_hit_ranks:\n total += ___\n\nmrr = total / len(first_hit_ranks)\nprint(f\"MRR: {mrr:.2f}\")\n","blanks":[{"id":"reciprocal","accept":["1 / rank","1/rank","1.0 / rank","1.0/rank"],"caseSensitive":true,"normalize":"trim"}]},{"type":"fix","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s05-05-fix-the-late-filter","xp":4,"hint":[{"level":1,"body":"Build the tenant's subset first — a list comprehension over `rows` keeping `r[\"tenant\"] == tenant` — then sort and slice that subset. The filter in the return line becomes unnecessary.","cost":0},{"level":2,"body":"```python\nmine = [r for r in rows if r[\"tenant\"] == tenant]\ntop = sorted(mine, key=lambda r: r[\"score\"], reverse=True)[:k]\nreturn [r[\"id\"] for r in top]\n```\nOnly acme's two rows compete for the two slots, so both survive.\n","cost":0}],"personalize":false,"phase":"build","estSeconds":100,"concept":"filter-before-rank","brokenCode":"rows = [\n {\"id\": \"acme/refunds\", \"score\": 0.81, \"tenant\": \"acme\"},\n {\"id\": \"globex/refunds\", \"score\": 0.93, \"tenant\": \"globex\"},\n {\"id\": \"globex/pricing\", \"score\": 0.88, \"tenant\": \"globex\"},\n {\"id\": \"acme/sla\", \"score\": 0.79, \"tenant\": \"acme\"},\n]\n\ndef search(rows, tenant, k=2):\n # bug: rank-and-slice happens globally; the filter runs too late.\n top = sorted(rows, key=lambda r: r[\"score\"], reverse=True)[:k]\n return [r[\"id\"] for r in top if r[\"tenant\"] == tenant]\n\nprint(search(rows, \"acme\"))\n","prompt":"The multiple-choice bug, live. `search` ranks every tenant's rows,\nslices the top-2, THEN filters by tenant — so acme gets an empty\nlist while its own refund policy sits in the index at 0.81.\n\nRearrange the function so the tenant filter runs before the\nrank-and-slice.\n\nExpected output:\n```\n['acme/refunds', 'acme/sla']\n```\n","grader":{"kind":"stdout-equality","expected":"['acme/refunds', 'acme/sla']","normalize":"collapse-trailing-newline"},"bugLines":[10,11],"revealAfter":3},{"type":"write","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s06-06-write-recall-at-k","xp":5,"hint":[{"level":1,"body":"Slice first (`retrieved[:k]`), then count members of `relevant` in that slice. The denominator is `len(relevant)` — the gold set — not k. Dividing by k would give you precision instead.","cost":0},{"level":2,"body":"Two lines: `hits = sum(1 for cid in retrieved[:k] if cid in relevant)` then `return hits / len(relevant)`. At k=1 the slice is just p1 (0 hits); at k=5, p2, p9, and p4 all land — 3 of the 4 gold chunks.","cost":0}],"personalize":false,"phase":"build","estSeconds":150,"concept":"recall-at-k-function","prompt":"Write `recall_at_k(retrieved, relevant, k)`: take the first `k`\nids from the ranked `retrieved` list, count how many are in the\n`relevant` set, and return that count divided by the size of the\nrelevant set.\n\nThe harness runs it at k=1, 3, and 5. Expected output:\n```\nrecall@1: 0.00\nrecall@3: 0.50\nrecall@5: 0.75\n```\n\nNotice the shape of those numbers: recall@k can only go up as k\ngrows. That's why \"just raise k\" always looks like a fix — until\nyou remember precision falls and the token bill climbs with it.\n","starter":"retrieved = [\"p1\", \"p2\", \"p9\", \"p5\", \"p4\"]\nrelevant = {\"p2\", \"p4\", \"p6\", \"p9\"}\n\n# define recall_at_k(retrieved, relevant, k) below\n\nprint(f\"recall@1: {recall_at_k(retrieved, relevant, 1):.2f}\")\nprint(f\"recall@3: {recall_at_k(retrieved, relevant, 3):.2f}\")\nprint(f\"recall@5: {recall_at_k(retrieved, relevant, 5):.2f}\")\n","grader":{"kind":"stdout-equality","expected":"recall@1: 0.00\nrecall@3: 0.50\nrecall@5: 0.75\n","normalize":"collapse-trailing-newline"},"solution":"retrieved = [\"p1\", \"p2\", \"p9\", \"p5\", \"p4\"]\nrelevant = {\"p2\", \"p4\", \"p6\", \"p9\"}\n\ndef recall_at_k(retrieved, relevant, k):\n hits = sum(1 for cid in retrieved[:k] if cid in relevant)\n return hits / len(relevant)\n\nprint(f\"recall@1: {recall_at_k(retrieved, relevant, 1):.2f}\")\nprint(f\"recall@3: {recall_at_k(retrieved, relevant, 3):.2f}\")\nprint(f\"recall@5: {recall_at_k(retrieved, relevant, 5):.2f}\")\n","hiddenTests":[]},{"type":"checkpoint","id":"context-and-retrieval/retrieval-metrics-and-vector-db-shape/s07-07-checkpoint","xp":8,"hint":[],"personalize":false,"phase":"check","estSeconds":240,"concept":"eval-harness-over-vector-rows","prompt":"Final drill: the whole lesson in one harness. The rows below are\nhonest vector-DB rows — id, vector, metadata payload — and the\neval set is three labeled queries. Build both halves:\n\n1. `search(q_vec, rows, tenant, k)` — filter rows to the tenant\n FIRST, rank the survivors by `dot(q_vec, row_vec)` descending,\n and return the top-k ids.\n2. `evaluate(eval_set, rows, k)` — run `search` for every case,\n then return a `(mean_recall, mrr)` tuple: recall is hits\n divided by the size of that case's `relevant` set, averaged\n over cases; MRR averages 1/rank of the first hit (0 for a\n case whose top-k has no hit).\n\nOne detail worth noticing in the data: for the first query,\n`globex/refunds` out-scores every acme row. Filter late and it\nsteals rank 1 for the wrong tenant — filter first and your\nmetrics measure what acme actually sees.\n\nExpected output:\n```\nrecall@2: 0.89\nmrr@2: 0.83\n```\n","starter":"def dot(a, b):\n return sum(x * y for x, y in zip(a, b))\n\nrows = [\n {\"id\": \"acme/refunds\", \"vec\": [0.9, 0.1, 0.0], \"meta\": {\"tenant\": \"acme\"}},\n {\"id\": \"acme/sla\", \"vec\": [0.1, 0.9, 0.1], \"meta\": {\"tenant\": \"acme\"}},\n {\"id\": \"acme/pricing\", \"vec\": [0.1, 0.2, 0.9], \"meta\": {\"tenant\": \"acme\"}},\n {\"id\": \"acme/onboarding\", \"vec\": [0.4, 0.5, 0.1], \"meta\": {\"tenant\": \"acme\"}},\n {\"id\": \"globex/refunds\", \"vec\": [0.95, 0.05, 0.0], \"meta\": {\"tenant\": \"globex\"}},\n]\n\neval_set = [\n {\"q\": [1.0, 0.0, 0.0], \"tenant\": \"acme\",\n \"relevant\": {\"acme/refunds\"}},\n {\"q\": [0.0, 1.0, 0.0], \"tenant\": \"acme\",\n \"relevant\": {\"acme/sla\", \"acme/pricing\", \"acme/onboarding\"}},\n {\"q\": [0.4, 0.5, 0.1], \"tenant\": \"acme\",\n \"relevant\": {\"acme/onboarding\"}},\n]\n\n# define search(q_vec, rows, tenant, k) below\n# define evaluate(eval_set, rows, k) below — returns (mean_recall, mrr)\n\nr, m = evaluate(eval_set, rows, k=2)\nprint(f\"recall@2: {r:.2f}\")\nprint(f\"mrr@2: {m:.2f}\")\n","grader":{"kind":"stdout-equality","expected":"recall@2: 0.89\nmrr@2: 0.83\n","normalize":"collapse-trailing-newline"},"solution":"$41","passThreshold":0.66}],"xpTotal":26}],"liveLessonCount":5,"liveStepCount":44,"xpTotal":144}},"chapter":"$1c:2:props:tree:detail","lesson":"$1c:2:props:tree:detail:lessons:0","step":"$1c:2:props:tree:detail:lessons:0:steps:1","stepIndex":1,"nav":{"tocNext":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"tocPrev":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"nextByPath":{"developer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"marketer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"designer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"customer-service":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"copywriter":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"data-analyst":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"project-manager":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"hr-specialist":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"operations":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2},"lawyer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":2}},"prevByPath":{"developer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"marketer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"designer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"customer-service":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"copywriter":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"data-analyst":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"project-manager":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"hr-specialist":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"operations":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0},"lawyer":{"chapterSlug":"context-and-retrieval","lessonSlug":"chunking-that-respects-structure","stepIndex":0}}}}]]

Chunking that respects structure — don't shred your own documents — step 2 of 9