{ "service": "ai-insights-for-humans", "title": "AI Insights for Humans", "description": "A working notebook of practical AI interpretability, prompting, reliability, and tooling notes for people who want evidence instead of hype.", "url": "https://ai-insights-for-humans.platphormnews.com", "generatedAt": "2026-11-11T20:39:41.067Z", "stats": { "totalEntries": 29, "kindCounts": { "tip": 10, "method": 16, "tool": 3 }, "tags": [ "evals", "interpretability", "prompting", "reliability", "tooling", "verification" ], "tagCount": 6, "readingPathCount": 4, "latestEntry": { "id": "retrieval-relevance", "kind": "method", "title": "Score retrieved context before you trust the answer", "body": "A retrieval system can hand the model a pile of documents and still miss the actual question. Score each chunk for direct relevance, answer support, and contradiction before you let the final answer inherit authority from the word 'retrieved'.", "tag": "verification", "date": "2026 · 06 · 28", "note": "Retrieved is not the same thing as relevant.", "extended": [ "RAG failures often look like model failures because the model speaks with the confidence of the context it was given. The first move is to separate retrieval quality from generation quality.", "Give each chunk three labels: directly answers, adjacent background, or contradiction. The answer should cite directly answering chunks first and name contradictions instead of smoothing them away.", "A good retrieval review is boring and mechanical. It asks whether the evidence was even in the room before debating whether the model reasoned over it correctly." ] }, "latestPublishedAt": "2026-06-28", "totalParagraphs": 74, "ahaMomentCount": 29 }, "tags": [ "evals", "interpretability", "prompting", "reliability", "tooling", "verification" ], "readingPaths": [ { "id": "prompt-reliability", "title": "Prompt reliability", "description": "Practical habits for getting less brittle model behavior before adding heavier tooling.", "entryIds": [ "schema-first-output", "semantic-diff", "short-system-prompts", "negative-space", "few-shot-format", "context-rot", "temperature", "self-consistency" ] }, { "id": "interpretability-fieldwork", "title": "Interpretability fieldwork", "description": "Ways to inspect internal signals and move from pretty explanations to causal evidence.", "entryIds": [ "logprobs", "feature-probing", "sae", "activation-patching", "logit-lens", "attention-viz" ] }, { "id": "verification-discipline", "title": "Verification discipline", "description": "Small rituals that keep AI-assisted work tied to evidence, tests, and explicit uncertainty.", "entryIds": [ "retrieval-relevance", "citation-triangulation", "confidence-buckets", "receipts", "counter-evidence", "memory-hygiene", "refusal-fixtures", "token-counter" ] }, { "id": "operator-safety", "title": "Operator safety", "description": "Patterns for agents that can act, route work, abstain, or spend extra verification budget.", "entryIds": [ "tool-dry-run", "abstention-design", "latency-budget", "model-routing", "red-team-mutants", "eval-sandwich" ] } ], "insightLabModules": [ { "id": "logprobs", "kind": "logprobs", "entryId": "logprobs", "eyebrow": "Uncertainty trace", "title": "The sentence is fluent. The token path may not be.", "aha": "A low-margin token is where the model nearly chose a different world.", "explanation": "This toy trace turns a polished answer into token confidence. The cold tokens are where a reviewer should slow down first.", "operatorMove": "Read names, dates, citations, and refusal-boundary words with the token margin visible.", "tokens": [ { "token": "The", "confidence": 0.98, "alternative": "A" }, { "token": "paper", "confidence": 0.81, "alternative": "study" }, { "token": "was", "confidence": 0.95, "alternative": "is" }, { "token": "published", "confidence": 0.72, "alternative": "released" }, { "token": "in", "confidence": 0.91, "alternative": "around" }, { "token": "2024", "confidence": 0.53, "alternative": "2023" }, { "token": "by", "confidence": 0.69, "alternative": "from" }, { "token": "OpenAI", "confidence": 0.56, "alternative": "Anthropic" } ], "stages": [ "Decoded answer", "Token margin", "Reviewer target" ] }, { "id": "receipts", "kind": "receipts", "entryId": "receipts", "eyebrow": "Evidence discipline", "title": "Claims become safer when each one has a handle.", "aha": "Unsupported prose can sound identical to supported prose until you force receipts into view.", "explanation": "The animation pairs each claim with the kind of proof a human can inspect. The point is not citation theater; it is making unsupported load-bearing text visible.", "operatorMove": "Require one receipt per important claim before accepting a summary, diagnosis, or recommendation.", "claims": [ { "claim": "Route compliance is complete.", "receipt": "Route inventory plus smoke result" }, { "claim": "The entry count is 17.", "receipt": "Derived from lib/entries.ts" }, { "claim": "Report generation is protected.", "receipt": "401 without PLATPHORM_API_KEY" } ], "stages": [ "Claim", "Receipt", "Decision" ] }, { "id": "counter-evidence", "kind": "counter-evidence", "entryId": "counter-evidence", "eyebrow": "Confidence check", "title": "Confidence is cheaper before the answer gets defended.", "aha": "Ask for the best objection early and the model often reveals the missing constraint.", "explanation": "This pattern makes the model argue against its first draft before assigning confidence, which makes the confidence score less ornamental.", "operatorMove": "Do not ask for confidence until the model has named what could falsify the answer.", "stages": [ "Draft", "Counter-evidence", "Calibrated answer" ] }, { "id": "temperature", "kind": "temperature", "entryId": "temperature", "eyebrow": "Sampling control", "title": "The creativity dial changes spread, not truth.", "aha": "High temperature can be great for options and terrible for extraction because it rewards variance.", "explanation": "The distributions below use the same candidate tokens. The only change is how much the sampler is allowed to wander.", "operatorMove": "Use low temperature when correctness has one target; raise it when diversity is the target.", "stages": [ "Extraction", "Drafting", "Brainstorming" ] }, { "id": "context-position", "kind": "context-position", "entryId": "context-rot", "eyebrow": "Prompt topology", "title": "The middle can be technically present and practically lost.", "aha": "Moving one critical fact from the middle to the edge can change whether the model uses it.", "explanation": "Long-context behavior often favors the beginning and end. This demo shows why constraints should live at the edges, not buried in filler.", "operatorMove": "Place the task contract first and repeat the highest-risk constraint at the end.", "stages": [ "Buried fact", "Edge facts", "Safer prompt" ] }, { "id": "attention-routing", "kind": "attention-routing", "entryId": "attention-viz", "eyebrow": "Routing map", "title": "Some heads act like tiny routing specialists.", "aha": "The interesting question is not whether attention is pretty; it is whether a routing pattern survives perturbation.", "explanation": "The animated lines are a toy view of heads tracking paired tokens. Real value comes from testing whether the pattern matters for behavior.", "operatorMove": "Use attention maps to form hypotheses, then perturb the prompt or run a causal check.", "stages": [ "Tokens", "Routing", "Hypothesis" ] } ], "fieldGuidePlaybooks": [ { "id": "hallucination-triage", "title": "Hallucination triage", "problem": "A model produced a confident answer and nobody knows whether the weak point is retrieval, reasoning, or source support.", "outcome": "A reviewer can identify which claims are supported, which claims need receipts, and which answer parts should be withheld.", "entryIds": [ "logprobs", "retrieval-relevance", "receipts", "citation-triangulation", "confidence-buckets" ], "steps": [ { "label": "Locate fragile spans", "action": "Highlight names, dates, numbers, citations, and low-margin tokens before reading the answer as prose.", "aha": "Hallucinations often hide where fluency is highest but token certainty is thinnest." }, { "label": "Separate retrieval from generation", "action": "Mark each retrieved chunk as direct support, background, contradiction, or irrelevant.", "aha": "Bad context can make a good model sound wrong with confidence." }, { "label": "Attach receipts", "action": "Force each important claim to point to a source, row, command, calculation, trace id, or assumption.", "aha": "The unsupported claim does not vanish; it becomes visible enough to reject." }, { "label": "Decide the confidence bucket", "action": "Assign high, medium, low, or unknown based on evidence quality and contradiction level.", "aha": "Confidence matters when it changes the next action." } ] }, { "id": "prompt-change-control", "title": "Prompt change control", "problem": "A system prompt needs improvement, but every wording edit risks changing refusal, format, or evidence behavior.", "outcome": "The prompt change ships with a semantic diff, targeted fixtures, and a clear before/during/after eval record.", "entryIds": [ "short-system-prompts", "semantic-diff", "eval-sandwich", "red-team-mutants", "refusal-fixtures" ], "steps": [ { "label": "Shrink the contract", "action": "Separate durable role, hard boundary, output contract, and escalation rule from task-specific detail.", "aha": "A smaller system prompt is not weaker; it is easier to audit." }, { "label": "Diff the meaning", "action": "List new obligations, removed constraints, softened verbs, and examples that imply new rules.", "aha": "A one-word modal change can quietly become a policy change." }, { "label": "Run the eval sandwich", "action": "Baseline before, targeted fixture during, broad regression after.", "aha": "The middle eval catches drift while the edit is still cheap." }, { "label": "Mutate the boundary", "action": "Test harmless variants that preserve intent while changing tone, order, language, and framing.", "aha": "A boundary that only catches the obvious phrasing is not a boundary." } ] }, { "id": "agent-action-gate", "title": "Agent action gate", "problem": "An agent can call tools, but the operator needs confidence before it mutates, publishes, replays, deletes, or spends.", "outcome": "Every action moves through preview, inspect, authorize, execute, and verify without hiding protected states.", "entryIds": [ "tool-dry-run", "abstention-design", "latency-budget", "model-routing", "memory-hygiene" ], "steps": [ { "label": "Preview the action", "action": "Require target ids, arguments, expected effect, refusal condition, and rollback notes.", "aha": "Permission to plan is not permission to execute; the preview is where unsafe scope becomes inspectable." }, { "label": "Respect blocked states", "action": "Return unknown, partial, protected-required, or needs-review when evidence or authorization is missing.", "aha": "A designed stop is better than a graceful guess because it preserves trust and gives the operator a next move." }, { "label": "Spend checks by risk", "action": "Wake up extra verification when impact is high, evidence is weak, or the action is protected.", "aha": "Latency can be evidence-driven instead of uniformly expensive." }, { "label": "Verify after execution", "action": "Check the actual changed state, not only the tool's success response.", "aha": "The world, not the model, gets the final vote, so post-action evidence is part of the action itself." } ] } ], "failureCaseStudies": [ { "id": "source-shaped-hallucination", "title": "The source-shaped hallucination", "setup": "A model summarizes a report and includes a link beside every claim.", "failure": "The links exist, but two citations point to documents that discuss the topic without supporting the exact sentence.", "hiddenSignal": "The unsupported claims are entity/date claims with weak token margins and broad source references.", "repair": "Use receipts per claim, then triangulate the high-impact facts with independent sources before publishing.", "aha": "A citation beside a sentence is not the same thing as support under the sentence.", "entryIds": [ "receipts", "citation-triangulation", "logprobs" ] }, { "id": "middle-context-loss", "title": "The buried constraint", "setup": "A long prompt includes a critical policy exception in the middle of retrieved context.", "failure": "The answer follows the opening task and closing format while ignoring the exception.", "hiddenSignal": "The missed fact was technically present but surrounded by lower-value context.", "repair": "Move the task contract to the opening, repeat the exception at the end, and rank retrieved chunks before generation.", "aha": "Presence in context is not the same thing as usable salience.", "entryIds": [ "context-rot", "retrieval-relevance", "schema-first-output" ] }, { "id": "tool-call-overreach", "title": "The tool call that did too much", "setup": "An agent is asked to clean a set of records and calls a bulk mutation tool.", "failure": "The selector is broader than the user intended and changes records outside the target set.", "hiddenSignal": "The model never previewed target ids or stated a rollback condition.", "repair": "Add a dry-run gate, require explicit target ids, and make protected-required a first-class state.", "aha": "Tool safety starts before execution, not after the audit log catches damage.", "entryIds": [ "tool-dry-run", "abstention-design", "latency-budget" ] }, { "id": "prompt-edit-regression", "title": "The helpful prompt edit that moved the boundary", "setup": "A prompt is edited to make answers friendlier and less formal.", "failure": "The model starts complying with requests it previously redirected because the new examples imply looser policy.", "hiddenSignal": "The text diff is small, but the semantic diff changes tone, authority, and refusal posture.", "repair": "Run a semantic diff, then use refusal fixtures and harmless mutants before shipping.", "aha": "Style examples can become behavioral instructions if you do not test the boundary.", "entryIds": [ "semantic-diff", "refusal-fixtures", "red-team-mutants" ] } ], "ahaLadder": [ { "id": "answer-to-signal", "title": "From answer to signal", "before": "Read the model's sentence and judge whether it sounds plausible.", "after": "Inspect token margins, retrieved evidence, receipts, and contradiction before judging.", "operatorQuestion": "Where would this answer break first?" }, { "id": "prompt-to-contract", "title": "From prompt to contract", "before": "Write instructions until the behavior feels described.", "after": "Define the output schema, failure states, hard boundary, and test fixtures.", "operatorQuestion": "What is the model allowed to leave blank?" }, { "id": "tool-to-action", "title": "From tool to action", "before": "Let the agent call a tool when it appears to understand the task.", "after": "Preview arguments, inspect target ids, authorize mutation, execute, then verify changed state.", "operatorQuestion": "What would stop this action before it runs?" }, { "id": "confidence-to-decision", "title": "From confidence to decision", "before": "Ask the model for a percentage and decide whether it feels high enough.", "after": "Bucket confidence by evidence quality, conflict level, and downstream impact.", "operatorQuestion": "What does this confidence label change?" } ], "routes": [ { "method": "GET", "path": "/", "status": "implemented", "public": true, "protected": false, "description": "Public AI insights notebook and reading paths." }, { "method": "GET", "path": "/api/health", "status": "implemented", "public": true, "protected": false, "description": "Platform health payload for the publication." }, { "method": "GET", "path": "/api/v1/health", "status": "implemented", "public": true, "protected": false, "description": "Versioned platform health payload." }, { "method": "GET", "path": "/api/docs", "status": "implemented", "public": true, "protected": false, "description": "Machine-readable API and route documentation." }, { "method": "GET", "path": "/api/v1/entries", "status": "implemented", "public": true, "protected": false, "description": "Read-only JSON index of notebook entries." }, { "method": "GET", "path": "/api/v1/entries/{id}", "status": "implemented", "public": true, "protected": false, "description": "Read-only JSON detail for one notebook entry." }, { "method": "POST", "path": "/api/v1/reports", "status": "implemented", "public": false, "protected": true, "description": "Protected generation of a real notebook/platform summary report." }, { "method": "GET", "path": "/api/mcp", "status": "implemented", "public": true, "protected": false, "description": "MCP metadata and JSON-RPC usage information." }, { "method": "POST", "path": "/api/mcp", "status": "implemented", "public": true, "protected": false, "description": "JSON-RPC 2.0 MCP endpoint for real notebook tools, resources, and prompts." }, { "method": "GET", "path": "/openapi.yaml", "status": "implemented", "public": true, "protected": false, "description": "OpenAPI 3.1 contract for public and protected routes." }, { "method": "GET", "path": "/llms.txt", "status": "implemented", "public": true, "protected": false, "description": "Concise LLM-readable site summary." }, { "method": "GET", "path": "/llms-full.txt", "status": "implemented", "public": true, "protected": false, "description": "Complete LLM-readable notebook and platform summary." }, { "method": "GET", "path": "/llms-index.json", "status": "implemented", "public": true, "protected": false, "description": "Structured LLM index generated from real entries." }, { "method": "GET", "path": "/robots.txt", "status": "implemented", "public": true, "protected": false, "description": "Crawler policy and sitemap pointer." }, { "method": "GET", "path": "/sitemap.xml", "status": "implemented", "public": true, "protected": false, "description": "Canonical URL sitemap for the notebook." }, { "method": "GET", "path": "/sitemap-index.xml", "status": "implemented", "public": true, "protected": false, "description": "Sitemap index for platform crawlers." }, { "method": "GET", "path": "/rss.xml", "status": "implemented", "public": true, "protected": false, "description": "RSS feed generated from notebook entries." }, { "method": "GET", "path": "/feed.xml", "status": "implemented", "public": true, "protected": false, "description": "RSS-compatible feed alias." }, { "method": "GET", "path": "/atom.xml", "status": "implemented", "public": true, "protected": false, "description": "Atom feed generated from notebook entries." }, { "method": "GET", "path": "/manifest.webmanifest", "status": "implemented", "public": true, "protected": false, "description": "Web app manifest for install and preview surfaces." }, { "method": "GET", "path": "/.well-known/mcp.json", "status": "implemented", "public": true, "protected": false, "description": "MCP capability discovery manifest." }, { "method": "GET", "path": "/.well-known/agents.json", "status": "implemented", "public": true, "protected": false, "description": "Agent-facing route and policy discovery manifest." }, { "method": "GET", "path": "/.well-known/security.txt", "status": "implemented", "public": true, "protected": false, "description": "Security contact and policy file." }, { "method": "GET", "path": "/.well-known/trust.json", "status": "implemented", "public": true, "protected": false, "description": "Trust, auth, public/protected, and exposure policy." } ], "routeCompliance": { "requiredRoutes": 20, "implementedRequiredRoutes": 20, "missingRequiredRoutes": [], "score": 1 }, "entries": [ { "id": "retrieval-relevance", "kind": "method", "kindLabel": "Method", "title": "Score retrieved context before you trust the answer", "body": "A retrieval system can hand the model a pile of documents and still miss the actual question. Score each chunk for direct relevance, answer support, and contradiction before you let the final answer inherit authority from the word 'retrieved'.", "tag": "verification", "date": "2026 · 06 · 28", "publishedAt": "2026-06-28", "note": "Retrieved is not the same thing as relevant.", "insight": { "aha": "Retrieval only proves that documents were fetched. It does not prove the fetched text answers the question.", "tryThis": "Label each retrieved chunk as direct support, background, contradiction, or irrelevant before reading the final answer.", "watchFor": [ "Answers citing adjacent background as if it were direct support", "Contradictory chunks hidden by smooth synthesis", "High source count used as a substitute for relevance" ] }, "extended": [ "RAG failures often look like model failures because the model speaks with the confidence of the context it was given. The first move is to separate retrieval quality from generation quality.", "Give each chunk three labels: directly answers, adjacent background, or contradiction. The answer should cite directly answering chunks first and name contradictions instead of smoothing them away.", "A good retrieval review is boring and mechanical. It asks whether the evidence was even in the room before debating whether the model reasoned over it correctly." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/retrieval-relevance" }, { "id": "schema-first-output", "kind": "tip", "kindLabel": "Tip", "title": "Design the output schema before the prompt", "body": "When the output has fields, types, allowed values, and failure states, the prompt stops being a prose wish and becomes an interface contract. The model has less room to improvise because the answer has a shape to satisfy.", "tag": "reliability", "date": "2026 · 06 · 24", "publishedAt": "2026-06-24", "note": null, "insight": { "aha": "A schema turns a vague prompt into an interface contract with explicit absence states.", "tryThis": "Write the JSON fields, allowed values, evidence pointers, and unknown states before drafting the prompt.", "watchFor": [ "Required fields that force the model to invent missing data", "Enums that do not include blocked, unknown, or partial", "Fields whose consumer or decision impact is unclear" ] }, "extended": [ "A schema makes ambiguity visible. If you cannot name the field, type, and allowed absence state, you probably do not know what you are asking the model to produce.", "Start with the downstream consumer. Does it need a label, a span, a confidence bucket, a source pointer, or an abstention? Write that contract first and then prompt the model to fill it.", "The aha is that schemas are not just for machines. They force human reviewers to decide what counts as complete, partial, unknown, and unsafe." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/schema-first-output" }, { "id": "eval-sandwich", "kind": "method", "kindLabel": "Method", "title": "Use the eval sandwich: before, during, after", "body": "One eval at the end is a smoke alarm, not a control system. Run a small eval before the change, a targeted eval during the change, and a regression eval after. The sandwich catches drift while the work is still cheap to fix.", "tag": "evals", "date": "2026 · 06 · 20", "publishedAt": "2026-06-20", "note": "The middle eval is where most prompt regressions get caught.", "insight": { "aha": "The most valuable eval is often the small one you run while the prompt is still being edited.", "tryThis": "Run a baseline set before the change, five targeted fixtures during the change, and a regression set after.", "watchFor": [ "Only testing the behavior you intended to improve", "Skipping refusal and edge fixtures after style changes", "Treating one green benchmark as proof of production safety" ] }, "extended": [ "Before-change evals establish the baseline you are trying not to lose. During-change evals are small and sharp: the five examples most likely to break because of the edit. After-change evals make sure the fix did not damage neighboring behavior.", "This is especially useful for system prompt work because every instruction edit has a shadow. You improve one behavior and accidentally move refusal, tone, verbosity, or citation discipline.", "Treat the eval sandwich as a change-management ritual. It is less glamorous than a benchmark, but it is the thing that keeps production behavior from drifting by inches." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/eval-sandwich" }, { "id": "tool-dry-run", "kind": "method", "kindLabel": "Method", "title": "Dry-run tool calls before letting them mutate anything", "body": "A tool call is not just an answer; it is an action. Before a model can send, delete, replay, publish, or spend, make it produce a dry-run plan with inputs, target, expected effect, and rollback path.", "tag": "tooling", "date": "2026 · 06 · 14", "publishedAt": "2026-06-14", "note": null, "insight": { "aha": "A tool call should become inspectable before it becomes irreversible.", "tryThis": "Require action preview, target ids, expected effects, refusal conditions, and rollback notes before execution.", "watchFor": [ "Tool arguments that include broad or ambiguous selectors", "Mutation plans without a verification step", "The model treating permission to plan as permission to execute" ] }, "extended": [ "Tool use changes the risk profile of a model. The model can be wrong in the world, not just wrong in text. The safe pattern is preview, inspect, authorize, execute, verify.", "The dry run should include exactly what will be called, what identifiers will be touched, which fields will change, and what success will look like. If the model cannot state those plainly, it should not run the tool.", "The best dry runs also include a refusal branch. They say what condition would stop execution and what evidence would be needed to proceed." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/tool-dry-run" }, { "id": "citation-triangulation", "kind": "method", "kindLabel": "Method", "title": "Triangulate citations instead of trusting the first source", "body": "One citation can prove that a sentence has a source-shaped object nearby. Two independent citations can show whether the claim survives contact with another account. For consequential facts, triangulation beats citation decoration.", "tag": "verification", "date": "2026 · 06 · 08", "publishedAt": "2026-06-08", "note": null, "insight": { "aha": "A second independent source often reveals whether a citation supports the claim or merely resembles it.", "tryThis": "Triangulate dates, quotes, benchmarks, allegations, and high-impact facts before presenting them as settled.", "watchFor": [ "Citations that support a nearby but different claim", "Two sources that repeat the same upstream source", "Disagreements collapsed into a false consensus" ] }, "extended": [ "Citation checking should ask three questions: does the source exist, does it support the exact claim, and would another independent source phrase the same claim the same way?", "When sources disagree, the disagreement is the content. Do not ask the model to hide the conflict behind a confident synthesis. Ask it to name the split and explain what would settle it.", "Triangulation is not needed for every sentence. Reserve it for claims that drive decisions: dates, quotes, prices, medical or legal facts, benchmarks, and allegations." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/citation-triangulation" }, { "id": "red-team-mutants", "kind": "method", "kindLabel": "Method", "title": "Red-team prompts with harmless mutants", "body": "If a safety or quality rule only works for the exact phrase you tested, you do not have a rule. Create harmless mutants that preserve intent while changing tone, language, order, and indirection.", "tag": "evals", "date": "2026 · 05 · 31", "publishedAt": "2026-05-31", "note": null, "insight": { "aha": "A boundary that only holds for one phrasing is a memorized example, not a reliable rule.", "tryThis": "Generate harmless variants that preserve intent while changing tone, order, indirection, language, and role framing.", "watchFor": [ "Safety behavior that fails when the user sounds polite", "Allowed prompts that become over-refused after hardening", "Fixtures that mutate the danger instead of just the wrapper" ] }, "extended": [ "Prompt boundaries are brittle when tests are too literal. A model may refuse the obvious bad phrasing and comply with the polite version, the translated version, or the version hidden inside a roleplay.", "Mutants make that brittleness visible without adding new dangerous content. They perturb the wrapper, not the underlying safety category.", "Use them for both sides of the boundary: prompts that should be refused and prompts that should remain allowed. Over-refusal is a product bug too." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/red-team-mutants" }, { "id": "semantic-diff", "kind": "tool", "kindLabel": "Tool", "title": "Diff the meaning, not just the text", "body": "A prompt edit can look tiny and change the whole behavior. Run a semantic diff that lists new obligations, removed constraints, softened verbs, and changed authority before shipping instruction changes.", "tag": "prompting", "date": "2026 · 05 · 18", "publishedAt": "2026-05-18", "note": "A one-word modal change can be a policy change.", "insight": { "aha": "The smallest text diff can be the biggest behavioral diff when it changes authority, scope, or obligation.", "tryThis": "Review prompt edits for new duties, removed constraints, softened verbs, and examples that imply new policy.", "watchFor": [ "May, should, and must changes hidden in wording cleanup", "Examples that contradict the written rule", "Output contracts changed without matching tests" ] }, "extended": [ "Text diff says what characters changed. Semantic diff says what the model is now being asked to believe, prioritize, refuse, or produce. That is the diff that matters for behavior.", "Look for modal verbs, scope words, exception clauses, examples that imply new rules, and output format changes. These are the places where small edits become big runtime differences.", "A useful semantic diff ends with a test plan: which fixtures should change, which must not change, and which new example proves the edit worked." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/semantic-diff" }, { "id": "latency-budget", "kind": "tip", "kindLabel": "Tip", "title": "Spend latency where uncertainty is highest", "body": "Do not add slow verification everywhere. Spend extra model calls, retrieval, self-consistency, or human review only where the first pass shows low confidence, high impact, or weak evidence.", "tag": "reliability", "date": "2026 · 05 · 09", "publishedAt": "2026-05-09", "note": null, "insight": { "aha": "Careful does not have to mean slow everywhere. Verification can wake up only where risk wakes it up.", "tryThis": "Escalate to extra checks when confidence is low, evidence is weak, impact is high, or an action is protected.", "watchFor": [ "Uniformly expensive checks on low-risk tasks", "Fast paths that ignore missing evidence", "Latency budgets without explicit escalation triggers" ] }, "extended": [ "Reliable AI systems are often cascades. Cheap checks run first; expensive checks wake up only when the situation deserves them. This keeps the product fast without pretending every answer has equal risk.", "A practical budget has triggers: low logprob margin, missing receipt, protected action, high-dollar decision, policy boundary, or user-visible publication.", "The aha is that latency can be evidence-driven. You do not need to choose between always fast and always careful." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/latency-budget" }, { "id": "model-routing", "kind": "method", "kindLabel": "Method", "title": "Route tasks by failure mode, not brand name", "body": "The best model for a job is the one whose failure mode you can tolerate. Route extraction, synthesis, coding, refusal review, and creative exploration by observed behavior, not by leaderboard aura.", "tag": "reliability", "date": "2026 · 04 · 26", "publishedAt": "2026-04-26", "note": null, "insight": { "aha": "Routing by failure mode is more useful than routing by reputation because every model breaks differently.", "tryThis": "Build a route table from observed task fixtures: extraction, synthesis, coding, refusal review, and long-context use.", "watchFor": [ "Leaderboards used as a substitute for local fixtures", "Model routes that cannot be changed without code edits", "Ignoring behavior drift after model updates" ] }, "extended": [ "Model routing is usually framed as cost or quality. The more useful frame is failure. Which model omits caveats, over-refuses, invents sources, follows schemas, or handles long contexts under pressure?", "Build small route tests around the jobs you actually run. A model that wins general reasoning may still be the wrong choice for structured extraction if it casually fills unknown fields.", "Keep routing reversible. When model behavior shifts, the route table should be an operational setting backed by evidence, not a belief system." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/model-routing" }, { "id": "memory-hygiene", "kind": "tip", "kindLabel": "Tip", "title": "Treat memory as evidence with an expiration date", "body": "Memory makes an agent feel continuous, but stale memory can be worse than no memory. Store decisions, constraints, and stable preferences; refresh facts that drift; never let memory outrank current evidence.", "tag": "verification", "date": "2026 · 04 · 18", "publishedAt": "2026-04-18", "note": null, "insight": { "aha": "Agent memory is a cache. Useful caches have scope, freshness, and invalidation rules.", "tryThis": "Store stable decisions and preferences, but re-check drift-prone facts before acting on them.", "watchFor": [ "Old status presented as current proof", "Private residue stored when only workflow shape was needed", "Memory outranking live repo, source, or user evidence" ] }, "extended": [ "Good memory is scoped. It records what the user decided, what a repo convention is, and which risks were already found. Bad memory stores private residue, stale status, or facts that should be rechecked.", "Every memory-derived claim should answer two questions: when was this learned, and how likely is it to have changed? If the answer is 'recently unstable,' verify before acting.", "The aha is that memory is not context magic. It is a cache, and caches need invalidation rules." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/memory-hygiene" }, { "id": "abstention-design", "kind": "tip", "kindLabel": "Tip", "title": "Design abstention as a first-class output", "body": "If the only valid-looking answer is a filled answer, the model will fill one. Give it a clean way to say unknown, insufficient evidence, unsafe, or needs human review, and make those states useful.", "tag": "reliability", "date": "2026 · 04 · 03", "publishedAt": "2026-04-03", "note": null, "insight": { "aha": "A model fills blanks when the product has no respected way to stop.", "tryThis": "Make unknown, partial, blocked, protected-required, and needs-review states valid outputs with next actions.", "watchFor": [ "Schemas where every field must be filled even when evidence is absent", "Refusals that leave the user without a useful next step", "Unknown states treated as errors instead of designed outcomes" ] }, "extended": [ "Abstention fails when it feels like an error. It works when it is a designed branch with a reason, missing evidence, and next action. The user should know what changed from answerable to not answerable.", "Use explicit output states: answered, partial, unknown, blocked, protected-required, and needs-review. Each state should carry what evidence exists and what evidence is missing.", "This is how you prevent graceful prose from hiding a dead end. The model can stop without leaving the user stuck." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/abstention-design" }, { "id": "confidence-buckets", "kind": "method", "kindLabel": "Method", "title": "Use confidence buckets, not fake precision", "body": "A model saying 87% confident usually means it learned that numbers sound calibrated. Buckets like high, medium, low, and unknown force confidence to be tied to evidence quality and decision impact.", "tag": "verification", "date": "2026 · 03 · 21", "publishedAt": "2026-03-21", "note": null, "insight": { "aha": "Confidence is useful when it changes what happens next, not when it looks mathematically precise.", "tryThis": "Use high, medium, low, and unknown buckets tied to evidence type, conflict level, and review action.", "watchFor": [ "Percentages that are not calibrated against historical outcomes", "Confidence labels without evidence reasons", "High confidence on claims with no direct receipt" ] }, "extended": [ "Precise confidence numbers look scientific and often are not. Unless they are tied to a calibrated model or measured historical error, they are mostly theater.", "Buckets work better in operator workflows because they can map to action: high can publish, medium can request a receipt, low can escalate, unknown can abstain.", "The bucket should cite the reason: direct source found, conflicting source found, no source found, or model-only inference. That reason matters more than the label." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/confidence-buckets" }, { "id": "counter-evidence", "kind": "method", "kindLabel": "Method", "title": "Ask for counter-evidence before confidence", "body": "A model that only explains why its answer is right will happily polish a mistake. Ask it to list the strongest evidence against its answer before it scores confidence. The missing objection is often the thing that saves you.", "tag": "reliability", "date": "2026 · 06 · 18", "publishedAt": "2026-06-18", "note": "Confidence after counter-evidence is more useful than confidence after a sales pitch.", "insight": { "aha": "The model is usually best at self-correction before it has been rewarded for sounding certain.", "tryThis": "Ask for the answer, the strongest objection, what evidence would change the answer, and then confidence.", "watchFor": [ "Objections that restate the answer instead of challenging it", "Confidence that stays high after missing evidence appears", "A final answer that ignores its own counter-evidence" ], "labId": "counter-evidence" }, "extended": [ "The useful move is not asking the model to be uncertain. The useful move is forcing it to search for the best case against itself while the answer is still provisional.", "A simple pattern: draft the answer, list disconfirming evidence, name what would change the answer, then assign confidence. The confidence number becomes attached to evidence quality instead of verbal smoothness.", "This is especially helpful for research summaries, legal-ish interpretation, data diagnosis, and anything where one missing constraint can flip the conclusion." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/counter-evidence" }, { "id": "receipts", "kind": "method", "kindLabel": "Method", "title": "Turn every important claim into a receipt", "body": "For high-stakes answers, make the model attach each claim to a source, calculation, line number, trace id, or explicit assumption. Unsupported claims do not disappear, but they become visible enough to reject.", "tag": "verification", "date": "2026 · 05 · 27", "publishedAt": "2026-05-27", "note": null, "insight": { "aha": "A citation is not decoration. It is a handle you can pull to see whether the sentence is load-bearing.", "tryThis": "Make every claim end with one receipt: source, row, command, calculation, trace id, or explicit assumption.", "watchFor": [ "Receipts that point to a broad document instead of the claim", "Numbers without denominators", "Source links that support nearby context but not the sentence itself" ], "labId": "receipts" }, "extended": [ "Models are good at moving from evidence to prose. They are also good at producing prose that looks like it came from evidence. The receipt requirement separates those two behaviors.", "The receipt can be a citation, a row count, a command output line, a test name, or a declared assumption. The format matters less than the discipline: every claim has to point somewhere.", "When the model cannot produce a receipt, that is not failure. That is the system doing its job and showing you where judgment is still needed." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/receipts" }, { "id": "short-system-prompts", "kind": "tip", "kindLabel": "Tip", "title": "Keep the system prompt short enough to audit", "body": "A giant instruction stack feels powerful until nobody can tell which rule caused the behavior. Keep the non-negotiables small, move task detail into the user prompt, and test the system prompt like code.", "tag": "prompting", "date": "2026 · 05 · 03", "publishedAt": "2026-05-03", "note": "If you cannot diff it cleanly, you cannot debug it cleanly.", "insight": { "aha": "A system prompt is product code. If it is too long to review, it is too long to trust.", "tryThis": "Rewrite the system prompt into role, hard boundary, output contract, and escalation rule.", "watchFor": [ "Repeated style instructions that fight each other", "Hidden policy changes bundled into wording edits", "Task detail that belongs in user input rather than permanent rules" ] }, "extended": [ "System prompts work best as durable operating constraints, not as a dumping ground for every possible preference. When the prompt grows without structure, the model starts resolving conflicts in ways you cannot predict.", "Split the prompt into policy, role, output contract, and refusal boundary. Then write a small regression set that exercises each section. You will catch more problems with five targeted tests than with another page of instructions." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/short-system-prompts" }, { "id": "refusal-fixtures", "kind": "method", "kindLabel": "Method", "title": "Use refusal boundaries as test fixtures", "body": "Do not only test the happy path. Save examples that should be refused, partially answered, or redirected, then run them every time you change instructions. Safety behavior drifts just like product behavior.", "tag": "evals", "date": "2026 · 04 · 12", "publishedAt": "2026-04-12", "note": null, "insight": { "aha": "Safety quality is not whether the model refuses. It is whether it refuses at the right boundary.", "tryThis": "Keep allowed, disallowed, and allowed-with-constraints prompts as fixtures and run them after prompt edits.", "watchFor": [ "Over-refusal on harmless transformed versions", "Under-refusal when the same intent is phrased politely", "Responses that refuse but fail to offer a safe next step" ] }, "extended": [ "Most prompt tests focus on whether the model can complete the intended workflow. That misses the other half of the product: whether it handles edge requests without over-refusing, leaking private context, or inventing authority it does not have.", "Build a small fixture set with three buckets: allowed, disallowed, and allowed-with-constraints. A good model response should land in the right bucket and explain the usable next step.", "This turns safety from a vibe into a regression target. When a prompt edit changes the refusal boundary, you see it immediately." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/refusal-fixtures" }, { "id": "logprobs", "kind": "method", "kindLabel": "Method", "title": "Read the logprobs, not the answer", "body": "The text a model gives you is the tip of the iceberg. Ask the API for token logprobs and you can see where it was confident versus where it basically flipped a coin. A fluent sentence built on 51% tokens is a guess wearing a suit.", "tag": "interpretability", "date": "2025 · 11 · 02", "publishedAt": "2025-11-02", "note": "Confidence ≠ correctness. But low confidence is a great place to look for hallucinations.", "insight": { "aha": "The final text is only the winning token path. The uncertainty is still visible in the runners-up.", "tryThis": "Render low-margin tokens first, especially names, dates, citations, and first tokens after a policy boundary.", "watchFor": [ "Fluent spans built from low-confidence token choices", "Nearly tied alternatives around entities", "A confident tone hiding a fragile probability margin" ], "labId": "logprobs" }, "extended": [ "Every token a model emits comes with a probability distribution over the whole vocabulary. The decoded text only shows you the winner. The logprobs show you the race.", "When you pull them back, scan for the spots where the top token barely beat the runner-up. Those are the seams. Names, dates, citations, and the first token of a refusal are where you most often find a coin-flip dressed up as a confident sentence.", "A practical loop: request the top 5 logprobs per token, render them as a heatmap over the output, and read the cold spots first. That is usually where the hallucination lives." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/logprobs" }, { "id": "think-out-loud", "kind": "tip", "kindLabel": "Tip", "title": "Make it think before it answers", "body": "Put the reasoning before the conclusion, never after. If you ask for the answer first and the explanation second, the explanation is just a story it invents to justify a choice it already made. Order matters more than wording.", "tag": "prompting", "date": "2025 · 10 · 28", "publishedAt": "2025-10-28", "note": null, "insight": { "aha": "Explanation after an answer often explains a commitment; reasoning before an answer can still change it.", "tryThis": "Put scratch work, constraints, and uncertainty checks before the final answer field.", "watchFor": [ "Post-hoc explanations that never revise the answer", "A final answer that appears before evidence is considered", "Reasoning sections that only paraphrase the conclusion" ] }, "extended": [ "Autoregressive models can only condition on what they have already written. If the answer comes first, the explanation is generated to fit a token that is already locked in. It is rationalization, not reasoning.", "Flip the order and the reasoning tokens become real scratch space the final answer can lean on. This is the entire mechanism behind chain-of-thought, and it is why 'answer, then explain' quietly underperforms." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/think-out-loud" }, { "id": "attention-viz", "kind": "tool", "kindLabel": "Tool", "title": "BertViz for watching attention move", "body": "An open-source visualizer that shows which tokens each attention head is looking at, layer by layer. You start to notice heads that only track quotation marks, or ones that follow subject-verb agreement across a whole paragraph.", "tag": "tooling", "date": "2025 · 10 · 19", "publishedAt": "2025-10-19", "note": "Most heads are boring. The interesting 5% are worth the dig.", "insight": { "aha": "Attention is not the whole mind, but it is a useful map of what information is being routed where.", "tryThis": "Inspect heads for repeated roles: bracket matching, quote tracking, subject agreement, or copied labels.", "watchFor": [ "Pretty attention maps with no behavioral test", "One dramatic head treated as the whole explanation", "Patterns that disappear after small prompt perturbations" ], "labId": "attention-routing" }, "extended": [ "BertViz draws the attention pattern as lines connecting tokens, split by head and layer. It runs in a notebook in a couple of lines and works on most Hugging Face transformer models.", "The value is not the pretty picture, it is the habit of looking. Once you have seen a head that exclusively tracks matching brackets, you stop thinking of the model as a black box and start thinking of it as a stack of small, legible mechanisms." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/attention-viz" }, { "id": "negative-space", "kind": "tip", "kindLabel": "Tip", "title": "Tell it what NOT to do, sparingly", "body": "Negative instructions are weak anchors. 'Don't mention X' often makes X more likely because you just put X in the context. Prefer describing the thing you DO want. Reserve negatives for hard safety rails, not style.", "tag": "prompting", "date": "2025 · 10 · 11", "publishedAt": "2025-10-11", "note": null, "insight": { "aha": "Negation still puts the forbidden idea into context. Attention does not forget because grammar says not to.", "tryThis": "Rewrite prohibitions as the positive shape of the output you want before adding hard safety rails.", "watchFor": [ "Forbidden words repeated in the response", "Style instructions expressed mostly as negatives", "A model fixating on the thing you tried to suppress" ] }, "extended": [ "Putting a word in the prompt raises its activation, even when the word is wrapped in 'do not'. The model does not have a clean negation operator the way code does; it has attention, and attention does not know the difference between mention and command.", "Rewrite prohibitions as positive descriptions of the target. Instead of 'do not be formal', say 'write like you are texting a friend'. Save the hard negatives for the few rails you cannot express any other way." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/negative-space" }, { "id": "feature-probing", "kind": "method", "kindLabel": "Method", "title": "Probe for a concept with a linear classifier", "body": "Freeze the model, grab the hidden activations for a pile of examples, and train a tiny linear probe to predict some property (is this text about money? is it past tense?). If a simple line separates them, the concept is already represented in there, plain as day.", "tag": "interpretability", "date": "2025 · 09 · 30", "publishedAt": "2025-09-30", "note": "Cheap, fast, and weirdly revealing.", "insight": { "aha": "If a tiny linear classifier can read a concept from hidden states, the model has already organized that signal.", "tryThis": "Probe one concept across layers and look for where accuracy first rises above a simple baseline.", "watchFor": [ "High probe accuracy mistaken for causal importance", "Labels that leak through surface wording", "Layer sweeps without a held-out validation split" ] }, "extended": [ "The setup is small: collect hidden states from a chosen layer for a labeled dataset, then fit logistic regression on top. High accuracy means the concept is linearly available at that layer.", "Sweep the probe across layers and you can watch where a concept first becomes readable. Abstract properties tend to sharpen in the middle layers, which is a clue about where the model is doing the relevant work." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/feature-probing" }, { "id": "temperature", "kind": "tip", "kindLabel": "Tip", "title": "Temperature is a creativity dial, not a quality dial", "body": "Turning it up doesn't make answers smarter, it makes them less predictable. For extraction, classification, and anything with a right answer, keep it near zero. For brainstorming, let it wander. People mix these up constantly.", "tag": "prompting", "date": "2025 · 09 · 22", "publishedAt": "2025-09-22", "note": null, "insight": { "aha": "Temperature changes variance, not intelligence. It widens the lottery; it does not improve the tickets.", "tryThis": "Set temperature near zero for extraction and scoring, then raise it only when variety is the product.", "watchFor": [ "Creative settings used for factual extraction", "One sampled answer treated as a stable result", "Quality claims that ignore variance between runs" ], "labId": "temperature" }, "extended": [ "Temperature reshapes the probability distribution before sampling. Low temperature sharpens it toward the most likely token; high temperature flattens it so unlikely tokens get a real chance.", "So the dial controls variance, not intelligence. If a task has a correct answer, variance is your enemy and you want it near zero. If you want range and surprise, turn it up and sample several times." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/temperature" }, { "id": "sae", "kind": "method", "kindLabel": "Method", "title": "Sparse autoencoders to untangle neurons", "body": "Individual neurons are polysemantic, one neuron fires for 'the Golden Gate Bridge' and also 'feeling trapped' and also semicolons. Train a sparse autoencoder on the activations and you pull those mixed signals apart into cleaner, single-meaning features.", "tag": "interpretability", "date": "2025 · 09 · 14", "publishedAt": "2025-09-14", "note": "This is roughly how the 'Golden Gate Claude' demo worked.", "insight": { "aha": "Sparse features give you named-ish knobs because they separate mixed neuron behavior into cleaner directions.", "tryThis": "Treat an SAE feature as a hypothesis, then test whether steering it changes the behavior you care about.", "watchFor": [ "Human-readable labels that overfit a few examples", "Feature steering without downstream behavior checks", "Polysemantic residue hiding inside a cleaner-looking feature" ] }, "extended": [ "The trick is over-completeness plus sparsity. You map activations into a much wider space but force only a few of those dimensions to be active at once. The pressure pushes tangled concepts apart into separate, more interpretable features.", "Once you have the features you can do surgery: amplify one and watch the model fixate on a bridge, suppress another and watch a behavior fade. It is the closest thing we have to named knobs inside the network." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/sae" }, { "id": "token-counter", "kind": "tool", "kindLabel": "Tool", "title": "Keep a tokenizer open in a tab", "body": "Half of weird model behavior is a tokenization story. ' apple' and 'apple' are different tokens. Numbers split in unintuitive ways. Pasting your prompt into a tokenizer viewer takes ten seconds and explains a surprising number of bugs.", "tag": "tooling", "date": "2025 · 09 · 03", "publishedAt": "2025-09-03", "note": null, "insight": { "aha": "The model never sees your characters. It sees chunks, spaces, merges, and weird number splits.", "tryThis": "Paste the exact prompt, including spaces and separators, into a tokenizer before debugging prompt behavior.", "watchFor": [ "Leading spaces changing label tokens", "Numbers split into surprising chunks", "Few-shot examples with inconsistent separators" ] }, "extended": [ "Leading spaces, casing, and digit grouping all change how text is split, and the model only ever sees the tokens, never your characters. A trailing space can silently sabotage a few-shot format.", "When something is off and you cannot explain it, paste the exact string into a tokenizer view before you theorize. The answer is in there more often than you would expect." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/token-counter" }, { "id": "self-consistency", "kind": "method", "kindLabel": "Method", "title": "Sample many, keep the majority", "body": "For reasoning tasks, generate the same answer five or ten times at moderate temperature and take the most common result. A single greedy answer is fragile; the consensus across samples is far more reliable. Slow, but it works.", "tag": "reliability", "date": "2025 · 08 · 25", "publishedAt": "2025-08-25", "note": null, "insight": { "aha": "Wrong reasoning paths tend to scatter; correct reasoning paths often converge.", "tryThis": "Sample several answers, normalize the final claim, and inspect disagreement before majority voting.", "watchFor": [ "Repeated wording counted as independent evidence", "Voting over outputs that all share the same missing assumption", "Compute spent on low-value questions" ] }, "extended": [ "Different samples take different reasoning paths but tend to converge on the same correct answer, while the wrong answers scatter. Voting over the final answers turns that asymmetry into accuracy.", "It costs you N times the compute, so reserve it for the questions that matter. A useful middle ground is to escalate to self-consistency only when the first answer looks low-confidence." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/self-consistency" }, { "id": "context-rot", "kind": "tip", "kindLabel": "Tip", "title": "Mind the middle of long contexts", "body": "Models attend strongly to the beginning and end of a long prompt and get foggy in the middle. Put the instructions and the most important facts at the edges. If something keeps getting ignored, it's probably buried in the soft center.", "tag": "prompting", "date": "2025 · 08 · 12", "publishedAt": "2025-08-12", "note": "Sometimes called 'lost in the middle'.", "insight": { "aha": "A fact can be present and still practically invisible if it sits in the soft middle of a long prompt.", "tryThis": "Move critical constraints to the opening contract and repeat the single highest-risk fact at the end.", "watchFor": [ "Middle facts ignored while opening and closing instructions are followed", "Retrieved context pasted without ranking", "Important constraints surrounded by low-value filler" ], "labId": "context-position" }, "extended": [ "Retrieval accuracy as a function of position tends to be U-shaped: strong at the start, strong at the end, sagging in the middle. A fact you need can be technically present and still effectively invisible.", "Design around it. Lead with the instruction, restate the critical constraint at the very end, and keep the soft middle for material you can afford to lose." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/context-rot" }, { "id": "activation-patching", "kind": "method", "kindLabel": "Method", "title": "Activation patching to find the cause", "body": "Run the model twice, once on a clean prompt and once on a corrupted one, then copy a single activation from one run into the other. If the output flips, you just found a component that carries the behavior. It's causal, not just correlational.", "tag": "interpretability", "date": "2025 · 07 · 30", "publishedAt": "2025-07-30", "note": "The closest thing in this field to a controlled experiment.", "insight": { "aha": "Correlation says a signal is present. Patching asks whether changing that signal changes the outcome.", "tryThis": "Patch one layer-position activation from a clean run into a corrupted run and track whether the answer flips.", "watchFor": [ "Large sweeps without a clean causal hypothesis", "Output flips caused by formatting rather than the target behavior", "A single patch interpreted without replication" ] }, "extended": [ "Probes tell you what is represented. Patching tells you what matters. By swapping one activation and watching the effect, you move from 'this information is present' to 'this component causes the behavior'.", "Sweep the patch across layers and positions to localize the circuit. The spots where patching flips the answer are the load-bearing ones; everything else is along for the ride." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/activation-patching" }, { "id": "few-shot-format", "kind": "tip", "kindLabel": "Tip", "title": "Your examples teach format louder than content", "body": "In few-shot prompts the model copies the shape of your examples first: the punctuation, the casing, the length, the label words. Get the format dead consistent and the task often solves itself. Sloppy examples teach sloppiness.", "tag": "prompting", "date": "2025 · 07 · 18", "publishedAt": "2025-07-18", "note": null, "insight": { "aha": "Few-shot examples teach surface form first. The model copies the pattern before it understands your intention.", "tryThis": "Make labels, punctuation, casing, separators, and example length boringly consistent.", "watchFor": [ "One example with a different label shape", "Multi-token labels competing with single-token labels", "Examples that demonstrate style drift instead of the task" ] }, "extended": [ "Few-shot learning is pattern continuation. The model is extremely sensitive to surface form, sometimes more than to the actual reasoning you intended to demonstrate.", "Make every example identical in structure, pick label words that are single clean tokens, and keep lengths similar. The more rigid the template, the more reliably the model fills in the next blank." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/few-shot-format" }, { "id": "logit-lens", "kind": "method", "kindLabel": "Method", "title": "The logit lens: decode the middle layers", "body": "Take the hidden state from an intermediate layer and run it through the model's output head early. You get the model's 'best guess so far' at that depth. Watching the prediction sharpen layer by layer is like a time-lapse of a thought forming.", "tag": "interpretability", "date": "2025 · 07 · 05", "publishedAt": "2025-07-05", "note": null, "insight": { "aha": "The answer often appears before the wording is polished. Middle layers can show the thought taking shape.", "tryThis": "Decode intermediate layers and watch when the target answer first becomes linearly readable.", "watchFor": [ "Early guesses treated as final causal proof", "Layer readouts that shift after prompt paraphrases", "Ignoring later layers that suppress unsafe or irrelevant continuations" ] }, "extended": [ "The output projection is just a linear map, so nothing stops you from applying it to an earlier layer. The result is a rough readout of what the model would say if it had to stop thinking right now.", "Often the correct answer appears in the middle layers and the later layers spend their budget refining phrasing and suppressing alternatives. Seeing that handoff demystifies a lot of model behavior." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/logit-lens" } ] }