{
  "title": "SEC filing demo: evidence and open questions",
  "review_date": "2026-09-14",
  "audited_revision": "ce4782cd78656fda50e27453b585b40ef1034fec",
  "verdict": "A bounded, inspectable demo with numerical answer binding and explicit remaining limits.",
  "scope": "Five Tesla PDFs, six supported financial metrics, 85 cached observations and 949 searchable passage windows across 305 pages. A small corpus is allowed by the assignment. Every question deserves an honest answer; every enterprise capability does not need to be implemented for this take-home.",
  "model": "Default: gpt-5.6-terra, reasoning none, Responses API with our tool loop. Thinking: gpt-6-astra, reasoning medium, service_tier fast, cloud Agents API. Fast is a service tier, not a reduction in reasoning effort or a guarantee about end-to-end time.",
  "priorities": [
    {
      "id": "P1",
      "title": "Independent and held-out evaluation",
      "finding": "The implementation and source review were performed by the assistant. Current measured results are narrow and do not establish independent ground truth.",
      "change": "Have a person review the source key and add unfamiliar phrasing, layouts and failure cases.",
      "done": "Signed source anchors and published exact-identity/coverage results on held-out questions."
    },
    {
      "id": "P2",
      "title": "Question meaning and narrative coverage",
      "finding": "Explicit numerical requests have deterministic admission; broad narrative entailment and unsupported financial identities do not.",
      "change": "Expose source-only narrative fallbacks, clarify ambiguity, and extend grammar only with reviewed examples.",
      "done": "Useful answers with no unsupported identity substitutions and measured refusal rates."
    },
    {
      "id": "P3",
      "title": "Unfamiliar layouts and operational evidence",
      "finding": "An image-only statement causes native-text extraction to abstain. Public Thinking still encountered managed-function HTTP 424; it answered by reading the original PDF. The transport cause remains unresolved.",
      "change": "Use the controlled scan for an OCR/Docling experiment and monitor real recovery failures.",
      "done": "Higher coverage at fixed observed precision, and durable recovery evidence for actual failure types."
    }
  ],
  "criteria": [
    {
      "name": "Applied reasoning and design judgment",
      "status": "Demonstrated in scope",
      "evidence": "Quarter/H1 exclusion, exact neighboring labels, PDF-first sources and deterministic arithmetic answer concrete failure risks.",
      "gap": "Explain the simpler alternative and when added complexity is worth its cost. More tools is not itself a better decision."
    },
    {
      "name": "Engineering ownership and traceability",
      "status": "Demonstrated with gaps",
      "evidence": "Bound numerical answers, source links, four-operand margins and named validation receipts are implemented; historical records are preserved.",
      "gap": "Full deployed failure coverage and old conversations with incomplete receipts remain limited."
    },
    {
      "name": "Evaluation and trust",
      "status": "Partial",
      "evidence": "Current twelve-case evaluation, historical runs, controlled answer mutations and an image-only extraction fixture are inspectable.",
      "gap": "Independent human sign-off and broad held-out accuracy remain open."
    },
    {
      "name": "Adaptability and hands-on skill",
      "status": "Not demonstrated by a finished app",
      "evidence": "Named parser, admission, arithmetic, transport and UI stages make fault localization possible.",
      "gap": "A live unfamiliar change still needs rehearsal: predict, localize, implement, test. A document or an agent-written test cannot prove the candidate can do this."
    },
    {
      "name": "Communication and product intuition",
      "status": "Partial",
      "evidence": "Practical shared UI, readable evidence and explicit five-document limits support analyst review.",
      "gap": "Time saved, reviewer comprehension and user demand have not been measured. Rehearse a concise eight-minute walkthrough."
    },
    {
      "name": "AI leverage and forward ownership",
      "status": "Demonstrated with gaps",
      "evidence": "Hosted model infrastructure, pdfplumber, Decimal, MiniSearch, jsdiff and the reused chat UI reduce custom machinery.",
      "gap": "Our responsibility remains the contracts, evaluation and failure recovery. No measured benefit yet from adding Docling or more model calls."
    }
  ],
  "questions": [
    {
      "id": "Q01",
      "group": "Choose the problem",
      "question": "Who is this for, and what job does it do?",
      "status": "Scoped answer",
      "answer": "An analyst reviewing a small known filing pack, checking reported values, doing simple comparisons and following citations. It is an evidence-assisted reading tool with human review.",
      "proof": "Public shared filing conversations and source/calculation receipts.",
      "limit": "Not autonomous financial advice or an enterprise archive search product. Analyst time savings have not been measured.",
      "next": "Observe an analyst answering the same tasks with and without the tool; measure correct completion and time spent verifying."
    },
    {
      "id": "Q02",
      "group": "Choose the problem",
      "question": "Why this scope and architecture?",
      "status": "Demonstrated",
      "answer": "The assignment allows a small PDF sample. Five filings make it possible to inspect exact table identities and expose difficult period and label choices. The model chooses useful tools; code owns repeatable checks.",
      "proof": "85 cached facts, six metrics, PDF page hashes, query_facts and final validation.",
      "limit": "The fixed layout and issuer make this easier than the full archive. A direct PDF-and-LLM baseline is simpler but gives fewer inspectable admission controls.",
      "next": "Expand only after recording which new layouts and question types fail, then compare benefit against complexity."
    },
    {
      "id": "Q03",
      "group": "Choose the problem",
      "question": "Are the PDFs really the primary data source?",
      "status": "Demonstrated",
      "answer": "Yes. Answers use the fixed source PDFs, prepared text, page images and coordinate-derived facts. Live XBRL does not supply the answer path.",
      "proof": "Public PDF links, fixed manifest/hashes and data/table-cache.json.",
      "limit": "A hash proves file identity, not that a filing is authoritative or the latest amendment.",
      "next": "Add filing lineage and amendment policy before supporting a changing archive."
    },
    {
      "id": "Q04",
      "group": "Understand the machine",
      "question": "Which retrieval mechanism do we use, and why?",
      "status": "Demonstrated",
      "answer": "Numerical lookup uses coordinate-based parsing and exact metric/period admission. Narrative discovery uses MiniSearch BM25+ lexical ranking over prepared text windows. jsdiff compares the selected passages' wording.",
      "proof": "query_facts selected/excluded table; search_passages matched terms; compare_passages original windows.",
      "limit": "Lexical search can miss paraphrases. A text diff does not establish comparable periods or causal meaning. The cache supports known layouts.",
      "next": "Measure retrieval misses separately from table extraction and answer interpretation errors."
    },
    {
      "id": "Q05",
      "group": "Understand the machine",
      "question": "What are embeddings good and bad at?",
      "status": "Deliberately unused",
      "answer": "They are useful candidates for topical or paraphrase retrieval. They do not establish exact magnitude, table position, reporting duration or whether two similar labels mean the same financial measure.",
      "proof": "No embeddings in the deployed path. Exact checks distinguish Net income from Net income attributable to common stockholders.",
      "limit": "Not using embeddings does not make all interpretation correct. A future semantic hit must still be grounded in the original source.",
      "next": "Add hybrid lexical/semantic retrieval only if a labeled passage-retrieval evaluation shows useful recall gains without relaxing fact admission."
    },
    {
      "id": "Q06",
      "group": "Understand the machine",
      "question": "How does a specific number travel from PDF to answer?",
      "status": "Bounded implementation",
      "answer": "PDF page → exact printed row and reporting column → admitted fact → Decimal when needed → mandatory validation → server-rendered numerical answer. The headline label, signed value, scale and period come from checked data, not free-form model prose.",
      "proof": "The same page also contains 41,831 for six months. The request gate excludes that valid but ineligible observation.",
      "limit": "The parser and question grammar remain narrow. Old saved answers retain their historical status.",
      "next": "Extend only to independently reviewed identities and question forms."
    },
    {
      "id": "Q07",
      "group": "Understand the machine",
      "question": "Where is the LLM, and where is ordinary code?",
      "status": "Demonstrated",
      "answer": "The LLM interprets language, selects pages/tools, proposes structured evidence and writes interpretation. Code extracts/indexes PDFs, admits supported facts, calculates and checks the structured report. A visual reader is another model call. The server, rather than the model, renders numerical conclusions from checked data.",
      "proof": "lib/model-config.ts, standard-provider.ts, provider.ts, request-admission.ts and report.ts.",
      "limit": "Standard uses our Responses tool loop; only Thinking uses the hosted Agents loop. The model still controls unsupported semantic interpretation.",
      "next": "Use typed request and answer contracts with explicit coverage, rather than making a model's paraphrase the sole authority."
    },
    {
      "id": "Q08",
      "group": "Understand the machine",
      "question": "Should preprocessing be automatic or chosen by the agent?",
      "status": "Demonstrated",
      "answer": "Run inexpensive extraction, page rendering, hashes and indexing once per document version. At question time, let the coordinator choose relevant pages, fact queries and optional checks. Final structured validation runs regardless of tool selection.",
      "proof": "Prepared cache/index versus per-turn tool ledger.",
      "limit": "Re-running extraction on every request wastes latency and can create inconsistent versions. Optional checks only establish what they actually examined.",
      "next": "Use document/parser version keys and ingestion status when the archive grows; trigger expensive OCR or alternative parsing from failures."
    },
    {
      "id": "Q09",
      "group": "Understand the machine",
      "question": "Does arithmetic happen in the LLM or deterministic code?",
      "status": "Demonstrated",
      "answer": "Standard uses decimal.js; Thinking can use the Python Decimal helper. The backend recomputes every final calculation. Margin change uses four operands and subtracts unrounded ratios, in percentage points.",
      "proof": "Calculation receipts and lib/report.ts; operand periods, units and basis are checked.",
      "limit": "A compatible calculation does not establish source authority. There is no universal amount size where LLM arithmetic becomes unsafe; row selection, sign, scale and rounding are separate failure risks.",
      "next": "Review operands and exact identities as well as the final result."
    },
    {
      "id": "Q10",
      "group": "Understand the machine",
      "question": "Why not put everything in SQL or use an MCP server?",
      "status": "Deliberate tradeoff",
      "answer": "The original prototype used SQLite/FTS5. The deployed app serves a small canonical fact cache and MiniSearch index; D1 stores shared conversations and execution records. Ordinary typed functions expose these capabilities without another MCP service.",
      "proof": "data/table-cache.json, data/passage-index.json, lib/store.ts and function schemas.",
      "limit": "SQL improves querying, indexing and integrity constraints; it does not infer financial meaning. This app does not expose unrestricted SQL or use SQL as numeric truth.",
      "next": "At archive scale use structured storage and constrained parameterized queries behind the same admission contract."
    },
    {
      "id": "Q11",
      "group": "Defend the evidence",
      "question": "Where does validation happen?",
      "status": "Partial",
      "answer": "Checks run at preparation, query admission, calculation and final publication. The final checklist names the actual passage, label, header, signed-token, period, exact-cache, operand, Decimal, contradiction, question and answer-binding checks. Inapplicable and unexamined checks are explicit.",
      "proof": "Expandable per-answer checklist with expected/observed comparisons and source links; offline control artifacts.",
      "limit": "The checklist proves only those comparisons. It does not verify narrative entailment, issuer authority, all accounting context or unknown layouts.",
      "next": "Use failures and unexamined fields to choose the next bounded check."
    },
    {
      "id": "Q12",
      "group": "Defend the evidence",
      "question": "How do we prevent the near-label and wrong-column mistakes?",
      "status": "Demonstrated in scope",
      "answer": "Require exact printed labels, metric identity, start/end dates, duration, units and eligible source coordinates. Return excluded candidates with reasons instead of selecting the nearest-looking amount.",
      "proof": "Net income 420 versus common-stockholder income 409 in Q1 2025; Q2 revenue 22,496 versus H1 41,831.",
      "limit": "Unknown layouts and unsupported financial identities cannot earn the exact-cache or question-admission guarantee. Numerical claims outside the grammar require review; a narrative passage can still be read as a source excerpt.",
      "next": "Add held-out paraphrase/negation tests and an explicit unverified state for unsupported layouts."
    },
    {
      "id": "Q13",
      "group": "Defend the evidence",
      "question": "Do multiple reading methods agreeing prove correctness?",
      "status": "Bounded check",
      "answer": "No. The visual tool separately reads the unmarked full-page image without candidate answers. We compare recorded label, value, sign, scale and period fields afterward. Agreement is useful evidence of consistency; disagreement triggers inspection.",
      "proof": "Blind-input contract, field-level output and the historical focused visual run.",
      "limit": "Same-family model errors can correlate. Currency, accounting basis and unsupported context remain unverified. Two parsers can agree on a wrong semantic interpretation.",
      "next": "Use independently reviewed source anchors and deliberate counterexamples, not a majority-vote badge."
    },
    {
      "id": "Q14",
      "group": "Defend the evidence",
      "question": "What does reconciliation really check?",
      "status": "Narrow implemented check",
      "answer": "For Tesla revenue, compare reported Q2 with H1 minus Q1 and preserve all sources, the residual and ±1.5 USD million allowance for whole-million rounding.",
      "proof": "reconcile_reporting_periods and its rounding/missing/conflicting observation controls.",
      "limit": "It is not a universal financial identity. Coordinated errors can still balance, and the equation cannot decide which conflicting source is authoritative.",
      "next": "Treat each further accounting relationship as a scoped rule with explicit preconditions and independently reviewed tests."
    },
    {
      "id": "Q15",
      "group": "Defend the evidence",
      "question": "How was numerical accuracy benchmarked?",
      "status": "Current narrow evaluation",
      "answer": "The current release is evaluated on the same twelve frozen questions with exact-identity scoring, refusals, failures, tool usage and latency retained. Earlier candidate runs and the historical Astra-low comparison remain separately available.",
      "proof": "Current evaluation, prior failed candidates and source-image review sheet linked on this page. Current public checks separately preserve a correct Standard margin comparison and a correct Thinking PDF fallback with a failed managed-function card.",
      "limit": "Source anchors were re-inspected by the assistant. Independent human sign-off is still missing. The same assistant helped implement and review the system; this is not independent ground truth.",
      "next": "Have a person sign the source key and add held-out questions before making broad accuracy claims."
    },
    {
      "id": "Q16",
      "group": "Defend the evidence",
      "question": "What exactly counts as correct?",
      "status": "Defined with gaps",
      "answer": "Correct numerical evidence means the right filing/page, exact label, metric, signed amount, units, basis and reporting interval. Wrong period is wrong even when the amount appears on the page. The comparison scorer uses 1e-8 percentage-point tolerance for computed results.",
      "proof": "Frozen cases and field-level comparison scoring.",
      "limit": "The runtime accepts a model receipt within 0.005, then replaces it with the canonical Decimal result. Evaluation tolerance is 1e-8. The ±1.5 million reconciliation allowance is separate. Bound numerical prose and source-only narrative excerpts have different guarantees.",
      "next": "Score exact facts, displayed claims, coverage and appropriate abstention separately."
    },
    {
      "id": "Q17",
      "group": "Defend the evidence",
      "question": "How do ambiguity and missing data behave?",
      "status": "Partial",
      "answer": "Ambiguous profit/period requests seek clarification. Missing covered facts are not replaced by neighboring values. Missing company/period coverage is stated. Conflicting implicated observations require review.",
      "proof": "Admission tests and frozen ambiguous/missing-company/missing-period cases.",
      "limit": "Segment, cash-flow, non-GAAP and narrative meaning remain outside numerical admission. Equal-duration gross/operating margin comparisons now use four admitted operands. Amendment and accounting-context policies remain incomplete.",
      "next": "Use the explicit coverage boundary; do not substitute another metric to force an answer."
    },
    {
      "id": "Q18",
      "group": "Defend the evidence",
      "question": "How do we reduce hallucination, and where can it still fail?",
      "status": "Partial",
      "answer": "Validated facts and Decimal results now generate numerical prose directly. Fabricated passages, wrong cells and valid H1 facts substituted for a Q2 request remain rejected. Numerical narrative prose without admitted facts falls back to literal source excerpts.",
      "proof": "Correct and rejected controls in this audit; source and calculation traces in the public demo.",
      "limit": "Source selection, narrative meaning, source authority and unknown layouts can still be wrong. A quoted amount in a narrative excerpt is not a validated financial answer.",
      "next": "Prioritize independent and held-out evaluation, then unfamiliar layouts."
    },
    {
      "id": "Q19",
      "group": "Handle surprise",
      "question": "What should happen when a parser or provider fails?",
      "status": "Partial; live failure retained",
      "answer": "Execution and result-delivery records are durable. Replays reuse the exact payload and idempotency key; confirmed deliveries are skipped. A visual request with uncertain execution is not automatically repeated.",
      "proof": "D1 tests cover lost acknowledgements, reloads, duplicates and cancellation. A real hosted run completed after an intentionally interrupted tool-result delivery.",
      "limit": "The earlier provider HTTP 424 has not been causally explained. A successful controlled recovery does not guarantee every upstream failure is recoverable, and billing totals were not independently reconciled.",
      "next": "Monitor actual provider failures and retain their receipts; broaden production failure tests only where evidence justifies it."
    },
    {
      "id": "Q20",
      "group": "Handle surprise",
      "question": "Can you handle an unfamiliar live change?",
      "status": "Needs rehearsal",
      "answer": "Predict the behavior before running it. Locate whether the fault belongs to question interpretation, source extraction, fact admission, arithmetic, transport or rendering. Change the smallest contract and run the new case plus relevant regressions.",
      "proof": "This audit provides a concrete example: unchanged correct evidence plus altered prose isolates the failure after structured validation.",
      "limit": "Agent-created code and a polished demo do not establish the candidate's hands-on understanding.",
      "next": "Rehearse adding a new supported label and diagnosing a changed table header without using a memorized script."
    },
    {
      "id": "Q21",
      "group": "Direct the next step",
      "question": "What is missing before enterprise deployment?",
      "status": "Explicitly out of scope",
      "answer": "Independent and broader evaluation, narrative/semantic validation, filing and amendment policies, issuer/layout coverage, tenant access controls, retention/deletion, monitoring, operational SLOs and enforceable spending controls.",
      "proof": "The current public shared app intentionally lacks enterprise tenant separation; request caps and execution logs are implemented.",
      "limit": "Five questions per visitor and twenty per site are request limits, not guaranteed dollar caps. Shared access is a demo choice, not an enterprise authorization model.",
      "next": "Define pilot users, allowable data, error/review policy, latency budget and stop criteria before expanding access."
    },
    {
      "id": "Q22",
      "group": "Direct the next step",
      "question": "What breaks first at 50,000 filings?",
      "status": "Reasoned forecast; unmeasured",
      "answer": "The current full-corpus attachment/prepared-bundle approach, ingestion throughput, cache memory and layout-specific assumptions become untenable. Per-question extraction and broad model context would raise cost and latency sharply.",
      "proof": "The existing design loads a tiny fixed corpus and uses a 128 MB Worker runtime. No archive-scale load test has been run.",
      "limit": "We cannot honestly rank the first observed bottleneck without a load study; several constraints could dominate.",
      "next": "Use object storage, queued versioned ingestion, a structured fact store, filtered retrieval, unsupported-layout routing and measured budgets. Preserve source IDs and the same admission contract."
    },
    {
      "id": "Q23",
      "group": "Direct the next step",
      "question": "How much did we outsource, and what do we still own?",
      "status": "Demonstrated",
      "answer": "Reuse the chat starter, OpenAI SDK/hosted infrastructure, pdfplumber, Decimal, MiniSearch, jsdiff and managed hosting/storage. We own the small adapters, source schema, admission policy, evaluation and user-visible failure semantics.",
      "proof": "Dependency manifest and named tool receipts.",
      "limit": "Packaged software reduces implementation effort, not responsibility for correctness. Extra agents or MCP wrappers would not automatically improve reliability.",
      "next": "Evaluate packages against a recorded failure and a clear acceptance test; retain attribution and pin reproducible versions."
    },
    {
      "id": "Q24",
      "group": "Direct the next step",
      "question": "Should we add Docling?",
      "status": "Researched; not integrated",
      "answer": "Use Docling as a bounded experiment on an actual extraction failure, retaining source coordinates and applying the same admission rules. The controlled image-only statement now provides that test: native-text parsing returns no facts for the same visible page.",
      "proof": "Scanned-fixture report: one correct selectable-text target, one abstention on its image-only copy, zero wrong accepted targets. This measures coverage, not a Docling gain.",
      "limit": "Docling is not installed or exposed as a live tool. No OCR or alternative-parser improvement has been measured. One accepted target is far too small for an accuracy claim.",
      "next": "Try the packaged OCR/layout pipeline on this fixture, then require correct complete identities and no wrong admissions before broader testing."
    },
    {
      "id": "Q25",
      "group": "Direct the next step",
      "question": "Should we encourage more tool calls to make the demo convincing?",
      "status": "Deliberate decision",
      "answer": "Encourage task-relevant evidence: admission and source page for a lookup, Decimal for a margin, passage search for narrative, diff for wording changes and visual/layout reading when uncertain. Show why each call was useful.",
      "proof": "The current margin smoke used four tool calls across three methods; the narrative run used five calls across three methods, including one incomplete search.",
      "limit": "Tool count is not confidence. Redundant model calls can repeat the same error while increasing cost and latency. Never label an unused or incomplete check as passed.",
      "next": "Measure useful checks, caught errors and final accuracy per added latency/cost, not the number of icons."
    }
  ],
  "walkthrough": [
    {
      "time": "0:00–1:00",
      "step": "Choose one consequential question",
      "detail": "Ask for Q2 2025 gross margin. State the analyst task and five-filing scope."
    },
    {
      "time": "1:00–3:00",
      "step": "Trace the answer",
      "detail": "Open PDF page 5. Show the exact rows and quarter columns, exclude H1, then inspect 3,878 ÷ 22,496 × 100 = 17.24%."
    },
    {
      "time": "3:00–4:30",
      "step": "Show a caught failure and the repaired boundary",
      "detail": "H1-as-Q2 evidence is rejected. A historical wrong sentence used to pass; now the same mutation renders the correct bound answer. Keep constructed controls separate from model results."
    },
    {
      "time": "4:30–6:00",
      "step": "Defend the measurements",
      "detail": "Show the current twelve-case run and its candidate failures separately. State exact identity rules and who reviewed the key."
    },
    {
      "time": "6:00–7:00",
      "step": "Explain ownership and recovery",
      "detail": "Identify the model, parser, admission, Decimal and durable ledger boundaries. Show a real incomplete tool receipt rather than hiding it."
    },
    {
      "time": "7:00–8:00",
      "step": "Make the next decision",
      "detail": "Prioritize independent evaluation, unsupported semantics and unfamiliar layouts. More tool calls are not the objective."
    }
  ],
  "sources": [
    {
      "title": "Saved public comparison and margin answers",
      "url": "https://sec-filing-workbench.vercel.app/?conversation=9e4b2609-9178-4797-9a10-ed433323ec93"
    },
    {
      "title": "Tesla Q2 2025 source PDF, page 5",
      "url": "https://sec-filing-workbench.vercel.app/filings/tsla-2025q2.pdf#page=5"
    },
    {
      "title": "OpenAI Astra model and fast-tier pricing",
      "url": "https://developers.openai.com/api/docs/models/gpt-6-astra"
    },
    {
      "title": "Docling table structure, cell matching and native pipeline",
      "url": "https://docling-project.github.io/docling/usage/advanced_options/"
    },
    {
      "title": "Docling toolkit paper",
      "url": "https://arxiv.org/abs/2501.17887"
    }
  ]
}