Chapter 10 · section 2 of 5
One turn, end to end
Listing: ask/pipeline.ts
import type { Ask, Turn } from "./answer.js";
import { checkAnswer, pointerFor } from "./check.js";
import type { Law } from "./corpus.js";
import { retrieve, type Search } from "./fuse.js";
import { chooseLaws, poolSize, type Rerank } from "./rerank.js";
import type { Rephrase } from "./rephrase.js";
import { RETRIEVED_LAWS } from "./retrieve.js";
import type { PenaltyRow } from "./schedule.js";
import { pagesLast, wholeSections, withPenalties } from "./sections.js";
import { ANSWER_MODEL } from "./setup.js";
import type { NearLaws } from "./vectors.js";
// One turn, from a question to what the learner is shown and what the log
// records. Everything between the request and the screen is here, so a test
// whose model answers from a script runs the same retrieval and the same
// checks as the deployed pipeline rather than a copy of them.
export interface Pipeline {
ask: Ask;
/** The state's laws: the one file a question is answered from. */
laws: Law[];
/** The ids nearest the question in meaning. Absent, or throwing, leaves words alone. */
near?: NearLaws;
/** The question again in the law's words. Absent, or throwing, retrieval runs on the question alone. */
rephrase?: Rephrase;
/** The reranker. Absent, or throwing, the model gets retrieval's first eight. */
rerank?: Rerank;
/** The court's fine schedule. Absent, the model is given the sections without it. */
penalties?: PenaltyRow[];
}
export interface Citation {
id: string;
location: string;
kind?: Law["kind"];
asOf?: string;
}
export type Body =
| { id: "answer"; answer: string; citations: Citation[] }
| { id: "no-match"; pointer: string };
/** What the log records about a turn: what was shown, and why, if nothing was. */
export interface Outcome {
answer: string;
sources: string[];
model: string;
}
function note(what: string, error: unknown): void {
console.error(`${what}:`, error instanceof Error ? error.message : error);
}
export async function answerQuestion(
question: string,
history: Turn[],
pipeline: Pipeline,
): Promise<{ body: Body; outcome: Outcome }> {
const { laws } = pipeline;
let near: string[] = [];
if (pipeline.near !== undefined) {
try {
near = await pipeline.near(question);
} catch (error) {
note("the vector index did not answer", error);
}
}
let searches: Search[] = [];
if (pipeline.rephrase !== undefined) {
try {
const worded = await pipeline.rephrase(question);
searches = await Promise.all(
worded.map(async (search) => ({
search,
near: await (pipeline.near?.(search) ?? Promise.resolve([])).catch(() => []),
})),
);
} catch (error) {
note("the rephraser did not answer", error);
}
}
const fallback = retrieve(laws, question, RETRIEVED_LAWS, near, searches);
const pool = pipeline.rerank === undefined ? [] : retrieve(laws, question, poolSize(searches), near, searches);
const chosen = await chooseLaws(pool, fallback, question, pipeline.rerank);
const sectioned = wholeSections(chosen, laws);
const given = pagesLast(
pipeline.penalties === undefined ? sectioned : withPenalties(sectioned, pipeline.penalties, chosen),
);
const reply = await pipeline.ask(question, given, history);
if (reply.kind === "failed") {
return {
body: { id: "no-match", pointer: pointerFor("failed") },
outcome: { answer: "", sources: [], model: reply.why },
};
}
if (reply.scope !== "road-law") {
return {
body: { id: "no-match", pointer: pointerFor("other") },
outcome: { answer: "", sources: reply.sources, model: `scope:${reply.scope}` },
};
}
const checked = checkAnswer(reply, given, question);
if (!checked.ok) {
return {
body: { id: "no-match", pointer: pointerFor(checked.reason) },
outcome: { answer: "", sources: reply.sources, model: `road-law:${checked.reason}` },
};
}
return {
body: {
id: "answer",
answer: checked.answer,
citations: checked.laws.map((law) => ({
id: law.id,
location: law.citation,
...(law.kind === undefined ? {} : { kind: law.kind }),
...(law.asOf === undefined ? {} : { asOf: law.asOf }),
})),
},
outcome: { answer: checked.answer, sources: checked.laws.map((law) => law.id), model: ANSWER_MODEL },
};
} Read it top to bottom once, because it is the introduction’s diagram as code.
The index is asked, inside a try, and a throw is logged and leaves near empty. The rephraser is asked, inside a try, and its searches are each sent to the index too, with a .catch of their own so that one failed search does not lose the others. Then two retrievals: the eight-law fallback and the 24-or-32-law pool, the second only when there is a reranker to read it. The reranker chooses, or falls back. The sections are made whole, the schedule’s rows join, the pages go last. The model writes. Then four exits, and only the last one puts the model’s words on the screen.
Two things about the exits are the design.
Every exit has a name in the log. outcome.model is the model’s ID on an answer, and on anything else it is why: unreachable, refused, scope:other, road-law:uncited, road-law:ungrounded. The deployed pipeline writes that column on every row of its question table, with nationwide:topic or nationwide:index in place of the model’s ID on a Nationwide answer, and reads the table by it. A day of unreachable is an outage. A day of road-law:uncited is a prompt problem. A day of scope:other is learners asking the wrong box. The names were chosen so that the log reads as a diagnosis without a person re-reading the questions.
The model’s text reaches the screen on exactly one path. The other three return pointerFor(...), which is one of the pipeline’s own sentences from Chapter 3, and none of those sentences contains a figure. An outage, when the model cannot be reached at all, gets a sentence saying so, with a different name in the log from a refusal, because the two are different problems with different fixes.