Chapter 3 · section 3 of 6
The checker
Listing: ask/check.ts
import type { Reply } from "./answer.js";
import type { Law } from "./corpus.js";
import { figuresIn } from "./figures.js";
// Whether a road-law answer may be shown, decided here and nowhere else.
//
// The model may only have written from `given`, so that is the whole of what
// it may cite: an id outside it is a law nobody gave it, which means memory,
// and a page the answer names that no cited law is on is a page it made up.
// The figures are grounded against the cited laws' rule and the source's own
// quoted words, both -- never against the page or section reference in front
// of them, because "p.66" must not ground "66 feet".
/** Three sentences in the register the rules ask for run to about four hundred characters. */
export const MAX_ANSWER_CHARS = 600;
/** Why an answer was not shown. Written to the log by name. */
export type Refusal = "not-covered" | "uncited" | "ungrounded" | "overlong";
export type Checked =
| { ok: true; answer: string; laws: Law[] }
| { ok: false; reason: Refusal };
/** The pages a citation names: "p.66 (printed) — '…'" → ["66"]; "pp.12-13" → ["12", "13"]. */
export function citedPages(citation: string): string[] {
const reference = /^\s*pp?\.\s*([\d–,\s-]+)/i.exec(citation);
return reference === null ? [] : (reference[1].match(/\d+/g) ?? []);
}
/** The pages an answer names in prose: "p.66", "page 66", "pages 12 and 13". */
export function pagesNamedIn(answer: string): string[] {
const found: string[] = [];
for (const match of answer.matchAll(/\b(?:pp?|pages?)\.?\s*(\d+(?:\s*(?:[-–,]|and)\s*\d+)*)/gi)) {
found.push(...(match[1].match(/\d+/g) ?? []));
}
return found;
}
/** The source's own words: everything from the " — " on. A reference with no quote grounds nothing. */
export function quotedWords(citation: string): string {
const quoted = citation.indexOf(" — ");
return quoted < 0 ? "" : citation.slice(quoted);
}
export function checkAnswer(
reply: Extract<Reply, { kind: "reply" }>,
given: Law[],
// The learner's own words. A figure the learner gave may be said back --
// "a 45 mph road is over the 35 limit" -- without a law stating it.
question = "",
): Checked {
const answer = reply.answer.trim();
const ids = [...new Set(reply.sources)];
if (answer.length === 0 || ids.length === 0) return { ok: false, reason: "not-covered" };
const cited: Law[] = [];
for (const id of ids) {
const law = given.find((each) => each.id === id);
if (law === undefined) return { ok: false, reason: "uncited" };
cited.push(law);
}
const pages = new Set(cited.flatMap((law) => citedPages(law.citation)));
if (pagesNamedIn(answer).some((page) => !pages.has(page))) {
return { ok: false, reason: "uncited" };
}
const grounded = new Set<string>();
for (const law of cited) {
for (const figure of figuresIn(`${law.law}\n${quotedWords(law.citation)}`)) grounded.add(figure);
}
for (const figure of figuresIn(question)) grounded.add(figure);
for (const figure of figuresIn(answer)) {
if (!grounded.has(figure)) return { ok: false, reason: "ungrounded" };
}
if (answer.length > MAX_ANSWER_CHARS) return { ok: false, reason: "overlong" };
return { ok: true, answer, laws: cited };
}
// The lines the pipeline writes when nothing the model wrote is shown. They
// are ours, not the model's, and none contains a figure.
const POINTERS: Record<Refusal | "other" | "failed", string> = {
"not-covered":
"Your state's manual and code don't say, as far as I can find, so I won't guess about road law.",
uncited:
"I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
ungrounded:
"I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
overlong:
"I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
other:
"That is outside what I can answer here: I answer road law from your state's manual and code, and not much else.",
failed: "I couldn't reach the model that writes the answers. Try again in a minute.",
};
export function pointerFor(reason: keyof typeof POINTERS): string {
return POINTERS[reason];
}
Three details are the design.
The order is the diagnosis. The checks run in a fixed order and the first to fail names the refusal, because the first reason is the one worth reading in the log. An answer with no sources is not-covered before it is anything else; an answer that cites a law it was not given is uncited before its figures are examined. The deployed pipeline’s log is read by its refusal names. The comment on its app checker says each refusal names itself so that a day of ungrounded rows reads as a prompt problem and a day of scope:road-law rows as learners asking the wrong box; the comment on its road-law checker says an id outside the laws given is a law nobody gave it, which means memory.
The quote grounds, the reference does not. quotedWords returns the part of a citation from the dash on, and that is what the figures are collected from, together with the law’s own rule. The reference before the dash is never used. A manual citation begins p.66, and if the reference grounded figures then an answer saying “66 feet” under a law on page 66 would pass. The test file below has a case for exactly that.
The learner’s own figures may be said back. A question like can I park 20 feet from a hydrant contains a 20 that no law states, and an answer that says “20 feet is more than the 15 the law asks for” is right. The deployed check was refusing Illinois’s scooter answer two times in three for echoing the question’s 45 before this was added. It grounds nothing the learner did not write.
The pointers at the bottom are the lines the learner reads when nothing the model wrote is shown. They are the pipeline’s own words, they are the same on every refusal of their kind, and none of them contains a figure, which is what makes them safe to show without checking.