ask/check.ts

import type { Reply } from "./answer.js";
import type { Law } from "./corpus.js";
import { figuresIn } from "./figures.js";

// Whether a road-law answer may be shown, decided here and nowhere else.
//
// The model may only have written from `given`, so that is the whole of what
// it may cite: an id outside it is a law nobody gave it, which means memory,
// and a page the answer names that no cited law is on is a page it made up.
// The figures are grounded against the cited laws' rule and the source's own
// quoted words, both -- never against the page or section reference in front
// of them, because "p.66" must not ground "66 feet".

/** Three sentences in the register the rules ask for run to about four hundred characters. */
export const MAX_ANSWER_CHARS = 600;

/** Why an answer was not shown. Written to the log by name. */
export type Refusal = "not-covered" | "uncited" | "ungrounded" | "overlong";

export type Checked =
  | { ok: true; answer: string; laws: Law[] }
  | { ok: false; reason: Refusal };

/** The pages a citation names: "p.66 (printed) — '…'" → ["66"]; "pp.12-13" → ["12", "13"]. */
export function citedPages(citation: string): string[] {
  const reference = /^\s*pp?\.\s*([\d–,\s-]+)/i.exec(citation);
  return reference === null ? [] : (reference[1].match(/\d+/g) ?? []);
}

/** The pages an answer names in prose: "p.66", "page 66", "pages 12 and 13". */
export function pagesNamedIn(answer: string): string[] {
  const found: string[] = [];
  for (const match of answer.matchAll(/\b(?:pp?|pages?)\.?\s*(\d+(?:\s*(?:[-–,]|and)\s*\d+)*)/gi)) {
    found.push(...(match[1].match(/\d+/g) ?? []));
  }
  return found;
}

/** The source's own words: everything from the " — " on. A reference with no quote grounds nothing. */
export function quotedWords(citation: string): string {
  const quoted = citation.indexOf(" — ");
  return quoted < 0 ? "" : citation.slice(quoted);
}

export function checkAnswer(
  reply: Extract<Reply, { kind: "reply" }>,
  given: Law[],
  // The learner's own words. A figure the learner gave may be said back --
  // "a 45 mph road is over the 35 limit" -- without a law stating it.
  question = "",
): Checked {
  const answer = reply.answer.trim();
  const ids = [...new Set(reply.sources)];
  if (answer.length === 0 || ids.length === 0) return { ok: false, reason: "not-covered" };

  const cited: Law[] = [];
  for (const id of ids) {
    const law = given.find((each) => each.id === id);
    if (law === undefined) return { ok: false, reason: "uncited" };
    cited.push(law);
  }

  const pages = new Set(cited.flatMap((law) => citedPages(law.citation)));
  if (pagesNamedIn(answer).some((page) => !pages.has(page))) {
    return { ok: false, reason: "uncited" };
  }

  const grounded = new Set<string>();
  for (const law of cited) {
    for (const figure of figuresIn(`${law.law}\n${quotedWords(law.citation)}`)) grounded.add(figure);
  }
  for (const figure of figuresIn(question)) grounded.add(figure);
  for (const figure of figuresIn(answer)) {
    if (!grounded.has(figure)) return { ok: false, reason: "ungrounded" };
  }

  if (answer.length > MAX_ANSWER_CHARS) return { ok: false, reason: "overlong" };
  return { ok: true, answer, laws: cited };
}

// The lines the pipeline writes when nothing the model wrote is shown. They
// are ours, not the model's, and none contains a figure.
const POINTERS: Record<Refusal | "other" | "failed", string> = {
  "not-covered":
    "Your state's manual and code don't say, as far as I can find, so I won't guess about road law.",
  uncited:
    "I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
  ungrounded:
    "I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
  overlong:
    "I can't answer that from your state's manual or code in a way I can check, so I won't guess about road law.",
  other:
    "That is outside what I can answer here: I answer road law from your state's manual and code, and not much else.",
  failed: "I couldn't reach the model that writes the answers. Try again in a minute.",
};

export function pointerFor(reason: keyof typeof POINTERS): string {
  return POINTERS[reason];
}