ask/pipeline.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import type { Ask, Reply, Turn } from "./answer.js";
import { LAWS, type Law } from "./corpus.js";
import { answerQuestion } from "./pipeline.js";
import { ROWS } from "./schedule.js";

// The one property the pipeline exists to keep: what the model writes is not
// what the learner sees unless it passes the checks. Every test drives a
// model that says the wrong thing on purpose, or a stage that fails on
// purpose, and asserts what reached the screen and what the log says.

interface Call {
  question: string;
  laws: Law[];
  history: Turn[];
}

/** A model that answers from a script, and records what it was asked. */
function model(...replies: Reply[]): { ask: Ask; calls: Call[] } {
  const calls: Call[] = [];
  let next = 0;
  return {
    calls,
    ask: async (question, laws, history) => {
      calls.push({ question, laws, history });
      const reply = replies[Math.min(next, replies.length - 1)] ?? { kind: "failed", why: "unreachable" };
      next += 1;
      return reply;
    },
  };
}

const says = (answer: string, sources: string[], scope: "road-law" | "other" = "road-law"): Reply => ({
  kind: "reply",
  scope,
  answer,
  sources,
});

const HYDRANT = "how far from a fire hydrant can i park";
const ids = (laws: Law[]): string[] => laws.map((law) => law.id);

test("an answer that cites a law it was given is shown, under that law's own citation", async () => {
  const { ask, calls } = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  const { body, outcome } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS });
  assert.ok(ids(calls[0].laws).includes("MD-S01"));
  assert.equal(body.id, "answer");
  if (body.id === "answer") {
    assert.equal(body.citations[0].id, "MD-S01");
    assert.ok(body.citations[0].location.startsWith("Md. Code, Transp. § 21-1003(m)"));
    assert.equal(body.citations[0].asOf, "2026-10-05");
  }
  assert.deepEqual(outcome.sources, ["MD-S01"]);
});

test("a law nobody gave it is refused, and the log says why", async () => {
  const { ask } = model(says("Stay at least 15 feet from a hydrant.", ["MD-S99"]));
  const { body, outcome } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS });
  assert.equal(body.id, "no-match");
  assert.equal(outcome.answer, "");
  assert.equal(outcome.model, "road-law:uncited");
});

test("an invented figure is refused", async () => {
  const { ask } = model(says("Stay at least 20 feet from a hydrant.", ["MD-S01"]));
  const { outcome } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS });
  assert.equal(outcome.model, "road-law:ungrounded");
});

test("the index throwing leaves retrieval by words, and the answer is still shown", async () => {
  const { ask, calls } = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  const near = async (): Promise<string[]> => {
    throw new Error("the index is down");
  };
  const { body } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS, near });
  assert.equal(body.id, "answer");
  assert.ok(ids(calls[0].laws).includes("MD-S01"));
});

test("an id the index holds that the corpus does not is never given to the model", async () => {
  const { ask, calls } = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  const near = async (): Promise<string[]> => ["MD-GONE", "MD-S01"];
  await answerQuestion(HYDRANT, [], { ask, laws: LAWS, near });
  assert.ok(!ids(calls[0].laws).includes("MD-GONE"));
  assert.ok(ids(calls[0].laws).includes("MD-S01"));
});

test("the rephraser failing is retrieval on the question alone", async () => {
  const { ask } = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  const rephrase = async (): Promise<string[]> => {
    throw new Error("timed out");
  };
  const { body } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS, rephrase });
  assert.equal(body.id, "answer");
});

test("the reranker failing gives the model exactly what it got before the reranker existed", async () => {
  const plain = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  await answerQuestion(HYDRANT, [], { ask: plain.ask, laws: LAWS });
  const failing = model(says("Stay at least 15 feet from a hydrant.", ["MD-S01"]));
  const rerank = async (): Promise<number[]> => {
    throw new Error("the reranker is down");
  };
  await answerQuestion(HYDRANT, [], { ask: failing.ask, laws: LAWS, rerank });
  assert.deepEqual(ids(failing.calls[0].laws), ids(plain.calls[0].laws));
});

test("the reranker's choice is what the model reads first", async () => {
  const { ask, calls } = model(says("Stay 20 feet from a fire station driveway.", ["MD-S05"]));
  const rerank = async (_question: string, texts: string[]): Promise<number[]> =>
    texts.map((text) => (text.includes("fire station driveway") ? 5 : Number.NEGATIVE_INFINITY));
  const { body } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS, rerank });
  assert.equal(calls[0].laws[0].id, "MD-S05");
  assert.equal(body.id, "answer");
});

test("a chosen subsection brings its section and the schedule's row, whose figure grounds the answer", async () => {
  const { ask, calls } = model(says("You'll get a $50 ticket you can pay by mail, with no points.", ["MD-P-16-112-b"]));
  const rerank = async (_question: string, texts: string[]): Promise<number[]> =>
    texts.map((text) => (text.startsWith("(b) Each individual driving") ? 5 : Number.NEGATIVE_INFINITY));
  const { body } = await answerQuestion("what happens if i get pulled over without my license", [], {
    ask,
    laws: LAWS,
    rerank,
    penalties: ROWS,
  });
  const given = ids(calls[0].laws);
  for (const id of ["MD-C-16-112-b", "MD-C-16-112-c", "MD-C-16-112-d", "MD-C-16-112-e", "MD-P-16-112-b"]) {
    assert.ok(given.includes(id), id);
  }
  assert.equal(body.id, "answer");
  if (body.id === "answer") assert.equal(body.citations[0].kind, "penalty");
});

test("without the schedule, the same question is given the section and nothing invented", async () => {
  const { ask, calls } = model(says("You'll get a $50 ticket.", ["MD-C-16-112-b"]));
  const { outcome } = await answerQuestion("what happens if i get pulled over without my license", [], { ask, laws: LAWS });
  assert.ok(!ids(calls[0].laws).some((id) => id.startsWith("MD-P-")));
  assert.equal(outcome.model, "road-law:ungrounded");
});

test("a question outside road law gets the pipeline's own line, never the model's words", async () => {
  const { ask } = model(says("Paris.", [], "other"));
  const { body, outcome } = await answerQuestion("what is the capital of France", [], { ask, laws: LAWS });
  assert.equal(body.id, "no-match");
  if (body.id === "no-match") assert.ok(!body.pointer.includes("Paris"));
  assert.equal(outcome.model, "scope:other");
});

test("a model outage is named, not answered", async () => {
  const { ask } = model();
  const { body, outcome } = await answerQuestion(HYDRANT, [], { ask, laws: LAWS });
  assert.equal(body.id, "no-match");
  assert.equal(outcome.model, "unreachable");
});

test("the thread reaches the model as the learner saw it", async () => {
  const { ask, calls } = model(says("Yes, 15 feet.", ["MD-S01"]));
  const history: Turn[] = [{ question: HYDRANT, answer: "Stay at least 15 feet from a hydrant.", sources: ["MD-S01"] }];
  await answerQuestion("is that measured from the curb", history, { ask, laws: LAWS });
  assert.deepEqual(calls[0].history, history);
});