ask/eval.ts

import type { Reply } from "./answer.js";
import { CASES, type Case } from "./eval-cases.js";
import { livePipeline } from "./live.js";
import { answerQuestion } from "./pipeline.js";
import { registerMisses } from "./register.js";
import { requireApiKey } from "./setup.js";

// How well the box answers, measured against the live model.
//
// Not part of the test suite, deliberately: it costs money and is not
// deterministic. The tests drive a model that answers from a script and
// everything around it; this is the one thing that measures the model
// itself, run by hand before a prompt or model change, and its table goes in
// the pull request. Every answer goes through the pipeline's own retrieval
// and checks, so an answer that would have been refused on the wire is a
// miss here even if its words were right: nobody would have read it.

requireApiKey();

function misses(
  found: { scope: string; shown: string; locations: string[]; ids: string[] },
  expected: Case["expect"],
): string[] {
  const out: string[] = [];
  if (expected.citesLaw !== undefined && !found.locations.some((location) => location.includes(expected.citesLaw!))) {
    out.push(`does not cite ${expected.citesLaw}`);
  }
  if (expected.citesId !== undefined && !found.ids.some((id) => id.startsWith(expected.citesId!))) {
    out.push(`does not cite ${expected.citesId}`);
  }
  if (expected.unanswered === true && found.shown !== "") out.push("answered");
  if (found.scope !== expected.scope) out.push(`scope ${found.scope}, wanted ${expected.scope}`);
  if (expected.lead !== undefined) {
    const first = found.shown.trim().split(/[\s.,!:]+/)[0]?.toLowerCase() ?? "";
    if (first !== expected.lead) out.push(`leads with "${first}", wanted "${expected.lead}"`);
  }
  for (const needle of expected.contains ?? []) {
    const any = Array.isArray(needle) ? needle : [needle];
    if (!any.some((each) => found.shown.toLowerCase().includes(each.toLowerCase()))) {
      out.push(`missing ${any.map((each) => `"${each}"`).join(" or ")}`);
    }
  }
  for (const needle of expected.notContains ?? []) {
    if (found.shown.toLowerCase().includes(needle.toLowerCase())) out.push(`contains "${needle}"`);
  }
  return out;
}

async function main(): Promise<number> {
  const argv = process.argv.slice(2);
  const only = argv.includes("--case") ? Number(argv[argv.indexOf("--case") + 1]) : undefined;
  const cases = only === undefined ? CASES : CASES.filter((_, index) => index + 1 === only);
  // The whole run is asked for by name: every case is a live model call.
  if (only === undefined && !argv.includes("--all")) {
    console.error(`${CASES.length} cases, every one a live model call. Iterate with --case N; run the whole set with --all.`);
    return 2;
  }
  const live = await livePipeline();
  const cell = (text: string): string => text.replace(/\|/g, "\\|").replace(/\s+/g, " ");
  console.log("| # | question | scope | shown | verdict |");
  console.log("|---|---|---|---|---|");
  let failed = 0;
  for (const [index, kase] of cases.entries()) {
    // What the model said, kept beside what the learner was shown: "refused"
    // alone does not say which figure, and the next prompt change needs to
    // know. A holder rather than a `let`: the model is called inside the closure.
    const seen: { said: Reply } = { said: { kind: "failed", why: "unreachable" } };
    const { body, outcome } = await answerQuestion(kase.question, [], {
      ...live,
      ask: async (question, laws, history) => (seen.said = await live.ask(question, laws, history)),
    });
    const said = seen.said;
    const scope = said.kind === "reply" ? said.scope : "other";
    const shown = body.id === "answer" ? body.answer : "";
    const locations = body.id === "answer" ? body.citations.map((citation) => citation.location) : [];
    const ids = body.id === "answer" ? body.citations.map((citation) => citation.id) : [];
    const wrong = [
      ...misses({ scope, shown, locations, ids }, kase.expect),
      ...registerMisses({ question: kase.question, shown }),
    ];
    const wanted = kase.expect.scope === "road-law" && kase.expect.unanswered !== true;
    if (shown === "" && wanted) {
      const wrote = said.kind === "reply" && said.answer !== "" ? ` (wrote: ${said.answer})` : "";
      wrong.push(`refused: ${outcome.model}${wrote}`);
    }
    if (wrong.length > 0) failed += 1;
    console.log(`| ${index + 1} | ${cell(kase.question)} | ${scope} | ${cell(shown)} | ${wrong.length === 0 ? "ok" : cell(wrong.join("; "))} |`);
  }
  console.log(`\n${cases.length - failed} of ${cases.length} right.`);
  return failed === 0 ? 0 : 1;
}

main().then(
  (code) => process.exit(code),
  (error: unknown) => {
    console.error(error instanceof Error ? error.message : String(error));
    process.exit(2);
  },
);