ask/eval-cases.ts

import type { Scope } from "./answer.js";

// The golden set: questions as a learner would type them, each with the few
// facts a right answer must carry. The checks are partial on purpose -- not
// a reference answer to match, but the figure that makes an answer right or
// wrong, read off the source by a person before the case went in.

export interface Case {
  question: string;
  expect: {
    scope: Scope;
    /** The first word, lower-cased: "yes" or "no". */
    lead?: string;
    /** What the shown answer must say. An entry that is a list is met by any one of it. */
    contains?: (string | string[])[];
    /** What it must not say: the wrong figure an answer once carried. */
    notContains?: string[];
    /** Text a shown citation must contain, e.g. "§ 21-1003(m)". */
    citesLaw?: string;
    /** The start of an id a shown citation must have, e.g. "MD-P-16-112". */
    citesId?: string;
    /** Nothing the model wrote may be shown. */
    unanswered?: boolean;
  };
}

export const CASES: Case[] = [
  { question: "how far from a fire hydrant can i park", expect: { scope: "road-law", contains: ["15"], citesLaw: "§ 21-1003(m)" } },
  { question: "can i leave my car right next to a fireplug", expect: { scope: "road-law", contains: ["15"], citesLaw: "§ 21-1003(m)" } },
  { question: "how far before a turn do i have to signal", expect: { scope: "road-law", contains: ["100"], citesLaw: "§ 21-604(d)" } },
  { question: "how close to a railroad crossing can i park", expect: { scope: "road-law", contains: ["50"], citesLaw: "§ 21-1003(t)" } },
  { question: "when do i have to turn my headlights on", expect: { scope: "road-law", contains: [["1,000", "1000"]] } },
  { question: "how close behind someone do i switch to low beams", expect: { scope: "road-law", contains: ["300"], notContains: ["500"] } },
  { question: "whats the fastest i can go on a residential street with no sign", expect: { scope: "road-law", contains: ["30"], citesLaw: "§ 21-801.1" } },
  { question: "what happens if i get pulled over and i don't have my license with me", expect: { scope: "road-law", contains: ["50"], citesId: "MD-P-16-112" } },
  { question: "if i fail my knowledge test how soon can i retake it", expect: { scope: "road-law", contains: [["7", "seven", "next business day"]], citesId: "MD-R-test-and-retake" } },
  { question: "can i wear headphones while driving", expect: { scope: "road-law", lead: "no", citesLaw: "§ 21-1120(c)" } },
  { question: "¿a qué distancia de un hidrante puedo estacionar?", expect: { scope: "road-law", contains: ["15"], citesLaw: "§ 21-1003(m)" } },
  { question: "what is the capital of France", expect: { scope: "other", unanswered: true } },
];