ask/manifest.ts

import { createHash } from "node:crypto";
import type { Law } from "./corpus.js";
import { embeddingText } from "./vectors.js";

// What was embedded, as one hash: each id and the exact text embedded for it.
// The index is filled by hand, so it can fall behind the corpus silently: a
// law reworded, a statute added, and the index ranks what it was given last
// time. The text shown is never stale -- it is read from the corpus by id --
// but the ranking would be, and nothing on screen would say so. The test in
// corpus.test.ts recomputes this and compares it with `EMBEDDED`, so a corpus
// change that was not re-embedded is a red test rather than a ranking that
// quietly drifts.

export function fingerprint(laws: Law[]): string {
  const hash = createHash("sha256");
  for (const law of laws) hash.update(`${law.id}\u0000${embeddingText(law)}\u0001`);
  return hash.digest("hex");
}

/** What the index holds, recorded the last time `vectors.ts` embedded the corpus. */
export const EMBEDDED = {
  laws: 27,
  sha256: "d2c7b42856b16f720a7754a0dde455084aab3d3ccd062dcd2628b1980f95bc01",
};

/**
 * A page as the text a passage is checked against: tags dropped, entities
 * read, curly quotes made straight and every run of white space one space.
 * Nothing else is forgiven: not a digit, not a letter, not case.
 */
export function pageText(raw: string): string {
  return raw
    .replace(/<(script|style|nav|header|footer)\b[\s\S]*?<\/\1>/gi, " ")
    .replace(/<[^>]+>/g, " ")
    .replace(/&amp;/g, "&")
    .replace(/&nbsp;/g, " ")
    .replace(/[‘’]/g, "'")
    .replace(/[“”]/g, '"')
    .replace(/[\s ]+/g, " ")
    .replace(/ ([.,;:!?)])/g, "$1")
    .trim();
}

/** Whether `passage` is in `text`, starting and ending where a word does: "5 days" is in "15 days" as letters, not as words. */
export function onThePage(passage: string, text: string): boolean {
  const word = /[\p{L}\p{N}]/u;
  for (let at = text.indexOf(passage); at >= 0; at = text.indexOf(passage, at + 1)) {
    const before = at === 0 ? "" : text[at - 1];
    const after = text[at + passage.length] ?? "";
    const startsWhole = !(word.test(passage[0]) && word.test(before));
    const endsWhole = !(word.test(passage[passage.length - 1]) && word.test(after));
    if (startsWhole && endsWhole) return true;
  }
  return false;
}

/** At most this many passages a family: the model is given the page's rule, not the page. */
export const MAX_PASSAGES = 5;

// The passages a model proposed that may be stored: each a substring of the
// page as fetched. A model asked to copy a passage can paraphrase it, and
// "you must wait 10 days" stored under a page that says 15 is a learner turned
// away at the counter on the eleventh day, under the agency's own name.
export function verifiedPassages(
  proposed: readonly string[],
  page: string,
): { passages: string[]; dropped: { passage: string; reason: string }[] } {
  const text = pageText(page);
  const passages: string[] = [];
  const dropped: { passage: string; reason: string }[] = [];
  for (const each of proposed) {
    const passage = pageText(each);
    if (passage === "") continue;
    if (!onThePage(passage, text)) dropped.push({ passage, reason: "not in the fetched page" });
    else if (passages.includes(passage)) continue;
    else if (passages.length >= MAX_PASSAGES) dropped.push({ passage, reason: `past the first ${MAX_PASSAGES}` });
    else passages.push(passage);
  }
  return { passages, dropped };
}