Joel Stevick

Chapter 5 · section 5 of 7

Proving it

Listing: ask/fuse.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import { LAWS } from "./corpus.js";
import { fuse, retrieve } from "./fuse.js";
import { bm25 } from "./retrieve.js";

test("a law high on both lists leads; one first on either list makes the shortlist", () => {
  assert.deepEqual(fuse([["a", "b", "c"], ["c", "a", "d"]]), ["a", "c", "b", "d"]);
  assert.deepEqual(fuse([["x", "y"], ["z"]]).slice(0, 2), ["x", "z"]);
});

test("with no index and no searches, retrieval is BM25 alone", () => {
  const question = "how far from a fire hydrant can i park";
  assert.deepEqual(retrieve(LAWS, question), bm25(LAWS, question));
});

test("a law the index found by meaning reaches the shortlist though it shares no word with the question", () => {
  const found = retrieve(LAWS, "can i leave my car right next to a fireplug", 8, ["MD-S01"]);
  assert.ok(found.some((law) => law.id === "MD-S01"));
});

test("an id the index returns that the corpus does not hold is never given", () => {
  const found = retrieve(LAWS, "how far from a fire hydrant can i park", 8, ["MD-GONE", "MD-S01"]);
  assert.ok(!found.some((law) => law.id === "MD-GONE"));
  assert.equal(found[0]?.id, "MD-S01");
});

test("a second search, in the law's words, brings its own laws onto the shortlist", () => {
  const found = retrieve(LAWS, "whats the fastest i can go in town with no sign", 8, [], [
    { search: "maximum speed limit business district", near: [] },
  ]);
  assert.ok(found.some((law) => law.id === "MD-S10"));
});

None of these tests calls the embedding service. The index’s answer is a list of ids, so a test supplies the list, and what the test proves is what the pipeline does with it: that a law found only by meaning reaches the shortlist, and that an id the corpus lacks never does. Whether the embedding model ranks well is a different question, measured differently, in Chapter 11.