tests: compare pqcli with upstream in node

`just test` now fetches Progress Quest web's game logic from the public
site into $XDG_CACHE_HOME/pqcli/pq-web, pinned by SHA-256 so a change on
the site fails loudly. Nothing from upstream is committed.

tests/upstream/harness.js runs that code headless. test_upstream.py
checks the word lists, MonsterTask and the other generators from 200
generator states, and three whole games of 10000 tasks each, comparing
all state including the generator after every task. Without node or the
fetched files, these tests are skipped.

tests/test_upstream.py

@@ -0,0 +1,152 @@
+"""Compare pqcli with Progress Quest web, run headless in node.
+
+`just test` fetches upstream first; see tests/upstream/fetch.py. Without
+node or the fetched files, these tests are skipped.
+"""
+
+from typing import Any, cast
+
+import pytest
+from upstream import bridge
+
+from pqcli import random
+from pqcli.game_data import DATA_FILES, load_game_data
+from pqcli.lingo import generate_name
+from pqcli.mechanic import (
+    impressive_guy,
+    monster_task,
+    named_monster,
+    special_item,
+)
+from pqcli.random import Alea, RngState
+
+pytestmark = pytest.mark.skipif(
+    bridge.unavailable_reason() is not None,
+    reason=bridge.unavailable_reason() or "",
+)
+
+DATA = load_game_data()
+STATES = [list(Alea(f"state {i}").state()) for i in range(200)]
+
+
+def test_word_lists_match_upstream() -> None:
+    (lists,) = bridge.run_harness([{"cmd": "lists"}])
+    for name in DATA_FILES:
+        if name == "combat_verbs":
+            continue  # pqcli only; upstream always says "Executing"
+        assert bridge.encode_list(name, DATA) == lists[bridge.LISTS[name]]
+
+
+def compare_calls(
+    fn: str,
+    args: list[Any],
+    ours: Any,
+    convert: Any = lambda result: result,
+    **extra: Any,
+) -> None:
+    requests = [
+        {"cmd": "call", "fn": fn, "args": args, "rng": state, **extra}
+        for state in STATES
+    ]
+    for state, theirs in zip(STATES, bridge.run_harness(requests)):
+        rng = Alea.from_state(cast(RngState, tuple(state)))
+        with random.using(rng):
+            result = ours()
+        assert (result, list(rng.state())) == (
+            convert(theirs["result"]),
+            theirs["rng"],
+        ), f"{fn}({args}) from state {state}"
+
+
+def kill_task(level: int, monster: Any) -> tuple[str, int]:
+    task = monster_task(DATA, level, monster)
+    return task.description, task.duration
+
+
+def upstream_kill_task(level: int) -> Any:
+    # Dequeue() turns MonsterTask's result into the task like this
+    return lambda result: (
+        "Executing " + result["description"],
+        (2 * 3 * result["level"] * 1000) // level,
+    )
+
+
+@pytest.mark.parametrize("level", [1, 2, 5, 10, 20, 40, 80, 150])
+def test_monster_task_matches_upstream(level: int) -> None:
+    compare_calls(
+        "MonsterTask",
+        [level],
+        lambda: kill_task(level, None),
+        upstream_kill_task(level),
+        questmonster="",
+    )
+
+
+@pytest.mark.parametrize("level", [1, 10, 40])
+def test_monster_task_with_quest_monster_matches_upstream(
+    level: int,
+) -> None:
+    index = 25
+    monster = DATA.monsters[index]
+    compare_calls(
+        "MonsterTask",
+        [level],
+        lambda: kill_task(level, monster),
+        upstream_kill_task(level),
+        questmonster=bridge.encode_monster(monster),
+        questmonsterindex=index,
+    )
+
+
+def test_named_monster_matches_upstream() -> None:
+    compare_calls("NamedMonster", [12], lambda: named_monster(DATA, 12))
+
+
+def test_impressive_guy_matches_upstream() -> None:
+    compare_calls("ImpressiveGuy", [], lambda: impressive_guy(DATA))
+
+
+def test_special_item_matches_upstream() -> None:
+    compare_calls("SpecialItem", [], lambda: special_item(DATA))
+
+
+def test_generate_name_matches_upstream() -> None:
+    compare_calls("GenerateName", [], generate_name)
+
+
+@pytest.mark.parametrize(
+    "seed,race,class_",
+    [
+        ("alpha", "Half Orc", "Ur-Paladin"),
+        ("beta", "Land Squid", "Vermineer"),
+        ("gamma", "Talking Pony", "Mage Illusioner"),
+    ],
+)
+def test_game_matches_upstream(seed: str, race: str, class_: str) -> None:
+    # Play both from the same character and generator state, 100 ms per
+    # tick, and compare the whole state, generator included, each time a
+    # task completes.
+    tasks = 10000
+    ticks = 100 * tasks * 1000
+    spec = bridge.make_spec(seed, race, class_)
+    (theirs,) = bridge.run_harness(
+        [{"cmd": "play", "spec": spec, "ticks": ticks, "tasks": tasks}]
+    )
+    ours = bridge.play(DATA, spec, ticks, tasks)
+    assert len(ours) == len(theirs) == tasks
+    for i, (mine, upstream) in enumerate(zip(ours, theirs)):
+        if mine != upstream:
+            previous = [snap["caption"] for snap in ours[max(0, i - 5) : i]]
+            fields = {
+                key: (mine[key], upstream[key])
+                for key in mine
+                if mine[key] != upstream[key]
+            }
+            pytest.fail(
+                f"diverged at task {i} after {previous}:\n"
+                + "\n".join(
+                    f"  {key}:\n    pqcli:    {ours_value}\n"
+                    f"    upstream: {theirs_value}"
+                    for key, (ours_value, theirs_value) in fields.items()
+                )
+            )

tests/upstream/bridge.py

@@ -0,0 +1,170 @@
+"""Run upstream's JS and pqcli side by side, producing comparable data."""
+
+import datetime
+import json
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Any, cast
+
+from pqcli.config import EquipmentType, Monster, StatType
+from pqcli.game_data import GameData
+from pqcli.lingo import to_roman
+from pqcli.mechanic import (
+    Player,
+    PlotTask,
+    RegularTask,
+    Simulation,
+    Stats,
+)
+from pqcli.random import Alea, RngState
+
+from .fetch import cache_dir, is_fetched
+
+HARNESS = Path(__file__).parent / "harness.js"
+
+# pqcli data file -> upstream K.* list
+LISTS = {
+    "spells": "Spells",
+    "offense_attrib": "OffenseAttrib",
+    "defense_attrib": "DefenseAttrib",
+    "offense_bad": "OffenseBad",
+    "defense_bad": "DefenseBad",
+    "shields": "Shields",
+    "armors": "Armors",
+    "weapons": "Weapons",
+    "specials": "Specials",
+    "item_attrib": "ItemAttrib",
+    "item_ofs": "ItemOfs",
+    "boring_items": "BoringItems",
+    "monsters": "Monsters",
+    "races": "Races",
+    "classes": "Klasses",
+    "titles": "Titles",
+    "impressive_titles": "ImpressiveTitles",
+}
+
+
+def unavailable_reason() -> str | None:
+    if shutil.which("node") is None:
+        return "node is not installed"
+    if not is_fetched(cache_dir()):
+        return (
+            f"upstream is not fetched into {cache_dir()}; "
+            "run `just fetch-upstream`"
+        )
+    return None
+
+
+def run_harness(requests: list[dict[str, Any]]) -> list[Any]:
+    result = subprocess.run(
+        ["node", str(HARNESS), str(cache_dir())],
+        input=json.dumps(requests),
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return json.loads(result.stdout)
+
+
+def encode_list(name: str, data: GameData) -> list[str]:
+    """Write a pqcli word list the way upstream's config.js spells it."""
+    items: list[Any] = list(getattr(data, name))
+    if name in {"offense_attrib", "defense_attrib"}:
+        return [f"{item.name}|{item.quality:+d}" for item in items]
+    if name in {"offense_bad", "defense_bad", "shields", "armors", "weapons"}:
+        return [f"{item.name}|{item.quality}" for item in items]
+    if name == "monsters":
+        return [encode_monster(item) for item in items]
+    if name in {"races", "classes"}:
+        return [
+            item.name + "|" + ",".join(stat.value for stat in item.attr)
+            for item in items
+        ]
+    return items
+
+
+def encode_monster(monster: Monster | None) -> str:
+    if monster is None:
+        return ""
+    return f"{monster.name}|{monster.level}|{monster.item or '*'}"
+
+
+def make_spec(seed: str, race: str, class_: str) -> dict[str, Any]:
+    rng = Alea(seed)
+    stats = {}
+    for stat in ["STR", "CON", "DEX", "INT", "WIS", "CHA"]:
+        stats[stat] = 3 + sum(rng.uint32() % 6 for _ in range(3))
+    stats["HP Max"] = rng.uint32() % 8 + stats["CON"] // 6
+    stats["MP Max"] = rng.uint32() % 8 + stats["INT"] // 6
+    return {
+        "name": "Tester",
+        "race": race,
+        "class": class_,
+        "stats": stats,
+        "seed": list(rng.state()),
+    }
+
+
+def _queue_entry(task: RegularTask) -> str:
+    kind = "plot" if isinstance(task, PlotTask) else "task"
+    text = "Loading" if isinstance(task, PlotTask) else task.description
+    return f"{kind}|{task.duration // 1000}|{text}"
+
+
+def snapshot(player: Player) -> dict[str, Any]:
+    assert player.task is not None
+    bars = {
+        "ExpBar": player.exp_bar,
+        "PlotBar": player.quest_book.plot_bar,
+        "QuestBar": player.quest_book.quest_bar,
+        "EncumBar": player.inventory.encum_bar,
+        "TaskBar": player.task_bar,
+    }
+    return {
+        "caption": player.task.description,
+        "level": player.level,
+        "act": player.quest_book.act,
+        "bars": {name: [bar.position, bar.max_] for name, bar in bars.items()},
+        "stats": {stat.value: player.stats[stat] for stat in StatType},
+        "equipment": {
+            kind.value: player.equipment[kind] or "" for kind in EquipmentType
+        },
+        "gold": player.inventory.gold,
+        "inventory": [[item.name, item.quantity] for item in player.inventory],
+        "spells": [
+            [spell.name, to_roman(spell.level)] for spell in player.spell_book
+        ],
+        "quests": list(player.quest_book.quests),
+        "questmonster": encode_monster(player.quest_book.monster),
+        "queue": [_queue_entry(task) for task in player.queue],
+        "rng": list(player.rng.state()),
+    }
+
+
+def play(
+    data: GameData, spec: dict[str, Any], ticks: int, tasks: int
+) -> list[dict[str, Any]]:
+    """Play pqcli the way the harness plays upstream: 100 ms ticks."""
+    race = next(race for race in data.races if race.name == spec["race"])
+    class_ = next(c for c in data.classes if c.name == spec["class"])
+    seed = cast(RngState, tuple(spec["seed"]))
+    player = Player(
+        name=spec["name"],
+        birthday=datetime.datetime(2000, 1, 1),
+        race=race,
+        class_=class_,
+        stats=Stats({StatType(k): v for k, v in spec["stats"].items()}),
+        rng=Alea.from_state(seed),
+    )
+    simulation = Simulation(player, data)
+    simulation.tick(0)  # sets up the Prologue, as upstream's sold() does
+    snapshots = []
+    for _ in range(ticks):
+        completing = player.task_bar.done
+        simulation.tick(100)
+        if completing:
+            snapshots.append(snapshot(player))
+            if len(snapshots) >= tasks:
+                break
+    return snapshots

tests/upstream/fetch.py

@@ -0,0 +1,75 @@
+"""Fetch Progress Quest web's game logic for the upstream comparison tests.
+
+The files are downloaded from the public site into a cache directory, never
+into this repository. Their hashes are pinned, so a change on the site fails
+loudly instead of silently changing what the tests compare against.
+"""
+
+import hashlib
+import os
+import sys
+import urllib.request
+from pathlib import Path
+
+BASE_URL = "https://progressquest.com/play/"
+
+# Matches pq-web 3e9431b (2024-03-18) apart from http -> https URL edits.
+FILES = {
+    "config.js": (
+        "cc76f256057d6ddbf7010f1b94e59913b935b9980138ed5e800bdd1e5ff5ec4e"
+    ),
+    "main.js": (
+        "63926cda54b232c5e5be4e97123710ee4ab2dbd1faa213b21ddc01dab54a1ce0"
+    ),
+    "newguy.js": (
+        "ac99e6524b1e8602f571515fc24f5c177e11d17ae4c6233dc43ff3aad079c7a5"
+    ),
+}
+
+
+def cache_dir() -> Path:
+    if "PQ_WEB_DIR" in os.environ:
+        return Path(os.environ["PQ_WEB_DIR"])
+    base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
+    return Path(base) / "pqcli" / "pq-web"
+
+
+def _sha256(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def is_fetched(directory: Path) -> bool:
+    return all(
+        (directory / name).is_file()
+        and _sha256((directory / name).read_bytes()) == digest
+        for name, digest in FILES.items()
+    )
+
+
+def fetch(directory: Path) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    for name, digest in FILES.items():
+        path = directory / name
+        if path.is_file() and _sha256(path.read_bytes()) == digest:
+            continue
+        url = BASE_URL + name
+        with urllib.request.urlopen(url, timeout=30) as response:
+            data = response.read()
+        actual = _sha256(data)
+        if actual != digest:
+            raise SystemExit(
+                f"{url} changed upstream: expected sha256 {digest}, "
+                f"got {actual}. Review the change, then update FILES in "
+                f"{__file__}."
+            )
+        path.write_bytes(data)
+
+
+if __name__ == "__main__":
+    target = cache_dir()
+    try:
+        fetch(target)
+    except OSError as ex:
+        print(f"cannot fetch upstream into {target}: {ex}", file=sys.stderr)
+        sys.exit(1)
+    print(target)

tests/upstream/harness.js

@@ -0,0 +1,154 @@
+// Runs Progress Quest web's game logic headless, for comparing with pqcli.
+//
+// Usage: node harness.js UPSTREAM_DIR < requests.json > responses.json
+//
+// The input is a JSON list of requests; the output is a JSON list with one
+// response per request. Upstream's files are loaded from UPSTREAM_DIR
+// (see fetch.py) and are not part of this repository.
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const vm = require("vm");
+
+const upstreamDir = process.argv[2];
+const store = {};
+
+// Just enough browser for upstream to run without a DOM.
+global.window = {
+  location: { href: "#" },
+  localStorage: {
+    getItem: (key) => (key in store ? store[key] : null),
+    setItem: (key, value) => {
+      store[key] = String(value);
+    },
+    removeItem: (key) => {
+      delete store[key];
+    },
+  },
+};
+Object.defineProperty(global, "navigator", {
+  value: { userAgent: "node" },
+  configurable: true,
+});
+global.document = null;
+let expectedAlert = null;
+global.alert = (message) => {
+  if (message === expectedAlert) return;
+  throw new Error("upstream alert: " + message);
+};
+// The clock Worker is replaced by explicit ticks, see play().
+global.Worker = function () {
+  this.postMessage = () => {};
+  this.addEventListener = () => {};
+};
+const $ = (selector) => (selector === "#multiplayer:checked" ? { length: 0 } : null);
+$.isFunction = (obj) => typeof obj === "function";
+$.each = (obj, callback) => {
+  if (Array.isArray(obj)) {
+    for (let i = 0; i < obj.length; i++) {
+      if (callback.call(obj[i], i, obj[i]) === false) break;
+    }
+  } else {
+    for (const key in obj) {
+      if (callback.call(obj[key], key, obj[key]) === false) break;
+    }
+  }
+  return obj;
+};
+$.ajax = () => ({ then: () => {}, done: () => {} });
+global.$ = $;
+
+for (const name of ["config.js", "main.js", "newguy.js"]) {
+  const file = path.join(upstreamDir, name);
+  vm.runInThisContext(fs.readFileSync(file, "utf8"), { filename: file });
+}
+
+let now = 0;
+global.timeGetTime = () => now;
+
+// FormCreate() tries to load the character named in the URL. There is
+// none yet, so upstream alerts and gives up, which is fine here.
+expectedAlert = "Error loading game";
+FormCreate();
+expectedAlert = null;
+
+// Without a DOM, ListBox.rows() has no table to read. WinItem picks a row
+// from it once the inventory passes 250 rows, so read the rows from game.
+Inventory.rows = () =>
+  game.Inventory.map((row) => ({ firstChild: { innerText: row[0] } }));
+
+function newGame(spec) {
+  // Mirrors the character creation form: stats.seed is the generator state
+  // the stats were rolled from, and the game continues from it.
+  stats = Object.assign({}, spec.stats, { seed: spec.seed });
+  traits = { Name: spec.name, Race: spec.race, Class: spec.class };
+  sold();
+  LoadGame(JSON.parse(store.roster)[spec.name]);
+}
+
+function snapshot() {
+  const bars = {};
+  for (const id of ["ExpBar", "PlotBar", "QuestBar", "EncumBar", "TaskBar"]) {
+    bars[id] = [game[id].position, game[id].max];
+  }
+  const statsOut = {};
+  for (const key of K.Stats) statsOut[key] = game.Stats[key];
+  return {
+    caption: game.kill.replace(/\.\.\.$/, ""),
+    level: game.Traits.Level,
+    act: game.act,
+    bars: bars,
+    stats: statsOut,
+    equipment: game.Equips,
+    gold: game.Inventory[0][1],
+    inventory: game.Inventory.slice(1),
+    spells: game.Spells,
+    quests: game.Quests,
+    questmonster: game.questmonster,
+    queue: game.queue,
+    rng: randseed(),
+  };
+}
+
+function play(request) {
+  newGame(request.spec);
+  const snapshots = [];
+  for (let i = 0; i < request.ticks; i++) {
+    const completing = TaskBar.done();
+    now += 100;
+    clock.lasttick = now - 100;
+    Timer1Timer();
+    if (completing) {
+      // copy, since game's lists keep changing after this tick
+      snapshots.push(JSON.parse(JSON.stringify(snapshot())));
+      if (snapshots.length >= request.tasks) break;
+    }
+  }
+  return snapshots;
+}
+
+function call(request) {
+  randseed(request.rng.slice());
+  if (request.questmonster !== undefined) {
+    game.questmonster = request.questmonster;
+    game.questmonsterindex = request.questmonsterindex;
+  }
+  const fn = global[request.fn];
+  const result = fn.apply(null, request.args || []);
+  return { result: result, rng: randseed() };
+}
+
+function lists() {
+  const result = {};
+  for (const key of Object.keys(K)) {
+    if (Array.isArray(K[key])) result[key] = K[key];
+  }
+  return result;
+}
+
+const handlers = { play: play, call: call, lists: lists };
+const requests = JSON.parse(fs.readFileSync(0, "utf8"));
+const responses = requests.map((request) => handlers[request.cmd](request));
+process.stdout.write(JSON.stringify(responses));