#!/usr/bin/env node// Validates the task set without calling a model.//// For every task and language:// - reference.<lang> must compile and pass every hidden test;// - mutant.<lang>, when present, is a plausible buggy solution (the kind of// bug the task is designed to provoke) and must NOT pass: either the// compiler rejects it or at least one hidden test fails. A mutant that// passes means the hidden tests are too weak.//// node evals/validate.mjs [task-id ...]import fs from "node:fs";import path from "node:path";import { compile, runHiddenTests, evalsDir } from "./lib/compile.mjs";const tasksDir = path.join(evalsDir, "tasks");const only = process.argv.slice(2);const ids = fs .readdirSync(tasksDir) .filter((n) => fs.existsSync(path.join(tasksDir, n, "task.json"))) .filter((n) => only.length === 0 || only.includes(n)) .sort();const workRoot = path.join(evalsDir, ".work", `validate-${process.pid}`);let problems = 0;const rows = [];for (const id of ids) { const dir = path.join(tasksDir, id); const required = ["prompt.md", "signature.ts", "signature.tsus", "reference.ts", "reference.tsus", "test.mjs"]; const missing = required.filter((f) => !fs.existsSync(path.join(dir, f))); if (missing.length > 0) { console.log(`FAIL ${id}: missing ${missing.join(", ")}`); problems += 1; continue; } for (const lang of ["ts", "tsus"]) { for (const kind of ["reference", "mutant"]) { const file = path.join(dir, `${kind}.${lang}`); if (!fs.existsSync(file)) { continue; } const workDir = path.join(workRoot, id, lang, kind); const compiled = compile(lang, workDir, fs.readFileSync(file, "utf8")); let outcome; let good; if (!compiled.ok) { outcome = `compile error: ${compiled.diagnostics.map((d) => `${d.code} ${d.message.split("\n")[0]}`).slice(0, 3).join("; ")}`; good = kind === "mutant"; } else { const t = runHiddenTests(lang, path.join(dir, "test.mjs"), compiled.emitted); outcome = `tests ${t.pass}/${t.total}${t.failures.length ? ` failing: ${t.failures.join(", ")}` : ""}${t.timedOut ? " (timed out)" : ""}`; good = kind === "reference" ? t.allPassed : !t.allPassed; if (kind === "reference" && !t.allPassed && t.rawTail) { outcome += `\n${t.rawTail.split("\n").filter((l) => /not ok|error|expected|actual|Error/.test(l)).slice(0, 25).join("\n")}`; } } if (!good) { problems += 1; } rows.push({ id, lang, kind, good, outcome }); console.log(`${good ? "ok " : "FAIL"} ${id} ${kind}.${lang}: ${outcome}`); } }}fs.rmSync(workRoot, { recursive: true, force: true });console.log(`\n${ids.length} task(s), ${problems} problem(s)`);process.exit(problems === 0 ? 0 : 1);