Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 68 lines · 2.8 KiB · at 2e2b18f

#!/usr/bin/env node// Validates the task set without calling a model.//// For every task and language://   - reference.<lang> must compile and pass every hidden test;//   - mutant.<lang>, when present, is a plausible buggy solution (the kind of//     bug the task is designed to provoke) and must NOT pass: either the//     compiler rejects it or at least one hidden test fails. A mutant that//     passes means the hidden tests are too weak.////   node evals/validate.mjs [task-id ...]import fs from "node:fs";import path from "node:path";import { compile, runHiddenTests, evalsDir } from "./lib/compile.mjs";const tasksDir = path.join(evalsDir, "tasks");const only = process.argv.slice(2);const ids = fs  .readdirSync(tasksDir)  .filter((n) => fs.existsSync(path.join(tasksDir, n, "task.json")))  .filter((n) => only.length === 0 || only.includes(n))  .sort();const workRoot = path.join(evalsDir, ".work", `validate-${process.pid}`);let problems = 0;const rows = [];for (const id of ids) {  const dir = path.join(tasksDir, id);  const required = ["prompt.md", "signature.ts", "signature.tsus", "reference.ts", "reference.tsus", "test.mjs"];  const missing = required.filter((f) => !fs.existsSync(path.join(dir, f)));  if (missing.length > 0) {    console.log(`FAIL ${id}: missing ${missing.join(", ")}`);    problems += 1;    continue;  }  for (const lang of ["ts", "tsus"]) {    for (const kind of ["reference", "mutant"]) {      const file = path.join(dir, `${kind}.${lang}`);      if (!fs.existsSync(file)) {        continue;      }      const workDir = path.join(workRoot, id, lang, kind);      const compiled = compile(lang, workDir, fs.readFileSync(file, "utf8"));      let outcome;      let good;      if (!compiled.ok) {        outcome = `compile error: ${compiled.diagnostics.map((d) => `${d.code} ${d.message.split("\n")[0]}`).slice(0, 3).join("; ")}`;        good = kind === "mutant";      } else {        const t = runHiddenTests(lang, path.join(dir, "test.mjs"), compiled.emitted);        outcome = `tests ${t.pass}/${t.total}${t.failures.length ? ` failing: ${t.failures.join(", ")}` : ""}${t.timedOut ? " (timed out)" : ""}`;        good = kind === "reference" ? t.allPassed : !t.allPassed;        if (kind === "reference" && !t.allPassed && t.rawTail) {          outcome += `\n${t.rawTail.split("\n").filter((l) => /not ok|error|expected|actual|Error/.test(l)).slice(0, 25).join("\n")}`;        }      }      if (!good) {        problems += 1;      }      rows.push({ id, lang, kind, good, outcome });      console.log(`${good ? "ok  " : "FAIL"} ${id} ${kind}.${lang}: ${outcome}`);    }  }}fs.rmSync(workRoot, { recursive: true, force: true });console.log(`\n${ids.length} task(s), ${problems} problem(s)`);process.exit(problems === 0 ? 0 : 1);