Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 172 lines · 6.4 KiB · at 503e839

// Compile one solution in a scratch project and run the hidden tests on it.//// .ts   -> tsc --strict (via a generated tsconfig), emits dist/solution.js// .tsus -> `tsus check --json` then `tsus build --json`, emits dist/solution.js//// Work directories live under evals/.work so the compiled output resolves// @tsus/runtime from the repository's node_modules.import fs from "node:fs";import path from "node:path";import { spawnSync } from "node:child_process";import { pathToFileURL } from "node:url";export const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");export const evalsDir = path.join(repoRoot, "evals");const tscBin = path.join(repoRoot, "node_modules/typescript/bin/tsc");export const defaultTsusBin = path.join(repoRoot, "packages/cli/bin/tsus.js");export const TS_COMPILER_OPTIONS = {  strict: true,  target: "ES2022",  module: "ESNext",  moduleResolution: "Bundler",  lib: ["ES2022"],  types: [],  rootDir: "src",  outDir: "dist",  skipLibCheck: true,  noEmitOnError: true,};const run = (cmd, args, cwd, timeout = 60_000) => {  const r = spawnSync(cmd, args, { cwd, encoding: "utf8", timeout });  return {    status: r.status,    stdout: r.stdout ?? "",    stderr: r.stderr ?? "",    timedOut: r.error?.code === "ETIMEDOUT",  };};const prepare = (workDir, file, code) => {  fs.rmSync(workDir, { recursive: true, force: true });  fs.mkdirSync(path.join(workDir, "src"), { recursive: true });  fs.writeFileSync(path.join(workDir, "package.json"), '{ "type": "module", "private": true }\n');  fs.writeFileSync(path.join(workDir, "src", file), code);};// tsc prints `src/solution.ts(12,5): error TS2322: Type ...` lines.const parseTscOutput = (text) => {  const diagnostics = [];  for (const line of text.split("\n")) {    const m = line.match(/^(.*?)\((\d+),(\d+)\): error (TS\d+): (.*)$/);    if (m) {      diagnostics.push({ code: m[4], line: Number(m[2]), col: Number(m[3]), message: m[5] });    } else if (line.startsWith("  ") && diagnostics.length > 0) {      diagnostics[diagnostics.length - 1].message += `\n${line}`;    } else if (/error TS\d+/.test(line)) {      diagnostics.push({ code: line.match(/TS\d+/)[0], line: 0, col: 0, message: line });    }  }  return diagnostics;};export const compileTs = (workDir, code) => {  prepare(workDir, "solution.ts", code);  fs.writeFileSync(    path.join(workDir, "tsconfig.json"),    JSON.stringify({ compilerOptions: TS_COMPILER_OPTIONS, include: ["src"] }, null, 2),  );  const r = run(process.execPath, [tscBin, "-p", "tsconfig.json"], workDir);  const diagnostics = parseTscOutput(r.stdout + r.stderr);  const emitted = path.join(workDir, "dist", "solution.js");  if (r.status !== 0 && diagnostics.length === 0) {    diagnostics.push({ code: "TSC", line: 0, col: 0, message: (r.stdout + r.stderr).trim() || "tsc failed" });  }  return { ok: r.status === 0 && fs.existsSync(emitted), diagnostics, emitted };};const parseTsusJson = (text) => {  try {    return JSON.parse(text);  } catch {    return { ok: false, diagnostics: [{ code: "TSUS_CLI", line: 0, col: 0, message: text.trim().slice(0, 2000) }] };  }};export const compileTsus = (workDir, code, tsusBin = defaultTsusBin) => {  prepare(workDir, "solution.tsus", code);  const check = run(process.execPath, [tsusBin, "check", workDir, "--json"], workDir);  const checked = parseTsusJson(check.stdout || check.stderr);  const toDiag = (d) => ({    code: d.code,    line: d.line ?? 0,    col: d.col ?? 0,    message: d.hint ? `${d.message} (hint: ${d.hint})` : d.message,  });  if (!checked.ok) {    return { ok: false, diagnostics: (checked.diagnostics ?? []).map(toDiag), emitted: "" };  }  const build = run(process.execPath, [tsusBin, "build", workDir, "--json"], workDir);  const built = parseTsusJson(build.stdout || build.stderr);  const emitted = path.join(workDir, "dist", "solution.js");  if (!built.ok || !fs.existsSync(emitted)) {    const diagnostics = (built.diagnostics ?? []).map(toDiag);    if (diagnostics.length === 0) {      diagnostics.push({ code: "TSUS_BUILD", line: 0, col: 0, message: (build.stderr || build.stdout).trim().slice(0, 2000) });    }    return { ok: false, diagnostics, emitted: "" };  }  return { ok: true, diagnostics: [], emitted };};export const compile = (lang, workDir, code, opts = {}) =>  lang === "ts" ? compileTs(workDir, code) : compileTsus(workDir, code, opts.tsusBin);export const formatDiagnostics = (lang, diagnostics) =>  diagnostics    .map((d) => `solution.${lang}:${d.line}:${d.col} error ${d.code}: ${d.message}`)    .join("\n");// Run the task's hidden node:test file against the compiled solution.// Returns counts plus the names of failing tests.export const runHiddenTests = (lang, testFile, emitted, timeoutMs = 30_000) => {  const r = spawnSync(    process.execPath,    ["--test", "--test-reporter=tap", "--test-timeout=5000", testFile],    {      cwd: path.dirname(testFile),      encoding: "utf8",      timeout: timeoutMs,      env: {        ...process.env,        TSUS_EVAL_LANG: lang,        TSUS_EVAL_SOLUTION: pathToFileURL(emitted).href,      },    },  );  const out = r.stdout ?? "";  const num = (label) => {    const m = out.match(new RegExp(`^# ${label} (\\d+)$`, "m"));    return m ? Number(m[1]) : 0;  };  const failures = [];  const lines = out.split("\n");  for (let i = 0; i < lines.length; i += 1) {    // Subtests are indented; top-level `not ok N - name` lines name failures.    const m = lines[i].match(/^\s*not ok \d+ - (.*)$/);    if (m && !/^\s{4,}/.test(lines[i])) {      failures.push(m[1].trim());    }  }  const pass = num("pass");  if (r.error?.code === "ETIMEDOUT") {    failures.push("(test process timed out: likely an infinite loop)");  }  const fail = num("fail") + num("cancelled");  const timedOut = r.error?.code === "ETIMEDOUT";  // A synchronous infinite loop blocks the test runner's own timers, so the  // process is killed with no summary. Count every declared test as failed  // rather than reporting 0/0.  const declared = (fs.readFileSync(testFile, "utf8").match(/^test\(/gm) ?? []).length;  const total = timedOut ? Math.max(num("tests"), declared) : num("tests");  return {    pass,    fail: timedOut ? Math.max(1, total - pass) : fail,    total,    timedOut,    failures,    allPassed: !timedOut && r.status === 0 && fail === 0 && total > 0,    rawTail: r.status === 0 ? "" : (out + (r.stderr ?? "")).slice(-3000),  };};