// Compile one solution in a scratch project and run the hidden tests on it.//// .ts -> tsc --strict (via a generated tsconfig), emits dist/solution.js// .tsus -> `tsus check --json` then `tsus build --json`, emits dist/solution.js//// Work directories live under evals/.work so the compiled output resolves// @tsus/runtime from the repository's node_modules.import fs from "node:fs";import path from "node:path";import { spawnSync } from "node:child_process";import { pathToFileURL } from "node:url";export const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");export const evalsDir = path.join(repoRoot, "evals");const tscBin = path.join(repoRoot, "node_modules/typescript/bin/tsc");export const defaultTsusBin = path.join(repoRoot, "packages/cli/bin/tsus.js");export const TS_COMPILER_OPTIONS = { strict: true, target: "ES2022", module: "ESNext", moduleResolution: "Bundler", lib: ["ES2022"], types: [], rootDir: "src", outDir: "dist", skipLibCheck: true, noEmitOnError: true,};const run = (cmd, args, cwd, timeout = 60_000) => { const r = spawnSync(cmd, args, { cwd, encoding: "utf8", timeout }); return { status: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "", timedOut: r.error?.code === "ETIMEDOUT", };};const prepare = (workDir, file, code) => { fs.rmSync(workDir, { recursive: true, force: true }); fs.mkdirSync(path.join(workDir, "src"), { recursive: true }); fs.writeFileSync(path.join(workDir, "package.json"), '{ "type": "module", "private": true }\n'); fs.writeFileSync(path.join(workDir, "src", file), code);};// tsc prints `src/solution.ts(12,5): error TS2322: Type ...` lines.const parseTscOutput = (text) => { const diagnostics = []; for (const line of text.split("\n")) { const m = line.match(/^(.*?)\((\d+),(\d+)\): error (TS\d+): (.*)$/); if (m) { diagnostics.push({ code: m[4], line: Number(m[2]), col: Number(m[3]), message: m[5] }); } else if (line.startsWith(" ") && diagnostics.length > 0) { diagnostics[diagnostics.length - 1].message += `\n${line}`; } else if (/error TS\d+/.test(line)) { diagnostics.push({ code: line.match(/TS\d+/)[0], line: 0, col: 0, message: line }); } } return diagnostics;};export const compileTs = (workDir, code) => { prepare(workDir, "solution.ts", code); fs.writeFileSync( path.join(workDir, "tsconfig.json"), JSON.stringify({ compilerOptions: TS_COMPILER_OPTIONS, include: ["src"] }, null, 2), ); const r = run(process.execPath, [tscBin, "-p", "tsconfig.json"], workDir); const diagnostics = parseTscOutput(r.stdout + r.stderr); const emitted = path.join(workDir, "dist", "solution.js"); if (r.status !== 0 && diagnostics.length === 0) { diagnostics.push({ code: "TSC", line: 0, col: 0, message: (r.stdout + r.stderr).trim() || "tsc failed" }); } return { ok: r.status === 0 && fs.existsSync(emitted), diagnostics, emitted };};const parseTsusJson = (text) => { try { return JSON.parse(text); } catch { return { ok: false, diagnostics: [{ code: "TSUS_CLI", line: 0, col: 0, message: text.trim().slice(0, 2000) }] }; }};export const compileTsus = (workDir, code, tsusBin = defaultTsusBin) => { prepare(workDir, "solution.tsus", code); const check = run(process.execPath, [tsusBin, "check", workDir, "--json"], workDir); const checked = parseTsusJson(check.stdout || check.stderr); const toDiag = (d) => ({ code: d.code, line: d.line ?? 0, col: d.col ?? 0, message: d.hint ? `${d.message} (hint: ${d.hint})` : d.message, }); if (!checked.ok) { return { ok: false, diagnostics: (checked.diagnostics ?? []).map(toDiag), emitted: "" }; } const build = run(process.execPath, [tsusBin, "build", workDir, "--json"], workDir); const built = parseTsusJson(build.stdout || build.stderr); const emitted = path.join(workDir, "dist", "solution.js"); if (!built.ok || !fs.existsSync(emitted)) { const diagnostics = (built.diagnostics ?? []).map(toDiag); if (diagnostics.length === 0) { diagnostics.push({ code: "TSUS_BUILD", line: 0, col: 0, message: (build.stderr || build.stdout).trim().slice(0, 2000) }); } return { ok: false, diagnostics, emitted: "" }; } return { ok: true, diagnostics: [], emitted };};export const compile = (lang, workDir, code, opts = {}) => lang === "ts" ? compileTs(workDir, code) : compileTsus(workDir, code, opts.tsusBin);export const formatDiagnostics = (lang, diagnostics) => diagnostics .map((d) => `solution.${lang}:${d.line}:${d.col} error ${d.code}: ${d.message}`) .join("\n");// Run the task's hidden node:test file against the compiled solution.// Returns counts plus the names of failing tests.export const runHiddenTests = (lang, testFile, emitted, timeoutMs = 30_000) => { const r = spawnSync( process.execPath, ["--test", "--test-reporter=tap", "--test-timeout=5000", testFile], { cwd: path.dirname(testFile), encoding: "utf8", timeout: timeoutMs, env: { ...process.env, TSUS_EVAL_LANG: lang, TSUS_EVAL_SOLUTION: pathToFileURL(emitted).href, }, }, ); const out = r.stdout ?? ""; const num = (label) => { const m = out.match(new RegExp(`^# ${label} (\\d+)$`, "m")); return m ? Number(m[1]) : 0; }; const failures = []; const lines = out.split("\n"); for (let i = 0; i < lines.length; i += 1) { // Subtests are indented; top-level `not ok N - name` lines name failures. const m = lines[i].match(/^\s*not ok \d+ - (.*)$/); if (m && !/^\s{4,}/.test(lines[i])) { failures.push(m[1].trim()); } } const pass = num("pass"); if (r.error?.code === "ETIMEDOUT") { failures.push("(test process timed out: likely an infinite loop)"); } const fail = num("fail") + num("cancelled"); const timedOut = r.error?.code === "ETIMEDOUT"; // A synchronous infinite loop blocks the test runner's own timers, so the // process is killed with no summary. Count every declared test as failed // rather than reporting 0/0. const declared = (fs.readFileSync(testFile, "utf8").match(/^test\(/gm) ?? []).length; const total = timedOut ? Math.max(num("tests"), declared) : num("tests"); return { pass, fail: timedOut ? Math.max(1, total - pass) : fail, total, timedOut, failures, allPassed: !timedOut && r.status === 0 && fail === 0 && total > 0, rawTail: r.status === 0 ? "" : (out + (r.stderr ?? "")).slice(-3000), };};