Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 297 lines · 10.3 KiB · at 355e324

#!/usr/bin/env node// The tsus eval runner.//// For each task and each language: ask the model for a solution, compile it,// feed compiler errors back for up to --repairs rounds, then run the hidden// tests once on the first version that compiles. Test results are never shown// to the model; only the compiler talks back.////   node evals/run.mjs --pilot                      # 10 pilot tasks, ts + tsus//   node evals/run.mjs --tasks lru-cache,parse-config --langs tsus//   node evals/run.mjs --provider reference         # harness self-test, no model//// Writes evals/results/<run-id>/{meta.json,results.jsonl,attempts/*}.import fs from "node:fs";import path from "node:path";import { execSync } from "node:child_process";import { parseArgs } from "node:util";import { compile, formatDiagnostics, runHiddenTests, evalsDir, repoRoot, defaultTsusBin, TS_COMPILER_OPTIONS } from "./lib/compile.mjs";import { makeProvider, extractCode, DEFAULT_MODEL } from "./lib/providers.mjs";const { values: opts } = parseArgs({  options: {    tasks: { type: "string" },    pilot: { type: "boolean", default: false },    limit: { type: "string" },    langs: { type: "string", default: "ts,tsus" },    model: { type: "string", default: process.env.TSUS_EVAL_MODEL ?? DEFAULT_MODEL },    provider: { type: "string" },    effort: { type: "string" },    repairs: { type: "string", default: "3" },    concurrency: { type: "string", default: "4" },    label: { type: "string" },    "tsus-bin": { type: "string", default: defaultTsusBin },    "max-cost": { type: "string" },    help: { type: "boolean", default: false },  },});if (opts.help) {  console.log(fs.readFileSync(new URL(import.meta.url), "utf8").split("\n").slice(1, 14).join("\n").replace(/^\/\/ ?/gm, ""));  process.exit(0);}const tasksDir = path.join(evalsDir, "tasks");const loadTasks = () =>  fs    .readdirSync(tasksDir)    .filter((name) => fs.existsSync(path.join(tasksDir, name, "task.json")))    .sort()    .map((name) => {      const dir = path.join(tasksDir, name);      const meta = JSON.parse(fs.readFileSync(path.join(dir, "task.json"), "utf8"));      return { ...meta, id: name, dir };    });let tasks = loadTasks();if (opts.tasks) {  const wanted = opts.tasks.split(",").map((s) => s.trim());  const missing = wanted.filter((w) => !tasks.some((t) => t.id === w));  if (missing.length > 0) {    console.error(`unknown task(s): ${missing.join(", ")}`);    process.exit(2);  }  tasks = tasks.filter((t) => wanted.includes(t.id));} else if (opts.pilot) {  tasks = tasks.filter((t) => t.pilot === true);}if (opts.limit) {  tasks = tasks.slice(0, Number(opts.limit));}const langs = opts.langs.split(",").map((s) => s.trim());const maxRepairs = Number(opts.repairs);const concurrency = Math.max(1, Number(opts.concurrency));const maxCost = opts["max-cost"] ? Number(opts["max-cost"]) : Infinity;const provider = makeProvider({ provider: opts.provider, model: opts.model, effort: opts.effort });const language = fs.readFileSync(path.join(evalsDir, "LANGUAGE.md"), "utf8");const systemPrompts = {  ts: fs.readFileSync(path.join(evalsDir, "prompts/system-ts.md"), "utf8"),  tsus: fs.readFileSync(path.join(evalsDir, "prompts/system-tsus.md"), "utf8").replace("{{LANGUAGE}}", language),};const taskPrompt = (task, lang) => {  const prompt = fs.readFileSync(path.join(task.dir, "prompt.md"), "utf8").trim();  const signature = fs.readFileSync(path.join(task.dir, `signature.${lang}`), "utf8").trim();  return [    `# Task: ${task.title}`,    "",    prompt,    "",    `## Signature file (solution.${lang})`,    "",    "```" + lang,    signature,    "```",    "",    `Write the complete solution.${lang}.`,  ].join("\n");};const repairPrompt = (task, lang, previous, diagnostics) =>  [    taskPrompt(task, lang),    "",    "## Your previous attempt",    "",    "```" + lang,    previous,    "```",    "",    "## The compiler rejected it",    "",    "```",    formatDiagnostics(lang, diagnostics),    "```",    "",    `Fix every error and reply with the complete corrected solution.${lang}.`,  ].join("\n");const gitSha = (dir) => {  try {    return execSync("git rev-parse --short HEAD", { cwd: dir, encoding: "utf8" }).trim();  } catch {    return "unknown";  }};const tsusCliDirty = (() => {  try {    return execSync("git status --porcelain packages/cli packages/runtime", { cwd: repoRoot, encoding: "utf8" }).trim() !== "";  } catch {    return false;  }})();const started = new Date();const runId = `${started.toISOString().slice(0, 19).replace(/[:T]/g, "-")}-${provider.name}`;const runDir = path.join(evalsDir, "results", runId);const attemptsDir = path.join(runDir, "attempts");const workRoot = path.join(evalsDir, ".work", runId);fs.mkdirSync(attemptsDir, { recursive: true });const meta = {  runId,  label: opts.label ?? (opts.pilot ? "pilot" : provider.name === "reference" || provider.name === "selftest" ? "harness self-test" : "run"),  startedAt: started.toISOString(),  provider: provider.name,  model: provider.model,  effort: opts.effort ?? "default",  maxRepairs,  langs,  tasks: tasks.map((t) => t.id),  repoCommit: gitSha(repoRoot),  tsusCliDirty,  tsusBin: path.relative(repoRoot, opts["tsus-bin"]),  tsCompilerOptions: TS_COMPILER_OPTIONS,  node: process.version,};fs.writeFileSync(path.join(runDir, "meta.json"), JSON.stringify(meta, null, 2) + "\n");const resultsFile = path.join(runDir, "results.jsonl");let spent = 0;let consecutiveProviderErrors = 0;let aborted = "";const runOne = async (task, lang) => {  const record = {    task: task.id,    lang,    tags: task.tags ?? [],    attempts: [],    status: "pending",    costUsd: 0,    ms: 0,    usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 },  };  const workDir = path.join(workRoot, task.id, lang);  let code = "";  let diagnostics = [];  for (let round = 0; round <= maxRepairs; round += 1) {    if (aborted) {      record.status = "aborted";      return record;    }    const prompt = round === 0 ? taskPrompt(task, lang) : repairPrompt(task, lang, code, diagnostics);    let reply;    try {      reply = await provider.generate({ system: systemPrompts[lang], prompt, task, lang, round });      consecutiveProviderErrors = 0;    } catch (error) {      consecutiveProviderErrors += 1;      record.status = "provider_error";      record.error = String(error.message ?? error).slice(0, 500);      if (consecutiveProviderErrors >= 3) {        aborted = `3 provider errors in a row, last: ${record.error}`;      }      return record;    }    spent += reply.costUsd;    record.costUsd += reply.costUsd;    record.ms += reply.ms;    for (const k of Object.keys(record.usage)) {      record.usage[k] += reply.usage[k] ?? 0;    }    code = extractCode(reply.text);    const base = path.join(attemptsDir, `${task.id}.${lang}.r${round}`);    fs.writeFileSync(`${base}.${lang}`, code);    const compileStarted = Date.now();    const result = compile(lang, workDir, code, { tsusBin: opts["tsus-bin"] });    diagnostics = result.diagnostics;    record.attempts.push({      round,      compiled: result.ok,      diagnosticCount: diagnostics.length,      codes: [...new Set(diagnostics.map((d) => d.code))],      modelMs: reply.ms,      compileMs: Date.now() - compileStarted,      costUsd: reply.costUsd,      outputTokens: reply.usage.outputTokens,    });    if (diagnostics.length > 0) {      fs.writeFileSync(`${base}.diagnostics.txt`, formatDiagnostics(lang, diagnostics) + "\n");    }    if (result.ok) {      const tests = runHiddenTests(lang, path.join(task.dir, "test.mjs"), result.emitted);      record.tests = tests;      record.status = tests.allPassed ? "passed" : "failed_tests";      if (!tests.allPassed && tests.rawTail) {        fs.writeFileSync(`${base}.tests.txt`, tests.rawTail);      }      delete record.tests.rawTail;      break;    }    record.status = "compile_failed";    if (spent >= maxCost) {      aborted = `cost cap $${maxCost} reached`;    }  }  const first = record.attempts[0];  record.compiledFirstTry = first?.compiled === true;  record.passFirstTry = record.compiledFirstTry && record.status === "passed";  record.passFinal = record.status === "passed";  record.repairRounds = Math.max(0, record.attempts.length - 1);  record.compileRejections = record.attempts.filter((a) => !a.compiled).length;  record.compileDiagnostics = record.attempts.reduce((n, a) => n + a.diagnosticCount, 0);  record.runtimeFailures = record.tests?.fail ?? 0;  if (spent >= maxCost && !aborted) {    aborted = `cost cap $${maxCost} reached`;  }  return record;};const jobs = [];for (const task of tasks) {  for (const lang of langs) {    if (fs.existsSync(path.join(task.dir, `signature.${lang}`))) {      jobs.push({ task, lang });    }  }}console.log(`run ${runId}: ${jobs.length} job(s), provider=${provider.name}, model=${provider.model}, repairs=${maxRepairs}`);let next = 0;let done = 0;const worker = async () => {  while (next < jobs.length) {    const { task, lang } = jobs[next];    next += 1;    const record = await runOne(task, lang);    if (record.status === "provider_error") {      providerErrors += 1;    }    fs.appendFileSync(resultsFile, JSON.stringify(record) + "\n");    done += 1;    const t = record.tests ? ` tests ${record.tests.pass}/${record.tests.total}` : "";    console.log(      `[${done}/${jobs.length}] ${task.id} (${lang}): ${record.status}, ${record.attempts.length} attempt(s)${t}, $${record.costUsd.toFixed(4)}${record.error ? ` (${record.error})` : ""}`,    );  }};let providerErrors = 0;await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, worker));meta.finishedAt = new Date().toISOString();meta.totalCostUsd = Number(spent.toFixed(4));meta.aborted = aborted || false;fs.writeFileSync(path.join(runDir, "meta.json"), JSON.stringify(meta, null, 2) + "\n");fs.rmSync(workRoot, { recursive: true, force: true });console.log(`done: total cost $${spent.toFixed(4)}${aborted ? `, aborted: ${aborted}` : ""}`);console.log(`results: ${path.relative(repoRoot, runDir)}`);console.log(`report:  node evals/report.mjs ${path.relative(repoRoot, runDir)}`);if (providerErrors === jobs.length && jobs.length > 0) {  console.error("every job failed at the provider; check model access (see evals/README.md)");  process.exit(1);}if (aborted) {  process.exit(1);}