Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 210 lines · 7.3 KiB · at f30ac63

// Model providers. Each provider exposes//   generate({ system, prompt, task, lang, round }) -> { text, usage, costUsd, ms }//// claude-cli  `claude -p` headless (default when no ANTHROPIC_API_KEY)// anthropic   Messages API through @anthropic-ai/sdk (default when ANTHROPIC_API_KEY is set)// reference   returns the task's reference solution (harness self-test, no model)// selftest    round 0 returns the reference with a planted compile error, round 1//             the clean reference; proves the repair loop end to end (no model)//// Keys are never logged: the CLI and the SDK read credentials themselves.import fs from "node:fs";import os from "node:os";import path from "node:path";import { spawn } from "node:child_process";// Price per million tokens, for providers that do not report cost themselves.const PRICES = {  "claude-sonnet-5-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },  "claude-sonnet-5": { input: 2, output: 10, cacheRead: 0.2, cacheWrite: 2.5 },  "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 },  "claude-opus-5-5": { input: 4, output: 20, cacheRead: 0.2, cacheWrite: 5 },  "claude-haiku-5-5": { input: 0.1, output: 0.5, cacheRead: 0.01, cacheWrite: 0.125 },};export const DEFAULT_MODEL = "claude-sonnet-5-5";const emptyUsage = () => ({ inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 });const priceUsage = (model, usage) => {  const p = PRICES[model];  if (!p) {    return 0;  }  return (    (usage.inputTokens * p.input +      usage.outputTokens * p.output +      usage.cacheReadTokens * p.cacheRead +      usage.cacheWriteTokens * p.cacheWrite) /    1_000_000  );};// Environment variables that tie a child `claude` process to a parent Claude// Code session. Inherited, they make the child try to borrow the parent's auth// and fail; the eval wants a standalone, stateless call.const NESTED_SESSION_VARS = [  "CLAUDECODE",  "CLAUDE_CODE_ENTRYPOINT",  "CLAUDE_CODE_CHILD_SESSION",  "CLAUDE_CODE_SESSION_ID",  "CLAUDE_CODE_HOST_SESSION_ID",  "CLAUDE_CODE_SDK_HAS_HOST_AUTH_REFRESH",  "CLAUDE_CODE_MESSAGING_SOCKET",  "CLAUDE_CODE_MESSAGING_TOKEN",  "CLAUDE_CODE_SIMPLE",  "CLAUDE_AGENT_SDK_VERSION",];const claudeCli = ({ model, timeoutMs = 600_000 }) => ({  name: "claude-cli",  model,  generate: ({ system, prompt }) =>    new Promise((resolve, reject) => {      const env = { ...process.env };      for (const key of NESTED_SESSION_VARS) {        delete env[key];      }      const started = Date.now();      const args = [        "-p",        "--model",        model,        "--system-prompt",        system,        "--tools",        "",        "--strict-mcp-config",        "--no-session-persistence",        "--output-format",        "json",      ];      // Run from an empty temp dir so no project CLAUDE.md leaks into context.      const cwd = fs.mkdtempSync(path.join(os.tmpdir(), "tsus-eval-"));      const child = spawn(process.env.CLAUDE_BIN ?? "claude", args, { cwd, env, stdio: ["pipe", "pipe", "pipe"] });      let stdout = "";      let stderr = "";      const timer = setTimeout(() => child.kill("SIGKILL"), timeoutMs);      child.stdout.on("data", (d) => {        stdout += d;      });      child.stderr.on("data", (d) => {        stderr += d;      });      child.on("error", reject);      child.on("close", () => {        clearTimeout(timer);        fs.rmSync(cwd, { recursive: true, force: true });        let data;        try {          data = JSON.parse(stdout);        } catch {          reject(new Error(`claude -p returned non-JSON output: ${(stdout || stderr).slice(0, 500)}`));          return;        }        if (data.is_error) {          reject(new Error(`claude -p failed: ${String(data.result).slice(0, 500)}`));          return;        }        const u = data.usage ?? {};        resolve({          text: data.result ?? "",          usage: {            inputTokens: u.input_tokens ?? 0,            outputTokens: u.output_tokens ?? 0,            cacheReadTokens: u.cache_read_input_tokens ?? 0,            cacheWriteTokens: u.cache_creation_input_tokens ?? 0,          },          costUsd: data.total_cost_usd ?? 0,          ms: Date.now() - started,          servedBy: Object.keys(data.modelUsage ?? {}).join(",") || model,        });      });      child.stdin.end(prompt);    }),});const anthropicApi = ({ model, effort }) => {  let clientPromise;  const getClient = () => {    clientPromise ??= import("@anthropic-ai/sdk")      .catch(() => {        throw new Error("the anthropic provider needs @anthropic-ai/sdk: run `npm install --prefix evals`");      })      .then((mod) => new mod.default());    return clientPromise;  };  return {    name: "anthropic",    model,    generate: async ({ system, prompt }) => {      const client = await getClient();      const started = Date.now();      const response = await client.messages.create({        model,        max_tokens: 16000,        system,        messages: [{ role: "user", content: prompt }],        ...(effort ? { output_config: { effort } } : {}),      });      if (response.stop_reason === "refusal") {        throw new Error(`model refused (${response.stop_details?.category ?? "no category"})`);      }      const text = response.content        .filter((b) => b.type === "text")        .map((b) => b.text)        .join("\n");      const usage = {        inputTokens: response.usage.input_tokens ?? 0,        outputTokens: response.usage.output_tokens ?? 0,        cacheReadTokens: response.usage.cache_read_input_tokens ?? 0,        cacheWriteTokens: response.usage.cache_creation_input_tokens ?? 0,      };      return { text, usage, costUsd: priceUsage(model, usage), ms: Date.now() - started, servedBy: response.model };    },  };};const fence = (lang, code) => "```" + lang + "\n" + code + "\n```\n";// A compile error each checker is guaranteed to reject.const PLANTED_ERROR = {  ts: 'const __planted: number = "not a number";\nvoid __planted;\n',  tsus: "let __planted = null;\nvoid __planted;\n",};const referenceProvider = ({ broken }) => ({  name: broken ? "selftest" : "reference",  model: "none",  generate: async ({ task, lang, round }) => {    const code = fs.readFileSync(path.join(task.dir, `reference.${lang}`), "utf8");    const text = broken && round === 0 ? code + "\n" + PLANTED_ERROR[lang] : code;    return { text: fence(lang, text), usage: emptyUsage(), costUsd: 0, ms: 0, servedBy: "none" };  },});export const makeProvider = ({ provider, model, effort }) => {  const chosen = provider ?? (process.env.ANTHROPIC_API_KEY ? "anthropic" : "claude-cli");  switch (chosen) {    case "claude-cli":      return claudeCli({ model });    case "anthropic":      return anthropicApi({ model, effort });    case "reference":      return referenceProvider({ broken: false });    case "selftest":      return referenceProvider({ broken: true });    default:      throw new Error(`unknown provider ${chosen}`);  }};// Pull the solution out of a reply: the longest fenced block wins, since models// sometimes show a snippet before the full file.export const extractCode = (text) => {  const blocks = [...text.matchAll(/```[a-zA-Z]*\n([\s\S]*?)```/g)].map((m) => m[1]);  if (blocks.length === 0) {    return text.trim();  }  return blocks.reduce((a, b) => (b.length >= a.length ? b : a));};