Rebasely Beta: data may be purged

justintypescript-ultra-strict Public

JavaScript · 742 lines · 23.8 KiB · at 9c48a39

// The tsus preprocessor: turns .tsus source into plain TypeScript in two// passes, tracking positions so every diagnostic and source map points back at// the .tsus file.//// Pass 1 (tokens): `match (expr) { ... }` blocks, which TypeScript cannot// parse, are found on the token stream from ts.createScanner and lowered:////   match (o) { Some(x) => a, None => b }   -> __tsus_match(o)({ Some: (x) => (a), None: () => (b) })//   match (o) { Some(x) => a, _ => b }      -> __tsus_match_or(o)({ Some: (x) => (a) }, () => (b))//   match (s) { "/" => a, _ => b }          -> __tsus_match_lit(s)([["/", () => (a)]], () => (b))//// The arms are an object keyed by variant tag, typed by @tsus/runtime with// mapped types so TypeScript itself enforces exhaustiveness and narrows each// arm's binding to its variant's payload.//// Pass 2 (AST): the `?` operator. `const x = load()?;` does not parse as// TypeScript, but the parser recovers it as a conditional expression with a// missing true branch, which pins down both the operand and the statement it// lives in. Each `?` is hoisted to an early return before its statement:////   const __tsus_t0 = __tsus_try(load()); if (__tsus_isBail(__tsus_t0)) return __tsus_bail(__tsus_t0); const x = __tsus_t0.value;import ts from "typescript";import { tokenize, findClose, isOpener, isCloser, SyntaxKind as K } from "./tokens.js";import { createEmitter, segmentLookup } from "./sourcemap.js";import { describe } from "./codes.js";// Kept for API compatibility; the preprocessor now reports diagnostics// instead of throwing.export class PreprocessError extends Error {  constructor(message, index) {    super(message);    this.code = "TSUS006";    this.index = index;  }}const diag = (code, offset, message) => ({  code,  message: message ?? describe(code).message,  hint: describe(code).hint,  offset,});// --- pass 1: match ---const NOT_MACRO_AFTER = new Set([  K.DotToken,  K.QuestionDotToken,  K.FunctionKeyword,  K.GetKeyword,  K.SetKeyword,  K.AsteriskToken,]);const LITERAL_PATTERN = new Set([  K.StringLiteral,  K.NumericLiteral,  K.BigIntLiteral,  K.NoSubstitutionTemplateLiteral,  K.TrueKeyword,  K.FalseKeyword,]);const FUNCTION_KINDS = new Set([  K.ArrowFunction,  K.FunctionExpression,  K.FunctionDeclaration,  K.MethodDeclaration,  K.GetAccessor,  K.SetAccessor,  K.Constructor,]);// True when the (already lowered) arm body awaits at its own level. Nested// functions, including nested match arms, own their awaits. Decided on the// parsed AST, so `"await"` in a string or comment never counts.const bodyAwaits = (bodyText) => {  const sf = ts.createSourceFile("arm.ts", `(async () => (\n${bodyText}\n))`, ts.ScriptTarget.Latest, true);  const root = sf.statements[0].expression.expression;  let found = false;  const visit = (node) => {    if (found) {      return;    }    if (node.kind === K.AwaitExpression || (node.kind === K.ForOfStatement && node.awaitModifier)) {      found = true;      return;    }    if (FUNCTION_KINDS.has(node.kind)) {      return;    }    ts.forEachChild(node, visit);  };  ts.forEachChild(root.body, visit);  return found;};// The token after `{` must look like the start of an arm, so an object// method named `match` (`{ match(x) { return x; } }`) is left alone.const looksLikeArms = (tokens, at, close) => {  if (at >= close) {    return true;  }  const t = tokens[at];  const next = tokens[at + 1];  if (t.kind === K.Identifier) {    if (next && next.kind === K.EqualsGreaterThanToken) {      return true;    }    if (next && next.kind === K.OpenParenToken) {      const c = findClose(tokens, at + 1, close);      return c !== -1 && tokens[c + 1] !== undefined && tokens[c + 1].kind === K.EqualsGreaterThanToken;    }    return false;  }  if (LITERAL_PATTERN.has(t.kind)) {    return next !== undefined && next.kind === K.EqualsGreaterThanToken;  }  if (t.kind === K.MinusToken) {    return next !== undefined && (next.kind === K.NumericLiteral || next.kind === K.BigIntLiteral);  }  return false;};const lowerMatches = (text) => {  const tokens = tokenize(text);  const diagnostics = [];  const sites = [];  const helpers = new Set();  // Detect `match (subject) {` at token i; returns { paren, brace, close } or null.  const matchHead = (i, hi) => {    const t = tokens[i];    if (t.kind !== K.Identifier || t.text !== "match") {      return null;    }    const prev = tokens[i - 1];    if (prev && NOT_MACRO_AFTER.has(prev.kind)) {      return null;    }    if (!tokens[i + 1] || tokens[i + 1].kind !== K.OpenParenToken) {      return null;    }    const paren = findClose(tokens, i + 1, hi);    if (paren === -1 || paren + 1 >= hi || tokens[paren + 1].kind !== K.OpenBraceToken) {      return null;    }    const brace = paren + 1;    const close = findClose(tokens, brace, hi);    if (close === -1) {      return null;    }    if (!looksLikeArms(tokens, brace + 1, close)) {      return null;    }    return { paren, brace, close };  };  // Split tokens [lo, hi) on top-level commas.  const splitArms = (lo, hi) => {    const parts = [];    let depth = 0;    let start = lo;    for (let t = lo; t < hi; t += 1) {      const kind = tokens[t].kind;      if (isOpener(kind)) {        depth += 1;      } else if (isCloser(kind)) {        depth -= 1;      } else if (kind === K.CommaToken && depth === 0) {        parts.push({ lo: start, hi: t });        start = t + 1;      }    }    parts.push({ lo: start, hi });    // A comma inside type arguments (`new Map<string, number>()`) is not    // bracketed, so it splits an arm in two. Every real arm starts with    // `pattern =>`; a fragment that does not belongs to the previous arm.    const merged = [];    for (const part of parts.filter((p) => p.lo < p.hi)) {      if (merged.length > 0 && !startsArm(part.lo, part.hi)) {        merged[merged.length - 1].hi = part.hi;      } else {        merged.push(part);      }    }    return merged;  };  const startsArm = (lo, hi) => {    let p = lo;    const t = tokens[p];    if (t.kind === K.MinusToken) {      p += 2;    } else if (t.kind === K.Identifier || LITERAL_PATTERN.has(t.kind)) {      p += 1;      if (t.kind === K.Identifier && p < hi && tokens[p].kind === K.OpenParenToken) {        const q = findClose(tokens, p, hi);        if (q === -1) {          return false;        }        p = q + 1;      }    } else {      return false;    }    return p < hi && tokens[p].kind === K.EqualsGreaterThanToken;  };  const parseArm = (lo, hi) => {    const first = tokens[lo];    let p = lo;    const arm = { lo, first };    if (first.kind === K.Identifier && first.text === "_") {      arm.kind = "wildcard";      p = lo + 1;    } else if (first.kind === K.Identifier) {      arm.kind = "variant";      arm.name = first.text;      p = lo + 1;      if (p < hi && tokens[p].kind === K.OpenParenToken) {        const q = findClose(tokens, p, hi);        if (q === -1) {          return { error: diag("TSUS006", tokens[p].start, "unclosed ( in match pattern") };        }        if (q > p + 1) {          arm.binding = { start: tokens[p + 1].start, end: tokens[q - 1].end };        }        p = q + 1;      }    } else if (LITERAL_PATTERN.has(first.kind)) {      arm.kind = "literal";      arm.literal = { start: first.start, end: first.end };      arm.value = first.text;      p = lo + 1;    } else if (      first.kind === K.MinusToken &&      lo + 1 < hi &&      (tokens[lo + 1].kind === K.NumericLiteral || tokens[lo + 1].kind === K.BigIntLiteral)    ) {      arm.kind = "literal";      arm.literal = { start: first.start, end: tokens[lo + 1].end };      arm.value = `-${tokens[lo + 1].text}`;      p = lo + 2;    } else {      return { error: diag("TSUS006", first.start, `unexpected "${first.text}" in match pattern`) };    }    if (p >= hi || tokens[p].kind !== K.EqualsGreaterThanToken) {      const at = p < hi ? tokens[p].start : tokens[hi - 1].end;      return { error: diag("TSUS006", at, "expected => after match pattern") };    }    arm.arrow = tokens[p];    if (p + 1 >= hi) {      return { error: diag("TSUS006", tokens[p].end, "empty match arm body") };    }    arm.bodyLo = p + 1;    arm.bodyHi = hi;    return { arm };  };  const lowerOne = (i, head, em) => {    const { paren, brace, close } = head;    const matchTok = tokens[i];    if (paren === i + 2) {      diagnostics.push(diag("TSUS006", matchTok.start, "match has no subject"));      return false;    }    const arms = [];    let failed = false;    for (const part of splitArms(brace + 1, close)) {      const parsed = parseArm(part.lo, part.hi);      if (parsed.error) {        diagnostics.push(parsed.error);        failed = true;      } else {        arms.push(parsed.arm);      }    }    if (failed) {      return false;    }    if (arms.length === 0) {      diagnostics.push(diag("TSUS006", matchTok.start, "match expression has no arms"));      return false;    }    const wildcards = arms.filter((a) => a.kind === "wildcard");    const variants = arms.filter((a) => a.kind === "variant");    const literals = arms.filter((a) => a.kind === "literal");    if (wildcards.length > 1) {      diagnostics.push(diag("TSUS006", wildcards[1].first.start, "match has more than one _ arm"));      return false;    }    if (variants.length > 0 && literals.length > 0) {      diagnostics.push(        diag("TSUS006", literals[0].first.start, "match mixes variant patterns and literal patterns"),      );      return false;    }    const seen = new Set();    for (const a of [...variants, ...literals]) {      const key = a.kind === "variant" ? a.name : a.value;      if (seen.has(key)) {        diagnostics.push(diag("TSUS006", a.first.start, `duplicate match pattern ${key}`));        return false;      }      seen.add(key);    }    // Lower the subject and each body first (they may hold nested matches),    // then decide asyncness: if any arm awaits, every arm becomes async so    // the match is uniformly a Promise.    const subjectEm = createEmitter();    emitRange(i + 2, paren, tokens[i + 2].start, tokens[paren - 1].end, subjectEm);    for (const a of arms) {      a.em = createEmitter();      emitRange(a.bodyLo, a.bodyHi, tokens[a.bodyLo].start, tokens[a.bodyHi - 1].end, a.em);    }    const isAsync = arms.some((a) => bodyAwaits(a.em.text()));    const asyncKw = isAsync ? "async " : "";    const wildcard = wildcards[0];    const kind = literals.length > 0 ? "literal" : "tag";    const helper = kind === "literal" ? "__tsus_match_lit" : wildcard ? "__tsus_match_or" : "__tsus_match";    helpers.add(helper);    // Trivia before an arm (whitespace, comments) is copied so the output    // keeps the source's line structure.    const copyTriviaBefore = (arm) => {      const prevTok = tokens[arm.lo - 1];      em.copy(text.slice(prevTok.end, arm.first.start), prevTok.end);    };    const closeTok = tokens[close];    const lastTok = tokens[close - 1];    em.synth(`${helper}(`, matchTok.start);    em.append(subjectEm);    em.synth(")(", matchTok.start);    const patterns = [];    if (kind === "tag") {      em.copy("{", tokens[brace].start);      for (const a of arms) {        copyTriviaBefore(a);        if (a.kind === "wildcard") {          continue;        }        em.copy(a.name, a.first.start);        em.synth(`: ${asyncKw}(`, a.first.start);        if (a.binding) {          em.copy(text.slice(a.binding.start, a.binding.end), a.binding.start);        }        em.synth(") => ", a.arrow.start);        em.synth("(", tokens[a.bodyLo].start);        em.append(a.em);        em.synth("),", tokens[a.bodyHi - 1].end);        patterns.push({ name: a.name, start: a.first.start });      }      em.copy(text.slice(lastTok.end, closeTok.start), lastTok.end);      em.copy("}", closeTok.start);    } else {      em.synth("[", tokens[brace].start);      for (const a of arms) {        copyTriviaBefore(a);        if (a.kind === "wildcard") {          continue;        }        em.synth("[", a.first.start);        em.copy(text.slice(a.literal.start, a.literal.end), a.literal.start);        em.synth(`, ${asyncKw}() => `, a.arrow.start);        em.synth("(", tokens[a.bodyLo].start);        em.append(a.em);        em.synth(")],", tokens[a.bodyHi - 1].end);        patterns.push({ name: a.value, start: a.first.start });      }      em.copy(text.slice(lastTok.end, closeTok.start), lastTok.end);      em.synth("]", closeTok.start);    }    if (wildcard) {      em.synth(`, ${asyncKw}() => `, wildcard.arrow.start);      em.synth("(", tokens[wildcard.bodyLo].start);      em.append(wildcard.em);      em.synth(")", tokens[wildcard.bodyHi - 1].end);    }    em.synth(")", closeTok.start);    sites.push({      kind,      matchStart: matchTok.start,      braceStart: tokens[brace].start,      subjectStart: tokens[i + 2].start,      end: closeTok.end,      patterns,      hasWildcard: Boolean(wildcard),    });    return true;  };  // Copies text [startOff, endOff), which covers tokens [lo, hi), lowering  // every match found along the way.  function emitRange(lo, hi, startOff, endOff, em) {    let cursor = startOff;    let i = lo;    while (i < hi) {      const head = matchHead(i, hi);      if (head) {        const sub = createEmitter();        if (lowerOne(i, head, sub)) {          em.copy(text.slice(cursor, tokens[i].start), cursor);          em.append(sub);          cursor = tokens[head.close].end;        }        i = head.close + 1;        continue;      }      i += 1;    }    em.copy(text.slice(cursor, endOff), cursor);  }  const em = createEmitter();  emitRange(0, tokens.length, 0, text.length, em);  return { text: em.text(), lookup: segmentLookup(em.segments), diagnostics, sites, helpers };};// --- pass 2: the ? operator ---const TERMINATORS = new Set([  K.SemicolonToken,  K.CloseParenToken,  K.CloseBracketToken,  K.CloseBraceToken,  K.CommaToken,  K.EndOfFileToken,]);const STATEMENT_LIST_PARENTS = new Set([K.Block, K.SourceFile, K.CaseClause, K.DefaultClause, K.ModuleBlock]);const SHORT_CIRCUIT = new Set([  K.AmpersandAmpersandToken,  K.BarBarToken,  K.QuestionQuestionToken,  K.AmpersandAmpersandEqualsToken,  K.BarBarEqualsToken,  K.QuestionQuestionEqualsToken,]);const isStatementNode = (node) =>  ts.isVariableStatement(node) ||  ts.isExpressionStatement(node) ||  ts.isReturnStatement(node) ||  ts.isIfStatement(node) ||  ts.isSwitchStatement(node) ||  ts.isThrowStatement(node) ||  ts.isForStatement(node) ||  ts.isForOfStatement(node) ||  ts.isForInStatement(node) ||  ts.isWhileStatement(node) ||  ts.isDoStatement(node);const isFunctionWithBody = (node) => FUNCTION_KINDS.has(node.kind) && node.body !== undefined;const typeNameIs = (typeNode, names) => {  if (!typeNode || !ts.isTypeReferenceNode(typeNode)) {    return false;  }  const n = typeNode.typeName;  const last = ts.isIdentifier(n) ? n.text : n.right.text;  return names.includes(last);};const returnsTryable = (typeNode) => {  if (typeNameIs(typeNode, ["Result", "Option"])) {    return true;  }  if (typeNameIs(typeNode, ["Promise"]) && typeNode.typeArguments && typeNode.typeArguments.length === 1) {    return typeNameIs(typeNode.typeArguments[0], ["Result", "Option"]);  }  return false;};// A function generated for a match arm: an arrow among the arms of a// __tsus_match*(subject)(...) call.const isMatchArm = (fn) => {  let n = fn.parent;  for (let depth = 0; n && depth < 4; depth += 1) {    if (ts.isCallExpression(n)) {      const callee = n.expression;      return (        ts.isCallExpression(callee) &&        ts.isIdentifier(callee.expression) &&        callee.expression.text.startsWith("__tsus_match")      );    }    n = n.parent;  }  return false;};const nextTokenKind = (text, from) => {  const scanner = ts.createScanner(ts.ScriptTarget.Latest, true, ts.LanguageVariant.Standard, text);  scanner.resetTokenState(from);  return scanner.scan();};const lowerTries = (text) => {  const sf = ts.createSourceFile("lowered.ts", text, ts.ScriptTarget.Latest, true, ts.ScriptKind.TS);  const diagnostics = [];  const helpers = new Set();  const trySites = new Map(); // ConditionalExpression -> site  const stmtSites = new Map(); // statement or arrow body -> { sites, braces, arrowBody }  // Candidates come from the token level (a bare `?` token, never `?.` or  // `??`, directly followed by a terminator); the parser's recovery tree then  // confirms each one and supplies its operand.  const find = (node) => {    if (      ts.isConditionalExpression(node) &&      node.questionToken.kind === K.QuestionToken &&      ts.isIdentifier(node.whenTrue) &&      node.whenTrue.text === "" &&      TERMINATORS.has(nextTokenKind(text, node.questionToken.end))    ) {      trySites.set(node, { node, question: node.questionToken.getStart(sf), operand: node.condition });    }    ts.forEachChild(node, find);  };  find(sf);  if (trySites.size === 0) {    return { text, lookup: (o) => o, diagnostics, helpers, sites: [] };  }  for (const site of trySites.values()) {    const { node } = site;    let fn = node.parent;    while (fn && !isFunctionWithBody(fn)) {      fn = fn.parent;    }    let problem = null;    if (!fn) {      problem = diag("TSUS300", site.question, "? used at top level, outside any function");    } else if (isMatchArm(fn)) {      problem = diag(        "TSUS300",        site.question,        "? used inside a match arm; arms are closures and cannot return from the enclosing function",      );    } else if (!fn.type) {      problem = diag("TSUS300", site.question, "? used in a function with no declared return type");    } else if (!returnsTryable(fn.type)) {      problem = diag(        "TSUS300",        site.question,        `? used in a function whose declared return type is ${fn.type.getText(sf)}, not Result or Option`,      );    }    // Find the statement that receives the hoisted early return, rejecting    // positions that would not always run.    let child = node;    let parent = node.parent;    let anchor = null;    while (!problem && parent) {      if (ts.isConditionalExpression(parent) && child !== parent.condition) {        problem = diag("TSUS303", site.question, "? inside a branch of ?: may not run; move it into its own statement");      } else if (        ts.isBinaryExpression(parent) &&        child === parent.right &&        SHORT_CIRCUIT.has(parent.operatorToken.kind)      ) {        problem = diag(          "TSUS303",          site.question,          `? on the right of ${parent.operatorToken.getText(sf)} may not run; move it into its own statement`,        );      } else if (        (ts.isForStatement(parent) && (child === parent.condition || child === parent.incrementor)) ||        ((ts.isWhileStatement(parent) || ts.isDoStatement(parent)) && child === parent.expression)      ) {        problem = diag("TSUS303", site.question, "? in a loop condition; move it into the loop body");      } else if (parent === fn) {        anchor = { node: child, arrowBody: true };        break;      } else if (isStatementNode(parent)) {        anchor = { node: parent, arrowBody: false };        break;      }      child = parent;      parent = parent.parent;    }    if (problem) {      diagnostics.push(problem);      site.invalid = true;      helpers.add("__tsus_unwrap");    } else {      const key = anchor.node;      if (!stmtSites.has(key)) {        stmtSites.set(key, {          sites: [],          arrowBody: anchor.arrowBody,          braces: !anchor.arrowBody && !STATEMENT_LIST_PARENTS.has(key.parent.kind),        });      }      stmtSites.get(key).sites.push(site);      helpers.add("__tsus_try");      helpers.add("__tsus_isBail");      helpers.add("__tsus_bail");    }  }  // Every ancestor of an edited node is re-rendered; everything else is  // copied verbatim.  const marked = new Set();  const markUp = (node) => {    for (let n = node; n; n = n.parent) {      marked.add(n);    }  };  for (const node of trySites.keys()) {    markUp(node);  }  for (const node of stmtSites.keys()) {    markUp(node);  }  let temp = 0;  const em = createEmitter();  const sites = [];  const renderChildren = (node) => {    const start = node.getStart(sf);    if (!marked.has(node)) {      em.copy(text.slice(start, node.end), start);      return;    }    let cursor = start;    ts.forEachChild(node, (child) => {      if (!marked.has(child)) {        return;      }      const cs = child.getStart(sf);      em.copy(text.slice(cursor, cs), cursor);      render(child);      cursor = child.end;    });    em.copy(text.slice(cursor, node.end), cursor);  };  const renderTry = (site) => {    if (site.invalid) {      em.synth("__tsus_unwrap(", site.question);      const argStart = em.length();      render(site.operand);      em.synth(")", site.question);      sites.push({ question: site.question, argStart, returnStart: -1 });      return;    }    em.synth(`${site.temp}.value`, site.operand.getStart(sf));  };  const renderStatement = (key, info) => {    // Inner and earlier operands first: an operand's own `?`s must already    // be hoisted when it is evaluated.    const ordered = [...info.sites].sort((a, b) => a.node.end - b.node.end || a.node.pos - b.node.pos);    for (const site of ordered) {      site.temp = `__tsus_t${temp}`;      temp += 1;    }    if (info.braces || info.arrowBody) {      em.synth("{ ", key.getStart(sf));    }    for (const site of ordered) {      em.synth(`const ${site.temp} = __tsus_try(`, site.question);      const argStart = em.length();      render(site.operand);      em.synth(`); if (__tsus_isBail(${site.temp})) `, site.question);      const returnStart = em.length();      em.synth(`return __tsus_bail(${site.temp}); `, site.question);      sites.push({ question: site.question, argStart, returnStart });    }    if (info.arrowBody) {      em.synth("return (", key.getStart(sf));      renderChildren(key);      em.synth("); }", key.end);    } else {      renderChildren(key);      if (info.braces) {        em.synth(" }", key.end);      }    }  };  function render(node) {    if (stmtSites.has(node)) {      renderStatement(node, stmtSites.get(node));      return;    }    const site = trySites.get(node);    if (site) {      renderTry(site);      return;    }    renderChildren(node);  }  // SourceFile.getStart skips leading trivia, so copy it explicitly.  em.copy(text.slice(0, sf.getStart(sf)), 0);  render(sf);  return { text: em.text(), lookup: segmentLookup(em.segments), diagnostics, helpers, sites };};// --- entry point ---const RUNTIME = "@tsus/runtime";// Returns the lowered TypeScript plus everything needed to map positions in// it back to the .tsus source:// - toSource(offset): lowered-code offset -> .tsus offset// - matchSites / trySites: where each macro came from, for classifying the//   TypeScript diagnostics that land on them// - diagnostics: preprocessor errors, with .tsus offsets// The runtime import is prepended on the first line (no newline), so line// numbers in the lowered code still line up with the source.export const preprocess = (source) => {  const a = lowerMatches(source);  const b = lowerTries(a.text);  const helpers = [...a.helpers, ...b.helpers];  const prefix = helpers.length > 0 ? `import { ${helpers.join(", ")} } from "${RUNTIME}"; ` : "";  const code = prefix + b.text;  const toSource = (offset) => a.lookup(b.lookup(Math.max(0, offset - prefix.length)));  const diagnostics = [...a.diagnostics, ...b.diagnostics.map((d) => ({ ...d, offset: a.lookup(d.offset) }))].sort(    (x, y) => x.offset - y.offset,  );  return {    code,    changed: code !== source,    diagnostics,    toSource,    matchSites: a.sites,    trySites: b.sites.map((s) => ({      question: a.lookup(s.question),      argStart: s.argStart + prefix.length,      returnStart: s.returnStart === -1 ? -1 : s.returnStart + prefix.length,    })),  };};