justintsus-experiment Public
// Splitting a query into its terms, as written: where each one starts and ends, whether it is// negated, and whether it is bare text, a quoted phrase or a qualifier with a quoted value. Nothing// is interpreted here (parse.tsus decides what a term means); a term whose quotes are wrong comes// back as Broken with its span.import { Some, None, type Option } from "@tsus/runtime";export type Raw = // bare text up to the next whitespace, without the leading "-" of a negated term | { readonly tag: "Word"; readonly start: number; readonly end: number; readonly negated: boolean; readonly text: string } // a quoted value standing alone: free text | { readonly tag: "Phrase"; readonly start: number; readonly end: number; readonly negated: boolean; readonly text: string } // key:"quoted value"; prefix is the key as written, without its colon | { readonly tag: "Keyed"; readonly start: number; readonly end: number; readonly negated: boolean; readonly prefix: string; readonly text: string } | { readonly tag: "Broken"; readonly start: number; readonly end: number; readonly code: "unterminated_quote" | "unexpected_quote" };const WS = /\s/;const KEY_COLON = /^[A-Za-z]+(-[A-Za-z]+)*:$/;const isWs = (c: string): boolean => c !== "" && WS.test(c);// The first index at or after `from` that is whitespace, or the end of the input.const nextWs = (s: string, from: number): number => { let i = from; while (i < s.length && !isWs(s.charAt(i))) i += 1; return i;};// The first index at or after `from` that is whitespace or a double quote, or the end.const nextStop = (s: string, from: number): number => { let i = from; while (i < s.length && !isWs(s.charAt(i)) && s.charAt(i) !== "\"") i += 1; return i;};type Quoted = { readonly text: string; readonly next: number };// A quoted value whose opening quote is just before `from`: \" is a quote, \\ a backslash, any other// backslash stays. None when no unescaped quote closes it.const readQuoted = (s: string, from: number): Option<Quoted> => { let i = from; let text = ""; while (i < s.length) { const c = s.charAt(i); if (c === "\"") return Some({ text, next: i + 1 }); const d = s.charAt(i + 1); if (c === "\\" && (d === "\"" || d === "\\")) { text += d; i += 2; } else { text += c; i += 1; } } return None;};type Step = { readonly raw: Raw; readonly next: number };const broken = (start: number, end: number, code: "unterminated_quote" | "unexpected_quote"): Step => ({ raw: { tag: "Broken", start, end, code }, next: end });// After a closing quote the term must end; otherwise the whole term, to the next whitespace, is an// unexpected quote.const closed = (s: string, start: number, q: Quoted, make: (text: string, end: number) => Raw): Step => q.next < s.length && !isWs(s.charAt(q.next)) ? broken(start, nextWs(s, q.next), "unexpected_quote") : { raw: make(q.text, q.next), next: q.next };const quoted = (s: string, start: number, from: number, make: (text: string, end: number) => Raw): Step => match (readQuoted(s, from)) { Some(q) => closed(s, start, q, make), None => broken(start, s.length, "unterminated_quote"), };const term = (s: string, start: number): Step => { const negated = s.charAt(start) === "-" && start + 1 < s.length && !isWs(s.charAt(start + 1)); const body = negated ? start + 1 : start; if (s.charAt(body) === "\"") { return quoted(s, start, body + 1, (text, end): Raw => ({ tag: "Phrase", start, end, negated, text })); } const stop = nextStop(s, body); if (s.charAt(stop) !== "\"") return { raw: { tag: "Word", start, end: stop, negated, text: s.slice(body, stop) }, next: stop }; const prefix = s.slice(body, stop); if (!KEY_COLON.test(prefix)) return broken(start, nextWs(s, stop), "unexpected_quote"); return quoted(s, start, stop + 1, (text, end): Raw => ({ tag: "Keyed", start, end, negated, prefix: prefix.slice(0, -1), text }));};// Every term of the input, in order. An unterminated quote runs to the end of the input, so it is// always the last term.export const lex = (s: string): readonly Raw[] => { const out: Raw[] = []; let i = 0; while (i < s.length) { if (isWs(s.charAt(i))) { i += 1; } else { const step = term(s, i); out.push(step.raw); i = step.next; } } return out;};