wellformed

import { FormulaError } from './ast.js';

export type TokenKind = 'num' | 'str' | 'ident' | 'punct' | 'eof';

export interface Token {
  kind: TokenKind;
  /** For `str` this is the decoded value; otherwise the source text. */
  value: string;
  pos: number;
}

const PUNCT = ['>=', '<=', '<>', '!=', '(', ')', ',', '+', '-', '*', '/', '&', '<', '>', '='];
const IDENT_START = /[A-Za-z_]/;
const IDENT_PART = /[A-Za-z0-9_]/;

/**
 * Tokenize a formula. Unlike the prototype's sticky-regex scanner, an
 * unrecognised character is an error rather than a silent truncation — a typo
 * must not quietly change what a column computes.
 */
export function lex(src: string): Token[] {
  const tokens: Token[] = [];
  let i = 0;

  while (i < src.length) {
    const ch = src[i]!;

    if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
      i++;
      continue;
    }

    if (ch === '"' || ch === "'") {
      const quote = ch;
      const start = i;
      i++;
      let out = '';
      let closed = false;
      while (i < src.length) {
        if (src[i] === '\\' && i + 1 < src.length) {
          out += src[i + 1];
          i += 2;
          continue;
        }
        if (src[i] === quote) {
          // A doubled quote is an escaped quote, SQL-style.
          if (src[i + 1] === quote) {
            out += quote;
            i += 2;
            continue;
          }
          i++;
          closed = true;
          break;
        }
        out += src[i];
        i++;
      }
      if (!closed) throw new FormulaError('Unterminated string literal', start);
      tokens.push({ kind: 'str', value: out, pos: start });
      continue;
    }

    if (/[0-9]/.test(ch) || (ch === '.' && /[0-9]/.test(src[i + 1] ?? ''))) {
      const start = i;
      while (i < src.length && /[0-9]/.test(src[i]!)) i++;
      if (src[i] === '.') {
        i++;
        while (i < src.length && /[0-9]/.test(src[i]!)) i++;
      }
      tokens.push({ kind: 'num', value: src.slice(start, i), pos: start });
      continue;
    }

    if (IDENT_START.test(ch)) {
      const start = i;
      while (i < src.length && (IDENT_PART.test(src[i]!) || src[i] === '.')) {
        if (src[i] === '.') {
          if (!IDENT_START.test(src[i + 1] ?? '')) break;
        }
        i++;
      }
      tokens.push({ kind: 'ident', value: src.slice(start, i), pos: start });
      continue;
    }

    const punct = PUNCT.find((p) => src.startsWith(p, i));
    if (punct) {
      tokens.push({ kind: 'punct', value: punct === '<>' ? '!=' : punct, pos: i });
      i += punct.length;
      continue;
    }

    throw new FormulaError(`Unexpected character ${JSON.stringify(ch)}`, i);
  }

  tokens.push({ kind: 'eof', value: '', pos: src.length });
  return tokens;
}