<commit_before>export class Token {
  constructor(index, text) {
    this.index = index;
    this.text = text;
  }

  withOp(op) {
    this.opKey = op;
    return this;
  }

  withGetterSetter(key) {
    this.key = key;
    return this;
  }

  withValue(value) {
    this.value = value;
    return this;
  }

  toString() {
    return `Token(${this.text})`;
  }
}

export class Lexer {
  lex(text, opts) {
    let scanner = new Scanner(text, opts);
    let tokens = [];
    let token = scanner.scanToken();

    while (token) {
      tokens.push(token);
      token = scanner.scanToken();
    }

    return tokens;
  }
}

const CONTEXT_NIL = 0; // out side of any context
const CONTEXT_OBJECT = 1; // between { and } for object
const CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION = 2; // between ` and ` but out of ${ and }
const CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION = 3; // between ${ and } in ``

export class Scanner {
  constructor(input, opts = {}) {
    this.input = input;
    this.length = input.length;
    this.peek = 0;
    this.index = -1;
    this.stringInterpolationMode = opts.stringInterpolationMode || false;

    const initialContext = this.stringInterpolationMode ?
                           CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION :
                           CONTEXT_NIL;
    this.contextStack = [initialContext];

    this.advance();
  }

  get context() {
    return this.contextStack[this.contextStack.length - 1];
  }

  get isRootLevelOfStringInterpolationMode() {
    return this.stringInterpolationMode && this.contextStack.length === 1;
  }

  scanToken() {
    let start = this.index;

    if (this.context !== CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION) {
      // normal mode, at least not in string part of interpolation

      // Skip whitespace.
      while (this.peek <= $SPACE) {
        if (++this.index >= this.length) {
          this.peek = $EOF;
          return null;
        }

        this.peek = this.input.charCodeAt(this.index);
      }

      // Handle identifiers and numbers.
      if (isIdentifierStart(this.peek)) {
        return this.scanIdentifier();
      }

      if (isDigit(this.peek)) {
        return this.scanNumber(this.index);
      }

      switch (this.peek) {
      case $PERIOD:
        this.advance();
        return isDigit(this.peek) ? this.scanNumber(start) : new Token(start, '.');

      case $LBRACE:
        this.contextStack.push(CONTEXT_OBJECT);
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $RBRACE:
        // '}' has two meanings,
        // in CONTEXT_OBJECT, means close of object,
        // in CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION, means close of interpolation
        this.contextStack.pop();
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $LPAREN:
      case $RPAREN:
      case $LBRACKET:
      case $RBRACKET:
      case $COMMA:
      case $COLON:
      case $SEMICOLON:
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $SQ:
      case $DQ:
        return this.scanString();
      case $PLUS:
      case $MINUS:
      case $STAR:
      case $SLASH:
      case $PERCENT:
      case $CARET:
      case $QUESTION:
        return this.scanOperator(start, String.fromCharCode(this.peek));
      case $LT:
      case $GT:
      case $BANG:
      case $EQ:
        return this.scanComplexOperator(start, $EQ, String.fromCharCode(this.peek), '=');
      case $AMPERSAND:
        return this.scanComplexOperator(start, $AMPERSAND, '&', '&');
      case $BAR:
        return this.scanComplexOperator(start, $BAR, '|', '|');
      case $BACKTICK:
        this.contextStack.push(CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION);
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $NBSP:
        while (isWhitespace(this.peek)) {
          this.advance();
        }

        return this.scanToken();
        // no default
      }

      let character = String.fromCharCode(this.peek);
      this.error(`Unexpected character [${character}]`);
      return null;
    } else {
      // in string interpolation's string part
      if (this.peek === $BACKTICK) {
        // in stringInterpolationMode, root level doesn't close at backtick
        if (this.isRootLevelOfStringInterpolationMode) {
          // backtick is part of root level string
          return this.scanString();
        } else {
          this.contextStack.pop();
          return this.scanCharacter(start, String.fromCharCode(this.peek));
        }
      } else if (this.isStartOfInterpolation()) {
        const token = this.scanStartOfInterpolation();
        this.contextStack.push(CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION);
        return token;
      } else if (this.peek) {
        return this.scanString();
      }
    }
  }

  scanCharacter(start, text) {
    assert(this.peek === text.charCodeAt(0));
    this.advance();
    return new Token(start, text);
  }<commit_msg>add comment.<commit_after>export class Token {
  constructor(index, text) {
    this.index = index;
    this.text = text;
  }

  withOp(op) {
    this.opKey = op;
    return this;
  }

  withGetterSetter(key) {
    this.key = key;
    return this;
  }

  withValue(value) {
    this.value = value;
    return this;
  }

  toString() {
    return `Token(${this.text})`;
  }
}

export class Lexer {
  lex(text, opts) {
    let scanner = new Scanner(text, opts);
    let tokens = [];
    let token = scanner.scanToken();

    while (token) {
      tokens.push(token);
      token = scanner.scanToken();
    }

    return tokens;
  }
}

const CONTEXT_NIL = 0; // out side of any context
const CONTEXT_OBJECT = 1; // between { and } for object
const CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION = 2; // between ` and ` but out of ${ and }
const CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION = 3; // between ${ and } in ``
// currently CONTEXT_NIL, CONTEXT_OBJECT, CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION
// are treated same

export class Scanner {
  constructor(input, opts = {}) {
    this.input = input;
    this.length = input.length;
    this.peek = 0;
    this.index = -1;
    this.stringInterpolationMode = opts.stringInterpolationMode || false;

    const initialContext = this.stringInterpolationMode ?
                           CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION :
                           CONTEXT_NIL;
    this.contextStack = [initialContext];

    this.advance();
  }

  get context() {
    return this.contextStack[this.contextStack.length - 1];
  }

  get isRootLevelOfStringInterpolationMode() {
    return this.stringInterpolationMode && this.contextStack.length === 1;
  }

  scanToken() {
    let start = this.index;

    if (this.context !== CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION) {
      // normal mode, at least not in string part of interpolation

      // Skip whitespace.
      while (this.peek <= $SPACE) {
        if (++this.index >= this.length) {
          this.peek = $EOF;
          return null;
        }

        this.peek = this.input.charCodeAt(this.index);
      }

      // Handle identifiers and numbers.
      if (isIdentifierStart(this.peek)) {
        return this.scanIdentifier();
      }

      if (isDigit(this.peek)) {
        return this.scanNumber(this.index);
      }

      switch (this.peek) {
      case $PERIOD:
        this.advance();
        return isDigit(this.peek) ? this.scanNumber(start) : new Token(start, '.');

      case $LBRACE:
        this.contextStack.push(CONTEXT_OBJECT);
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $RBRACE:
        // '}' has two meanings,
        // in CONTEXT_OBJECT, means close of object,
        // in CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION, means close of interpolation
        this.contextStack.pop();
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $LPAREN:
      case $RPAREN:
      case $LBRACKET:
      case $RBRACKET:
      case $COMMA:
      case $COLON:
      case $SEMICOLON:
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $SQ:
      case $DQ:
        return this.scanString();
      case $PLUS:
      case $MINUS:
      case $STAR:
      case $SLASH:
      case $PERCENT:
      case $CARET:
      case $QUESTION:
        return this.scanOperator(start, String.fromCharCode(this.peek));
      case $LT:
      case $GT:
      case $BANG:
      case $EQ:
        return this.scanComplexOperator(start, $EQ, String.fromCharCode(this.peek), '=');
      case $AMPERSAND:
        return this.scanComplexOperator(start, $AMPERSAND, '&', '&');
      case $BAR:
        return this.scanComplexOperator(start, $BAR, '|', '|');
      case $BACKTICK:
        this.contextStack.push(CONTEXT_TEXT_PART_OF_STRING_INTERPOLATION);
        return this.scanCharacter(start, String.fromCharCode(this.peek));
      case $NBSP:
        while (isWhitespace(this.peek)) {
          this.advance();
        }

        return this.scanToken();
        // no default
      }

      let character = String.fromCharCode(this.peek);
      this.error(`Unexpected character [${character}]`);
      return null;
    } else {
      // in string interpolation's text part
      if (this.peek === $BACKTICK) {
        // in stringInterpolationMode, root level doesn't close at backtick
        if (this.isRootLevelOfStringInterpolationMode) {
          // backtick is part of root level string
          return this.scanString();
        } else {
          this.contextStack.pop();
          return this.scanCharacter(start, String.fromCharCode(this.peek));
        }
      } else if (this.isStartOfInterpolation()) {
        const token = this.scanStartOfInterpolation();
        this.contextStack.push(CONTEXT_INTERPOLATION_PART_OF_STRING_INTERPOLATION);
        return token;
      } else if (this.peek) {
        return this.scanString();
      }
    }
  }

  scanCharacter(start, text) {
    assert(this.peek === text.charCodeAt(0));
    this.advance();
    return new Token(start, text);
  }
{'fraction_non_alphanumeric': 0.1171875, 'fraction_numerical': 0.0017800632911392405, 'mean_word_length': 2.428135593220339, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1143344', 'n_tokens_mistral': 3207, 'n_tokens_neox': 3143, 'n_words': 678}