<commit_before>class Lexer {
  constructor(input) {
    this.input = input;
    this.position = 0;
    this.line = 1;
    this.column = 1;
  }

  die(message) {
    throw new Error(`${message} (${this.line}:${this.column})`);
  }

  peek() {
    return this.input.charAt(this.position);
  }

  next() {
    const char = this.input.charAt(this.position);
    this.position += 1;
    if (char === '\n') {
      this.line += 1;
      this.column = 1;
    } else {
      this.column += 1;
    }
    return char;
  }

  lookAhead() {
    return this.input.charAt(this.position + 1);
  }

  eof() {
    return this.peek() === '';
  }

  isWhitespace(char) {
    return /\s/.test(char);
  }

  isDigit(char) {
    return /\d/.test(char);
  }

  isAlpha(char) {
    return /[a-z]/i.test(char);
  }

  skipWhitespace() {
    while (this.isWhitespace(this.peek())) {
      this.next();
    }
  }

  readNumber() {
    let str = '';
    if (this.peek() === '-') {
      str += this.next();
    }
    while (this.isDigit(this.peek())) {
      str += this.next();
    }
    if (this.peek() === '.' && this.isDigit(this.lookAhead())) {
      str += this.next();
      while (this.isDigit(this.peek())) {
        str += this.next();
      }
    }
    return parseFloat(str);
  }

  readSymbol() {
    let str = '';
    if (this.peek() === '_') {
      return this.next();
    }
    while (this.isAlpha(this.peek()) || (this.peek() === '-' && this.isAlpha(this.lookAhead()))) {
      str += this.next();
    }
    return str;
  }

  readString() {
    let str = '';
    this.next();
    while (this.peek() && this.peek() !== '"') {
      str += this.next();
    }
    if (this.peek() === '"') {
      this.next();
    } else {
      throw new Error('unterminated string');
    }
    return str;
  }

  readToken() {
    this.skipWhitespace();
    if (this.eof()) {
      return null;
    }
    const char = this.peek();

    if (this.isDigit(char) || (char === '-' && this.isDigit(this.lookAhead()))) {<commit_msg>Support newline characters in strings with \n<commit_after>class Lexer {
  constructor(input) {
    this.input = input;
    this.position = 0;
    this.line = 1;
    this.column = 1;
  }

  die(message) {
    throw new Error(`${message} (${this.line}:${this.column})`);
  }

  peek() {
    return this.input.charAt(this.position);
  }

  next() {
    const char = this.input.charAt(this.position);
    this.position += 1;
    if (char === '\n') {
      this.line += 1;
      this.column = 1;
    } else {
      this.column += 1;
    }
    return char;
  }

  lookAhead() {
    return this.input.charAt(this.position + 1);
  }

  eof() {
    return this.peek() === '';
  }

  isWhitespace(char) {
    return /\s/.test(char);
  }

  isDigit(char) {
    return /\d/.test(char);
  }

  isAlpha(char) {
    return /[a-z]/i.test(char);
  }

  skipWhitespace() {
    while (this.isWhitespace(this.peek())) {
      this.next();
    }
  }

  readNumber() {
    let str = '';
    if (this.peek() === '-') {
      str += this.next();
    }
    while (this.isDigit(this.peek())) {
      str += this.next();
    }
    if (this.peek() === '.' && this.isDigit(this.lookAhead())) {
      str += this.next();
      while (this.isDigit(this.peek())) {
        str += this.next();
      }
    }
    return parseFloat(str);
  }

  readSymbol() {
    let str = '';
    if (this.peek() === '_') {
      return this.next();
    }
    while (this.isAlpha(this.peek()) || (this.peek() === '-' && this.isAlpha(this.lookAhead()))) {
      str += this.next();
    }
    return str;
  }

  readString() {
    let str = '';
    this.next();
    while (this.peek() && this.peek() !== '"') {
      if (this.peek() === '\\' && this.lookAhead() === 'n') {
        str += '\n';
        this.next();
        this.next();
      } else {
        str += this.next();
      }
    }
    if (this.peek() === '"') {
      this.next();
    } else {
      throw new Error('unterminated string');
    }
    return str;
  }

  readToken() {
    this.skipWhitespace();
    if (this.eof()) {
      return null;
    }
    const char = this.peek();

    if (this.isDigit(char) || (char === '-' && this.isDigit(this.lookAhead()))) {
{'fraction_non_alphanumeric': 0.21325993298228818, 'fraction_numerical': 0.003829583532790809, 'mean_word_length': 2.043699927166788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6132428', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1542, 'n_words': 267}