<commit_before>const InputStream = require('./input-stream');

class TokenStream {
  constructor(input) {
    this.inputStream = new InputStream(input);
    this.current = null;
  }

  isWhitespace(char) {
    return /\s/.test(char);
  }

  isDigit(char) {
    return /\d/.test(char);
  }

  isLetter(char) {
    return /[a-z]/i.test(char);
  }

  skipWhitespace() {
    while (this.isWhitespace(this.inputStream.peek())) {
      this.inputStream.next();
    }
  }

  readNumber() {
    let str = '';
    if (this.inputStream.peek() === '-') {
      str += this.inputStream.next();
    }
    while (this.isDigit(this.inputStream.peek())) {
      str += this.inputStream.next();
    }
    if (this.inputStream.peek() === '.') {
      str += this.inputStream.next();
      while (this.isDigit(this.inputStream.peek())) {
        str += this.inputStream.next();
      }
    }
    return parseFloat(str);
  }

  readSymbol() {
    let str = '';
    while (this.isLetter(this.inputStream.peek()) || (this.inputStream.peek() === '-' && this.isLetter(this.inputStream.lookAhead()))) {
      str += this.inputStream.next();
    }
    return str;
  }

  peek() {
    if (this.current === null) {
      this.current = this.readToken();
    }<commit_msg>Do not allow numbers to end with a decimal<commit_after>const InputStream = require('./input-stream');

class TokenStream {
  constructor(input) {
    this.inputStream = new InputStream(input);
    this.current = null;
  }

  isWhitespace(char) {
    return /\s/.test(char);
  }

  isDigit(char) {
    return /\d/.test(char);
  }

  isLetter(char) {
    return /[a-z]/i.test(char);
  }

  skipWhitespace() {
    while (this.isWhitespace(this.inputStream.peek())) {
      this.inputStream.next();
    }
  }

  readNumber() {
    let str = '';
    if (this.inputStream.peek() === '-') {
      str += this.inputStream.next();
    }
    while (this.isDigit(this.inputStream.peek())) {
      str += this.inputStream.next();
    }
    if (this.inputStream.peek() === '.' && this.isDigit(this.inputStream.lookAhead())) {
      str += this.inputStream.next();
      while (this.isDigit(this.inputStream.peek())) {
        str += this.inputStream.next();
      }
    }
    return parseFloat(str);
  }

  readSymbol() {
    let str = '';
    while (this.isLetter(this.inputStream.peek()) || (this.inputStream.peek() === '-' && this.isLetter(this.inputStream.lookAhead()))) {
      str += this.inputStream.next();
    }
    return str;
  }

  peek() {
    if (this.current === null) {
      this.current = this.readToken();
    }
{'fraction_non_alphanumeric': 0.18781725888324874, 'fraction_numerical': 0.0, 'mean_word_length': 2.6186440677966103, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1636347', 'n_tokens_mistral': 909, 'n_tokens_neox': 884, 'n_words': 133}