<commit_before>library Lexer;

import "../../codeunitstream/lib/codeunitstream.dart"; 
import "../../str/lib/str.dart";
import "../../lang/lib/lang.dart";


class LexerStatus {
  
  var lexer, tokenizer, spaceTokenizer;
  
  LexerStatus({this.lexer}) {
    tokenizer = lexer.entryTokenizer;
    spaceTokenizer = lexer.spaceTokenizer;
  }
  
}


class LexerCommon {
  
  var entryTokenizer, defaultTokenizer, spaceTokenizer;

  doParseStream(stream, status, resultFn(tokenType)) {
    var tt;
    while (!stream.isEol) {
      tt = null;
      if (status.spaceTokenizer != null) {
        tt = status.spaceTokenizer(stream, status);
      }
      if (tt == null) {
        tt = status.tokenizer(stream, status);
        if (tt == null) {
          status.tokenizer = defaultTokenizer;
          tt = defaultTokenizer(stream, status);
        }
      }
      resultFn(tt);
      stream.startIndex = stream.currentIndex;
    }
  }
  
  parseTokenStrings(stream, status, resultFn(tokenType, tokenString)) {
    parse(stream, status, (tt) => resultFn(tt, stream.currentTokenString));
  }
  
  parse(stream, status, resultFn(tokenType)) {
    var s = stream.text, i = Str.indexOfNewLine(s),
      len = s.length, si = 0;
    while (i >= 0) {
      if (i - si > 1) {
        stream.lineEndIndex = i;
        doParseStream(stream, status, resultFn);
      }
      si = i + 1;
      stream.startIndex = si;
      stream.currentIndex = si;
      stream.lineStartIndex = si;
      i = Str.indexOfNewLine(s, si);
    }
    if (len > si) {
      stream.lineEndIndex = len;
      doParseStream(stream, status, resultFn);
    }
  }
  
  spawnStatus() {
    return new LexerStatus(lexer: this);<commit_msg>Refactored LexerCommon by giving a parseLine name to one of its methods.<commit_after>library Lexer;

import "../../codeunitstream/lib/codeunitstream.dart"; 
import "../../str/lib/str.dart";
import "../../lang/lib/lang.dart";


class LexerStatus {
  
  var lexer, tokenizer, spaceTokenizer;
  
  LexerStatus({this.lexer}) {
    tokenizer = lexer.entryTokenizer;
    spaceTokenizer = lexer.spaceTokenizer;
  }
  
}


class LexerCommon {
  
  var entryTokenizer, defaultTokenizer, spaceTokenizer;

  parseLine(stream, status, resultFn(tokenType)) {
    var tt;
    while (!stream.isEol) {
      tt = null;
      if (status.spaceTokenizer != null) {
        tt = status.spaceTokenizer(stream, status);
      }
      if (tt == null) {
        tt = status.tokenizer(stream, status);
        if (tt == null) {
          status.tokenizer = defaultTokenizer;
          tt = defaultTokenizer(stream, status);
        }
      }
      resultFn(tt);
      stream.startIndex = stream.currentIndex;
    }
  }
  
  parseTokenStrings(stream, status, resultFn(tokenType, tokenString)) {
    parse(stream, status, (tt) => resultFn(tt, stream.currentTokenString));
  }
  
  parse(stream, status, resultFn(tokenType)) {
    var s = stream.text, i = Str.indexOfNewLine(s),
      len = s.length, si = 0;
    while (i >= 0) {
      if (i - si > 1) {
        stream.lineEndIndex = i;
        parseLine(stream, status, resultFn);
      }
      si = i + 1;
      stream.startIndex = si;
      stream.currentIndex = si;
      stream.lineStartIndex = si;
      i = Str.indexOfNewLine(s, si);
    }
    if (len > si) {
      stream.lineEndIndex = len;
      parseLine(stream, status, resultFn);
    }
  }
  
  spawnStatus() {
    return new LexerStatus(lexer: this);
{'fraction_non_alphanumeric': 0.1321855850598191, 'fraction_numerical': 0.0023344032681645753, 'mean_word_length': 2.494393476044852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '473496', 'n_tokens_mistral': 1142, 'n_tokens_neox': 1129, 'n_words': 240}