<commit_before>var Lexer = module.exports = function Lexer(str) {
  this.tokens = [];
  this.line = 1;
  this.indented = 0;
  this.str = str
    .trim()
    .replace(/\r\n|\r|\n *\n/g, '\n');
}

Lexer.prototype = {

  exec: function () {
    while (this.str.length) {
      this.next();
    }
  },

  token: function (type, line, match, value) {
    return {
      type: type
      , line: line
      , match: match
      , value: value
    }
  },

  push: function (type, line, match, value) {
    this.tokens.push(this.token(type, line, match, value));
  },

  scan: function (regexp, type, callback) {
    var captures
      , token;
    if (captures = regexp.exec(this.str)) {
      this.str = this.str.substr(captures[0].length);
      this.push(type, this.line, captures[0], captures[1]);
      if (typeof callback == 'function') {
        callback.call(this, captures);
      }
    }
  },

  next: function () {
    return this.scan(/^\n( *)(?! *-#)/, 'indent', this.indent)
      || this.scan(/^\{(.*?)\}/, 'attrs')
      || this.scan(/^([a-zA-Z][a-zA-Z0-9:]*)/, 'tag')
      || this.scan(/^\- *block (\w+)*/, 'block')
      || this.scan(/^\.([\w\-]+)/, 'class')
      || this.scan(/^\#([\w\-]+)/, 'id')
      || this.scan(/^([^\n]+)/, 'text')
  },

  indent: function (captures) {
    var indents = captures[1].length / 2;
    this.line++;
    if (indents % 1) {
      throw(new Error('indentation line ' + this.line));
    }
  }

}<commit_msg>scan returns true and outdent recognition<commit_after>var Lexer = module.exports = function Lexer(str) {
  this.tokens = [];
  this.line = 1;
  this.indented = 0;
  this.str = str
    .trim()
    .replace(/\r\n|\r|\n *\n/g, '\n');
}

Lexer.prototype = {

  exec: function () {
    while (this.str.length) {
      this.next();
    }
    return this.tokens;
  },

  token: function (type, line, match, value) {
    return {
      type: type
      , line: line
      , match: match
      , value: value
    }
  },

  push: function (type, line, match, value) {
    this.tokens.push(this.token(type, line, match, value));
  },

  scan: function (regexp, type, callback) {
    var captures
      , token;
    if (captures = regexp.exec(this.str)) {
      this.str = this.str.substr(captures[0].length);
      if (typeof callback == 'function') {
        callback.call(this, captures);
      } else {
        this.push(type, this.line, captures[0], captures[1]);
      }
      return true;
    }
  },

  next: function () {
    return this.scan(/^\n( *)(?! *-#)/, 'indent', this.indent)
      || this.scan(/^\- *block (\w+)*/, 'block')
      || this.scan(/^\{(.*)\}/, 'attrs')
      || this.scan(/^([a-zA-Z][a-zA-Z0-9:]*)/, 'tag')
      || this.scan(/^\.([\w\-]+)/, 'class')
      || this.scan(/^\#([\w\-]+)/, 'id')
      || this.scan(/^([^\n]+)/, 'text')
  },

  indent: function (captures) {
    var indents = captures[1].length / 2;
    this.line++;
    if (indents % 1) {
      throw(new Error('indentation line ' + this.line));
    } else if (this.indented > indents) {
      while (this.indented-- > indents) {
        this.push("outdent", this.line);
      }
    } else if (this.indented !== indents) {
      this.push("indent", this.line);
    }
    this.indented = indents;
  }

}
{'fraction_non_alphanumeric': 0.22150139017608897, 'fraction_numerical': 0.006178560395427865, 'mean_word_length': 2.215491559086395, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2166161', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1175, 'n_words': 237}