<commit_before>/**
 * Liquor - Lexer
 * Copyright (c) 2011, <NAME>. (MIT Licensed)
 */

var lexer = function(str, options) {
  str = str
    .replace(/\r\n|\r/g, '\n')
    .replace(/"/g, '\\"');

  var i = 0
    , l = str.length
    , ch
    , buff = ''
    , key
    , line = 1
    , offset = 0
    , tokens = []
    , stack = []
    , indent
    , indents = []
    , newline = true
    , indentSize;

  if (options.pretty && options.indent) {
    str = applyIndent(str, options.indent);
    l = str.length;
  }

  var state = function() {
    return stack[stack.length-1];
  };

  var out = function() {
    if (!options.pretty) return buff;
    var i = indents.length * indentSize;
    if (!i) return buff;
    return applyOutdent(buff, i);
  };

  for (; i < l; i++) {
    ch = str[i];
    offset++;

    if (ch > ' ' && newline) {
      if (!indentSize && indent != null && indent < buff.length) {
        indentSize = buff.length - indent;
      }
      indent = buff.length;
      // assert(indent % indentSize === 0);
      while (indents[indents.length-1] >= indent) {
        tokens.push({
          type: 'end',
          line: line
        });
        indents.pop();
      }
      if (!options.pretty) buff = '';
      newline = false;
    }

    switch(ch) {
      case '\n':
        line++;
        offset = 0;
        switch (state()) {
          case 'evaluate':
          case 'interpolate':
            buff += ch;
            break;
          default:
            if (stack.length) {
              tokens.push({
                type: stack.pop(),
                name: buff,
                line: line
              });
            } else {
              if (options.pretty) buff += '\\n';
              tokens.push({
                type: 'text',
                text: out(),
                line: line
              });
            }
            buff = '';
            newline = true;
            break;
        }
        break;
      case '@':
      case '?':
      case '!':
        if (str[i+1] === ':' && !buff.trim()) {
          switch (ch) {
            case '@':
              stack.push('iterate');
              break;
            case '?':
              stack.push('if');
              break;
            case '!':
              stack.push('not');
              break;
          }
          indents.push(indent);
          i++;
          buff = '';
        } else {
          buff += ch;
        }
        break;
      case '`':
        switch (state()) {
          case 'evaluate':
            tokens.push({
              type: 'evaluate',
              code: buff,
              line: line
            });
            buff = '';
            stack.pop();
            break;
          case 'interpolate':
            buff += ch;
            break;
          default:
            tokens.push({
              type: 'text',
              text: out(),
              line: line
            });
            buff = '';
            stack.push('evaluate');
            break;
        }
        break;
      case '#':
        if (str[i+1] === '{') {
          switch (state()) {
            case 'interpolate':
            case 'evaluate':
              buff += ch;
              break;
            default:
              tokens.push({
                type: 'text',
                text: out(),
                line: line
              });
              buff = '';
              i++;
              stack.push('interpolate');
              break;
          }
        } else {
          buff += ch;
        }
        break;
      case '}':
        switch (state()) {
          case 'interpolate':
            tokens.push({
              type: 'interpolate',
              code: buff,
              line: line
            });
            buff = '';
            stack.pop();
            break;
          default:
            buff += ch;
            break;
        }
        break;
      default:
        buff += ch;
        break;
    }
  }

  if (buff) {
    tokens.push({
      type: 'text',
      text: out(),
      line: line
    });
  }

  return tokens;
};

var applyOutdent = function(str, n) {
  if (!n) return str;
  return str.replace(new RegExp('^[ \t]{' + n + '}', 'gm'), '');
};

var applyIndent = function(str, n) {
  if (!n) return str;
  var s = '';
  while (n--) s += ' ';
  return str.replace(/^/gm, s);
};

/**<commit_msg>drop regexes<commit_after>/**
 * Liquor - Lexer
 * Copyright (c) 2011, <NAME>. (MIT Licensed)
 */

var lexer = function(str, options) {
  var i = 0
    , l = str.length
    , ch
    , buff = ''
    , key
    , line = 1
    , offset = 0
    , tokens = []
    , stack = []
    , indent
    , indents = []
    , newline = true
    , indentSize;

  if (options.pretty && options.indent) {
    str = applyIndent(str, options.indent);
    l = str.length;
  }

  var state = function() {
    return stack[stack.length-1];
  };

  var out = function() {
    if (!options.pretty) return buff;
    var i = indents.length * indentSize;
    if (!i) return buff;
    return applyOutdent(buff, i);
  };

  for (; i < l; i++) {
    ch = str[i];
    offset++;

    if (ch > ' ' && newline) {
      if (!indentSize && indent != null && indent < buff.length) {
        indentSize = buff.length - indent;
      }
      indent = buff.length;
      // assert(indent % indentSize === 0);
      while (indents[indents.length-1] >= indent) {
        tokens.push({
          type: 'end',
          line: line
        });
        indents.pop();
      }
      if (!options.pretty) buff = '';
      newline = false;
    }

    switch(ch) {
      case '\r':
        if (str[i+1] === '\n') break;
        ;
      case '\n':
        line++;
        offset = 0;
        switch (state()) {
          case 'evaluate':
          case 'interpolate':
            buff += ch;
            break;
          default:
            if (stack.length) {
              tokens.push({
                type: stack.pop(),
                name: buff,
                line: line
              });
            } else {
              if (options.pretty) buff += '\\n';
              tokens.push({
                type: 'text',
                text: out(),
                line: line
              });
            }
            buff = '';
            newline = true;
            break;
        }
        break;
      case '@':
      case '?':
      case '!':
        if (str[i+1] === ':' && !buff.trim()) {
          switch (ch) {
            case '@':
              stack.push('iterate');
              break;
            case '?':
              stack.push('if');
              break;
            case '!':
              stack.push('not');
              break;
          }
          indents.push(indent);
          i++;
          buff = '';
        } else {
          buff += ch;
        }
        break;
      case '`':
        switch (state()) {
          case 'evaluate':
            tokens.push({
              type: 'evaluate',
              code: buff,
              line: line
            });
            buff = '';
            stack.pop();
            break;
          case 'interpolate':
            buff += ch;
            break;
          default:
            tokens.push({
              type: 'text',
              text: out(),
              line: line
            });
            buff = '';
            stack.push('evaluate');
            break;
        }
        break;
      case '#':
        if (str[i+1] === '{') {
          switch (state()) {
            case 'interpolate':
            case 'evaluate':
              buff += ch;
              break;
            default:
              tokens.push({
                type: 'text',
                text: out(),
                line: line
              });
              buff = '';
              i++;
              stack.push('interpolate');
              break;
          }
        } else {
          buff += ch;
        }
        break;
      case '}':
        switch (state()) {
          case 'interpolate':
            tokens.push({
              type: 'interpolate',
              code: buff,
              line: line
            });
            buff = '';
            stack.pop();
            break;
          default:
            buff += ch;
            break;
        }
        break;
      case '"':
        switch (state()) {
          case 'interpolate':
          case 'evaluate':
            buff += '"';
            break;
          case 'if':
          case 'not':
          case 'iterate':
          default:
            buff += '\\"';
            break;
        }
        break;
      default:
        buff += ch;
        break;
    }
  }

  if (buff) {
    tokens.push({
      type: 'text',
      text: out(),
      line: line
    });
  }

  return tokens;
};

var applyOutdent = function(str, n) {
  if (!n) return str;
  return str.replace(new RegExp('^[ \t]{' + n + '}', 'gm'), '');
};

var applyIndent = function(str, n) {
  if (!n) return str;
  var s = '';
  while (n--) s += ' ';
  return str.replace(/^/gm, s);
};

/**
{'fraction_non_alphanumeric': 0.15187486132682493, 'fraction_numerical': 0.00299534058131795, 'mean_word_length': 1.0349887133182845, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '447495', 'n_tokens_mistral': 2663, 'n_tokens_neox': 2573, 'n_words': 615}