<commit_before>var sugardoll = exports;

var vm = require('vm'),
    uglify = require('uglify-js');

sugardoll.plugins = require('./sugardoll/plugins');

function getFnBody(ast) {
  return ['toplevel', ast[1][0][3]];
};

function generateFromTokens(tokens) {
  // Clone tokens
  tokens = [{type: 'keyword', value: 'function'},
            {type: 'name', value: '_'},
            {type: 'punc', value: '('},
            {type: 'punc', value: ')'},
            {type: 'punc', value: '{'}]
            .concat(tokens)
            .concat([{type: 'punc', value: '}'}]);

  function tok() {
    if (tokens.length == 0) return {type: 'eof'};
    return tokens.shift();
  };
  tok.context = function() {
    return {};
  };

  return getFnBody(uglify.parser.parse(tok));
};

function collectTokens(code, tokens) {
  var tok = uglify.parser.tokenizer(code),
      token;

  do {
    tokens.push(token = tok());
  } while (token.type !== 'eof');
};

sugardoll.tokenizer = function tokenizer(code, plugins, options) {
  var tokens = [];

  plugins || (plugins = []);

  collectTokens(code, tokens);

  for (var index = tokens.length - 1; index >= 0; index--) {
    var token = tokens[index],
        prev = tokens[index - 1] || {};

    if (token.type === 'name' &&
        (prev.type !== 'keyword' || prev.value !== 'function') &&
        (prev.type !== 'punc' || prev.value !== '.')) {
      // Caught invoke
      var plugin;
      if (plugin = plugins[token.value]) {
        function collector() {
          return tokens[++collector.index];
        };
        collector.context = function() {
          return {};
        };
        collector.index = index;

        var collected = plugin.collect(collector);

        if (collected !== false) {
          // Token name should be collected too
          if (collected === true) {
            collected = collector.index - index;
          }
          collected++;

          // Result is replacement for the matched code
          var trailing = tokens.slice(index + collected),
              collected = tokens.slice(index + 1, index + collected);

          tokens = tokens.slice(0, index);

          if (plugin.prepare) {
            collected = plugin.prepare(collected);
          }

          var ast = generateFromTokens(collected),
              code = plugin.generate.call({
                compile: function(ast) {
                  return uglify.uglify.gen_code(ast, options);
                }
              }, ast);

          if (Array.isArray(code)) {
            // code is AST
            code = uglify.uglify.gen_code(code, options);
          }

          // Collect tokens from result
          collectTokens(code, tokens);

          // Remove eof token
          tokens.pop();

          tokens = tokens.concat(trailing);
        }
      }

    }
  };

  function nextToken() {
    return tokens.shift();
  };
  nextToken.context = function() {
    return {};
  };

  return nextToken;
};

sugardoll.parse = function parse(code, plugins, options) {
  // Wrap code in function, so `return` outside function won't
  // throw error
  code = 'function _() {' + code + '}';<commit_msg>[lib] fixed typo, added _preprocess plugin, fix error when plugin collects all available tokens up to the end<commit_after>var sugardoll = exports;

var vm = require('vm'),
    uglify = require('uglify-js');

sugardoll.plugins = require('./sugardoll/plugins');

function getFnBody(ast) {
  return ['toplevel', ast[1][0][3]];
};

function generateFromTokens(tokens) {
  // Clone tokens
  tokens = [{type: 'keyword', value: 'function'},
            {type: 'name', value: '_'},
            {type: 'punc', value: '('},
            {type: 'punc', value: ')'},
            {type: 'punc', value: '{'}]
            .concat(tokens)
            .concat([{type: 'punc', value: '}'}]);

  function tok() {
    if (tokens.length == 0) return {type: 'eof'};
    return tokens.shift();
  };
  tok.context = function() {
    return {};
  };

  return getFnBody(uglify.parser.parse(tok));
};

function collectTokens(code, tokens) {
  var tok = uglify.parser.tokenizer(code),
      token;

  do {
    tokens.push(token = tok());
  } while (token.type !== 'eof');
};

sugardoll.tokenizer = function tokenizer(code, plugins, options) {
  var tokens = [];

  plugins || (plugins = {});

  collectTokens(code, tokens);

  if (plugins._preprocess) {
    tokens = plugins._preprocess(tokens);
  }

  for (var index = tokens.length - 1; index >= 0; index--) {
    var token = tokens[index],
        prev = tokens[index - 1] || {};

    if (token.type === 'name' &&
        (prev.type !== 'keyword' || prev.value !== 'function') &&
        (prev.type !== 'punc' || prev.value !== '.')) {
      // Caught invoke
      var plugin;
      if (token.value !== '_preprocess' && plugin = plugins[token.value]) {
        function collector() {
          return tokens[++collector.index];
        };
        collector.context = function() {
          return {};
        };
        collector.index = index;

        var collected = plugin.collect(collector);

        if (collected !== false) {
          // Token name should be collected too
          if (collected === true) {
            collected = collector.index - index;
          }
          collected++;

          // Result is replacement for the matched code
          var trailing = tokens.slice(index + collected),
              collected = tokens.slice(index + 1, index + collected);

          tokens = tokens.slice(0, index);

          if (plugin.prepare) {
            collected = plugin.prepare(collected);
          }

          var ast = generateFromTokens(collected),
              code = plugin.generate.call({
                compile: function(ast) {
                  return uglify.uglify.gen_code(ast, options);
                }
              }, ast);

          if (Array.isArray(code)) {
            // code is AST
            code = uglify.uglify.gen_code(code, options);
          }

          // Collect tokens from result
          collectTokens(code, tokens);

          if (trailing.length > 0) {
            // Remove eof token
            tokens.pop();
          }

          tokens = tokens.concat(trailing);
        }
      }

    }
  };

  function nextToken() {
    return tokens.shift();
  };
  nextToken.context = function() {
    return {};
  };

  return nextToken;
};

sugardoll.parse = function parse(code, plugins, options) {
  // Wrap code in function, so `return` outside function won't
  // throw error
  code = 'function _() {' + code + '}';
{'fraction_non_alphanumeric': 0.15436241610738255, 'fraction_numerical': 0.0028981086028065893, 'mean_word_length': 2.0412801484230054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5169166', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1915, 'n_words': 466}