<commit_before>var debug = require('debug')('vmplate');
var vm = require('vm');

module.exports = function vmplate(string, filename) {

  var script = vm.createScript(compile(string), filename || 'template.vm');

  return function render(locals) {
    return script.runInNewContext(locals);
  };

  function compile(string) {
    var token = '<%=';
    var index;
    var start;
    var match;
    var output = 'var str = ""\n';

    function push(str) {
      output += 'str += ' + str + '\n';
    }

    function code(str) {
      debug('code %j', str);
      push(trim(str));
    }

    function text(str) {
      debug('text %j', str);
      push(quote(str));
    }

    while ((index = string.indexOf(token, start)) > -1) {
      debug('token=%j start=%s index=%d', token, start, index);

      match = string.slice(start, index);
      start = index + token.length

      switch (token) {
      case '<%=':
        text(match);
        token = '%>';
        break;
      case '%>':
        code(match);
        token = '<%=';
        break;
      }
    }

    if (start < string.length) {
      if (token === '%>') {
        throw new SyntaxError('Unclosed code section');
      }
      text(string.slice(start));
    }

    debug('compiled\n' + output);

    return output;
  }<commit_msg>Back to regex<commit_after>var debug = require('debug')('vmplate');
var vm = require('vm');

module.exports = function vmplate(string, filename) {
  var SCRIPT = /<%=/;
  var STRING = /%>/;
  var script = vm.createScript(compile(string), filename || 'template.vm');

  return function render(locals) {
    return script.runInNewContext(locals);
  };

  function compile(string) {
    var output = 'var str = ""\n';
    var token = SCRIPT;
    var start = 0;
    var input;
    var match;

    function push(str) {
      output += 'str += ' + str + '\n';
    }

    function code(str) {
      debug('code %j', str);
      push(trim(str));
    }

    function text(str) {
      debug('text %j', str);
      push(quote(str));
    }

    function scan(input) {
      debug('input %j', input);
      return token.exec(input);
    }

    while (match = scan(string.substr(start))) {
      debug('token=%s start=%s match=%s', token, start, match);

      input = string.substr(start, match.index);
      start = start + match.index + match[0].length;

      switch (match[0]) {
      case '<%=':
        text(input);
        token = STRING;
        break;
      case '%>':
        code(input);
        token = SCRIPT;
        break;
      }
    }

    if (start < string.length) {
      if (token === STRING) {
        throw new SyntaxError('Unclosed code section');
      }
      text(string.substr(start));
    }

    debug('compiled\n' + output);

    return output;
  }
{'fraction_non_alphanumeric': 0.16871832005792903, 'fraction_numerical': 0.001448225923244026, 'mean_word_length': 2.2775800711743774, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3411679', 'n_tokens_mistral': 912, 'n_tokens_neox': 870, 'n_words': 207}