<commit_before>const EXPR = /{([\s\S]+?)}/g;
const BLOCK_START = 'block:';
const BLOCK_END = '/block:';
//const BLOCK_START_EXPR = /{block:([\s\S]+?)}/g;
//const BLOCK_END_EXPR = /{\/block:([\s\S]+?)}/g;

export default class Parser {
  constructor() {}

  // Tokenizes a raw Tumblr template
  // Returns a list of tokens
  tokenize(template) {
    let tokens = [],
        lastIndex = 0,
        match, token;
    while ((match = EXPR.exec(template)) !== null) {
      if (match.index !== lastIndex) {
        tokens.push({type: 'text', value: template.slice(lastIndex, match.index)});
      }
      lastIndex = EXPR.lastIndex;

      if (match[1].startsWith(BLOCK_START)) {<commit_msg>Remove obsolete variable from tokenize<commit_after>const EXPR = /{([\s\S]+?)}/g;
const BLOCK_START = 'block:';
const BLOCK_END = '/block:';
//const BLOCK_START_EXPR = /{block:([\s\S]+?)}/g;
//const BLOCK_END_EXPR = /{\/block:([\s\S]+?)}/g;

export default class Parser {
  constructor() {}

  // Tokenizes a raw Tumblr template
  // Returns a list of tokens
  tokenize(template) {
    let tokens = [],
        lastIndex = 0,
        match;

    while ((match = EXPR.exec(template)) !== null) {
      if (match.index !== lastIndex) {
        tokens.push({type: 'text', value: template.slice(lastIndex, match.index)});
      }
      lastIndex = EXPR.lastIndex;

      if (match[1].startsWith(BLOCK_START)) {
{'fraction_non_alphanumeric': 0.1873653984206748, 'fraction_numerical': 0.002871500358937545, 'mean_word_length': 3.0405797101449274, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2678035', 'n_tokens_mistral': 500, 'n_tokens_neox': 475, 'n_words': 109}