<commit_before>'use strict';


var Parser = module.exports = {};


function ScalarNode(value) {
  this.type = 'text';
  this.value = value;
}


function VariableNode(anchor) {
  this.type = 'variable';
  this.anchor = anchor;
}


function PluralNode(anchor, forms) {
  this.type = 'plural';
  this.anchor = anchor;
  this.forms = forms;
}


// catch valid variable and object accessor notation
var ANCHOR_REGEXP = new RegExp(
  '([a-z_$](?:[a-z0-9_$]*|\\.[a-z_$][a-z0-9_$]*)*)',
  'i'
);


// finds macros in the translations
/*var MACROS_REGEXP = new RegExp(
  "(^|[^\\\\])" + // [1] match begining of string or any char except \ (backslash)
    "([#%])" + // [2] match operation notation # or %
    "{((?:[^\\\\}]|\\})+)}" + // [3] arguments
    "(?::([_$a-z](?:[_$a-z0-9.]*?[_$a-z0-9])?))?" + // [4] plural anchor
    "(?:[^_$a-z0-9]|$)",
  "i"
);*/
var MACROS_REGEXP = new RegExp(
  '(^|.*?[^\\\\])' +            // either nothing, or anything which doesn't end with \ (backslash)
    '(?:' +
      '#{' +                    // interpolation
        ANCHOR_REGEXP.source +  // interpolating variable
      '}' +
    '|' +
      '%{' +                    // pluralization
        '(.*?[^\\\\])(?=})' +   // word forms
      '}:' +
      ANCHOR_REGEXP.source +    // controlling variable
    ')',
  'i'
);

// used to unescape critical chars
var UNESCAPE_CHARS = new RegExp('\\\\([#%}|\\\\])', 'g');
function unescapeString(str) {
  return str.replace(UNESCAPE_CHARS, '$1');
}

// used to split arguments of plural
var FORMS_SEPARATOR_REGEXP = new RegExp("(^|[^\\\\])[|]", 'g');
function parseForms(str) {
  return str
    // split by non-escaped '|'
    .replace(FORMS_SEPARATOR_REGEXP, '$1( . )( . )')
    .split('( . )( . )')
    // strip escaping
    .map(unescapeString);
}


// parses string into array of nodes
Parser.parse = function parse(str) {
  var nodes = [], match;

  while (!!str.length) {
    match = str.match(MACROS_REGEXP);

    if (null === match) {
      nodes.push(new ScalarNode(unescapeString(str)));
      break;<commit_msg>Refactor forms splitter<commit_after>'use strict';


var Parser = module.exports = {};


function ScalarNode(value) {
  this.type = 'text';
  this.value = value;
}


function VariableNode(anchor) {
  this.type = 'variable';
  this.anchor = anchor;
}


function PluralNode(anchor, forms) {
  this.type = 'plural';
  this.anchor = anchor;
  this.forms = forms;
}


// catch valid variable and object accessor notation
var ANCHOR_REGEXP = new RegExp(
  '([a-z_$](?:[a-z0-9_$]*|\\.[a-z_$][a-z0-9_$]*)*)',
  'i'
);


// finds macros in the translations
/*var MACROS_REGEXP = new RegExp(
  "(^|[^\\\\])" + // [1] match begining of string or any char except \ (backslash)
    "([#%])" + // [2] match operation notation # or %
    "{((?:[^\\\\}]|\\})+)}" + // [3] arguments
    "(?::([_$a-z](?:[_$a-z0-9.]*?[_$a-z0-9])?))?" + // [4] plural anchor
    "(?:[^_$a-z0-9]|$)",
  "i"
);*/
var MACROS_REGEXP = new RegExp(
  '(^|.*?[^\\\\])' +            // either nothing, or anything which doesn't end with \ (backslash)
    '(?:' +
      '#{' +                    // interpolation
        ANCHOR_REGEXP.source +  // interpolating variable
      '}' +
    '|' +
      '%{' +                    // pluralization
        '(.*?[^\\\\])(?=})' +   // word forms
      '}:' +
      ANCHOR_REGEXP.source +    // controlling variable
    ')',
  'i'
);

// used to unescape critical chars
var UNESCAPE_CHARS = new RegExp('\\\\([#%}|\\\\])', 'g');
function unescapeString(str) {
  return str.replace(UNESCAPE_CHARS, '$1');
}

// used to split arguments of plural
var FORMS_SEPARATOR_REGEXP = /([\\\\]*)[|]/;
function parseForms(str) {
  var forms = [], match, tmp = '';

  while (!!str.length) {
    match = str.match(FORMS_SEPARATOR_REGEXP);

    if (null === match) {
      forms.push(str);
      str = '';
    } else if (1 === match[1].length % 2) {
      tmp += str.slice(0, match.index + match[0].length);
      str = str.slice(match.index + match[0].length);
    } else {
      forms.push(tmp + str.slice(0, match.index + match[0].length - 1));
      str = str.slice(match.index + match[0].length);
      tmp = '';
    }
  }

  return forms.map(unescapeString);
}


// parses string into array of nodes
Parser.parse = function parse(str) {
  var nodes = [], match;

  while (!!str.length) {
    match = str.match(MACROS_REGEXP);

    if (null === match) {
      nodes.push(new ScalarNode(unescapeString(str)));
      break;
{'fraction_non_alphanumeric': 0.22055194076733228, 'fraction_numerical': 0.009199012788871438, 'mean_word_length': 2.535289452815226, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2847424', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1628, 'n_words': 336}