<commit_before>/**
 * A map describing parentheses-like grouping characters and their behavior. The
 * keys are the group's start character and the value's `end` property contains
 * the end character. The `matchNestedParens` property indicates whether or not
 * other grouping characters nested in the group retain their grouping behavior
 * within that group - i.e. if nested grouping characters need to be matched.
 *
 * @type {!Map<string, {end: string, matchNestedParens: boolean}>}
 */
const PARENS = new Map([
  ['(', {end: ')', matchNestedParens: true}],
  ['[', {end: ']', matchNestedParens: true}],
  ['"', {end: '"', matchNestedParens: false}],
  ["'", {end: "'", matchNestedParens: false}],
]);

/**
 * Finds the first character of `queryChars` within `str` from index `start`
 * onwards that is _not_ nested within any parentheses, brackets, or
 * single/double-quoted strings or part of an escape sequence.
 *
 * If the end of the string is reached without finding a character in
 * `queryChars`, the length of the string is returned.
 *
 * @param {string} str
 * @param {number} start
 * @param {!Array<string>} queryChars
 * @param {boolean} matchNestedParens
 * @return {number}
 */
const findNext = (str, start, queryChars, matchNestedParens = true) => {
  for (let i = start; i < str.length; i++) {
    if (str[i] === '\\' && i < str.length - 1 && str[i + 1] !== '\n') {
      // Skip escape sequences.
      i++;
    } else if (queryChars.indexOf(str[i]) !== -1) {
      // One of `queryChars` was found.
      return i;
    } else if (matchNestedParens && PARENS.has(str[i])) {
      // Handle any grouping characters by skipping over the group.
      const parenInfo = PARENS.get(str[i]);
      i = findNext(str, i + 1, [parenInfo.end], parenInfo.matchNestedParens);
      continue;
    }
<commit_msg>IE 11 doesn't support `Map`'s constructor parameter.<commit_after>/**
 * A map describing parentheses-like grouping characters and their behavior. The
 * keys are the group's start character and the value's `end` property contains
 * the end character. The `matchNestedParens` property indicates whether or not
 * other grouping characters nested in the group retain their grouping behavior
 * within that group - i.e. if nested grouping characters need to be matched.
 *
 * @type {!Map<string, {end: string, matchNestedParens: boolean}>}
 */
const PARENS = new Map();
// IE 11 doesn't support `Map`'s constructor parameter.
[
  ['(', {end: ')', matchNestedParens: true}],
  ['[', {end: ']', matchNestedParens: true}],
  ['"', {end: '"', matchNestedParens: false}],
  ["'", {end: "'", matchNestedParens: false}],
].forEach(([k, v]) => {
  PARENS.set(k, v);
});

/**
 * Finds the first character of `queryChars` within `str` from index `start`
 * onwards that is _not_ nested within any parentheses, brackets, or
 * single/double-quoted strings or part of an escape sequence.
 *
 * If the end of the string is reached without finding a character in
 * `queryChars`, the length of the string is returned.
 *
 * @param {string} str
 * @param {number} start
 * @param {!Array<string>} queryChars
 * @param {boolean} matchNestedParens
 * @return {number}
 */
const findNext = (str, start, queryChars, matchNestedParens = true) => {
  for (let i = start; i < str.length; i++) {
    if (str[i] === '\\' && i < str.length - 1 && str[i + 1] !== '\n') {
      // Skip escape sequences.
      i++;
    } else if (queryChars.indexOf(str[i]) !== -1) {
      // One of `queryChars` was found.
      return i;
    } else if (matchNestedParens && PARENS.has(str[i])) {
      // Handle any grouping characters by skipping over the group.
      const parenInfo = PARENS.get(str[i]);
      i = findNext(str, i + 1, [parenInfo.end], parenInfo.matchNestedParens);
      continue;
    }

{'fraction_non_alphanumeric': 0.14730728616684266, 'fraction_numerical': 0.0031678986272439284, 'mean_word_length': 3.6094890510948905, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '919301', 'n_tokens_mistral': 1242, 'n_tokens_neox': 1184, 'n_words': 421}