<commit_before>'use strict';

const _ = require('lodash');
const chalk = require('chalk');
var logger = require('./../../lib/logger').create();

function diff(left, right) {
  var curly = /^\{.*}/;
  var quote = /^".*"/;
  var round = /^\(.*\)/;
  var square = /^\[.*]/;
  var union = /\s+/;

  try {
    if (curly.test(left) || curly.test(right)) {
      return diffRecord(left, right);
    } else if (quote.test(left) || quote.test(right)) {
      return diffValueWithToken(left, right, '"');
    } else if (round.test(left) || round.test(right)) {
      return diffValueWithToken(left, right, '(');
    } else if (square.test(left) || square.test(right)) {
      return diffList(left, right);
    } else if (union.test(left) || union.test(right)) {
      return diffUnion(left, right);
    }

    return diffValue(left, right);
  } catch (err) {
    var unknown = chalk.yellow('?');
    logger.error('Error during diff ( see \'' + unknown + '\' below)');
    logger.error('Please re-run with verbose option and report the issue');
    logger.debug('Error during diff - Left', left);
    logger.debug('Error during diff - Right', right);
    logger.debug('Error during diff - Error', err);
    var leftLength = left ? left.length : 0;
    var rightLength = right ? right.length : 0;

    return {
      left: _.repeat(unknown, leftLength),
      right: _.repeat(unknown, rightLength)
    };
  }
}

function diffValueWithToken(left, right, token) {
  var l = left.indexOf(token) === -1 ? left : left.substring(1, left.length - 1);
  var r = right.indexOf(token) === -1 ? right : right.substring(1, right.length - 1);
  var value;

  if (token === '"') {
    value = diffValue(l, r);
  } else {
    value = diff(l, r);
  }

  var spacerOnTokenDiff = token === '"' ? ' ' : '^';
  var spacer = left === l && right === r ? ' ' : spacerOnTokenDiff;
  var leftSpacer = left === l ? '' : spacer;
  var rightSpacer = right === r ? '' : spacer;
  value.left = leftSpacer + value.left + leftSpacer;
  value.right = rightSpacer + value.right + rightSpacer;

  return value;
}

function diffUnion(left, right) {
  var leftUnion = parse(left, ' ');
  var rightUnion = parse(right, ' ');
  var token = /[{\[("]/;
  var i = 0;
  var j = 0;
  var acc = {left: '', right: ''};
  var leftMax = leftUnion.length - 1;
  var rightMax = rightUnion.length - 1;

  while (i <= leftMax || j <= rightMax) {
    var l = i > leftMax ? '' : leftUnion[i];
    var r = j > rightMax ? '' : rightUnion[j];
    var value;

    // check for union where args are of different types
    if ((token.test(l) || token.test(r)) && l[0] !== r[0]) {
      value = {left: _.repeat(' ', l.length), right: _.repeat('^', r.length)};
    } else {
      value = diff(l, r);
    }

    var isLastItem = i > leftMax && j === rightMax || j > rightMax && j === leftMax;
    var itemsExistInBothLists = l !== '' && r !== '';
    var spacer = isLastItem || itemsExistInBothLists ? ' ' : '^';
    var leftSpacer = i === leftMax || value.left === '' ? '' : spacer;
    var rightSpacer = j === rightMax || value.right === '' ? '' : spacer;
    acc.left += value.left + leftSpacer;
    acc.right += value.right + rightSpacer;
    i++;
    j++;
  }

  return acc;
}<commit_msg>Fixes broken build by removing un-required regex escape<commit_after>'use strict';

const _ = require('lodash');
const chalk = require('chalk');
var logger = require('./../../lib/logger').create();

function diff(left, right) {
  var curly = /^\{.*}/;
  var quote = /^".*"/;
  var round = /^\(.*\)/;
  var square = /^\[.*]/;
  var union = /\s+/;

  try {
    if (curly.test(left) || curly.test(right)) {
      return diffRecord(left, right);
    } else if (quote.test(left) || quote.test(right)) {
      return diffValueWithToken(left, right, '"');
    } else if (round.test(left) || round.test(right)) {
      return diffValueWithToken(left, right, '(');
    } else if (square.test(left) || square.test(right)) {
      return diffList(left, right);
    } else if (union.test(left) || union.test(right)) {
      return diffUnion(left, right);
    }

    return diffValue(left, right);
  } catch (err) {
    var unknown = chalk.yellow('?');
    logger.error('Error during diff ( see \'' + unknown + '\' below)');
    logger.error('Please re-run with verbose option and report the issue');
    logger.debug('Error during diff - Left', left);
    logger.debug('Error during diff - Right', right);
    logger.debug('Error during diff - Error', err);
    var leftLength = left ? left.length : 0;
    var rightLength = right ? right.length : 0;

    return {
      left: _.repeat(unknown, leftLength),
      right: _.repeat(unknown, rightLength)
    };
  }
}

function diffValueWithToken(left, right, token) {
  var l = left.indexOf(token) === -1 ? left : left.substring(1, left.length - 1);
  var r = right.indexOf(token) === -1 ? right : right.substring(1, right.length - 1);
  var value;

  if (token === '"') {
    value = diffValue(l, r);
  } else {
    value = diff(l, r);
  }

  var spacerOnTokenDiff = token === '"' ? ' ' : '^';
  var spacer = left === l && right === r ? ' ' : spacerOnTokenDiff;
  var leftSpacer = left === l ? '' : spacer;
  var rightSpacer = right === r ? '' : spacer;
  value.left = leftSpacer + value.left + leftSpacer;
  value.right = rightSpacer + value.right + rightSpacer;

  return value;
}

function diffUnion(left, right) {
  var leftUnion = parse(left, ' ');
  var rightUnion = parse(right, ' ');
  var token = /[{[("]/;
  var i = 0;
  var j = 0;
  var acc = {left: '', right: ''};
  var leftMax = leftUnion.length - 1;
  var rightMax = rightUnion.length - 1;

  while (i <= leftMax || j <= rightMax) {
    var l = i > leftMax ? '' : leftUnion[i];
    var r = j > rightMax ? '' : rightUnion[j];
    var value;

    // check for union where args are of different types
    if ((token.test(l) || token.test(r)) && l[0] !== r[0]) {
      value = {left: _.repeat(' ', l.length), right: _.repeat('^', r.length)};
    } else {
      value = diff(l, r);
    }

    var isLastItem = i > leftMax && j === rightMax || j > rightMax && j === leftMax;
    var itemsExistInBothLists = l !== '' && r !== '';
    var spacer = isLastItem || itemsExistInBothLists ? ' ' : '^';
    var leftSpacer = i === leftMax || value.left === '' ? '' : spacer;
    var rightSpacer = j === rightMax || value.right === '' ? '' : spacer;
    acc.left += value.left + leftSpacer;
    acc.right += value.right + rightSpacer;
    i++;
    j++;
  }

  return acc;
}
{'fraction_non_alphanumeric': 0.19052030260923267, 'fraction_numerical': 0.004322989038134939, 'mean_word_length': 2.846793349168646, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4649944', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2151, 'n_words': 579}