<commit_before>/*
 *  Copyright (c) 2015-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

'use strict';

require('es6-promise').polyfill();

const child_process = require('child_process');
const colors = require('colors/safe');
const fs = require('fs');
const path = require('path');
const http = require('http');
const https = require('https');
const temp = require('temp');
const ignores = require('./ignoreFiles');

const availableCpus = (require('os').cpus().length - 1) || 1;
const CHUNK_SIZE = 50;

function lineBreak(str) {
  return /\n$/.test(str) ? str : str + '\n';
}

const log = {
  ok(msg, verbose) {
    verbose >= 2 && process.stdout.write(colors.white.bgGreen(' OKK ') + msg);
  },
  nochange(msg, verbose) {
    verbose >= 1 && process.stdout.write(colors.white.bgYellow(' NOC ') + msg);
  },
  skip(msg, verbose) {
    verbose >= 1 && process.stdout.write(colors.white.bgYellow(' SKIP ') + msg);
  },
  error(msg, verbose) {
    verbose >= 0 && process.stdout.write(colors.white.bgRed(' ERR ') + msg);
  },
};

function concatAll(arrays) {
  return arrays.reduce(
    (result, array) => (result.push.apply(result, array), result),
    []
  );
}<commit_msg>Changes based on PR feedback<commit_after>/*
 *  Copyright (c) 2015-present, Facebook, Inc.
 *  All rights reserved.
 *
 *  This source code is licensed under the BSD-style license found in the
 *  LICENSE file in the root directory of this source tree. An additional grant
 *  of patent rights can be found in the PATENTS file in the same directory.
 *
 */

'use strict';

require('es6-promise').polyfill();

const child_process = require('child_process');
const colors = require('colors/safe');
const fs = require('fs');
const path = require('path');
const http = require('http');
const https = require('https');
const temp = require('temp');
const ignores = require('./ignoreFiles');

const availableCpus = Math.max(require('os').cpus().length - 1, 1);
const CHUNK_SIZE = 50;

function lineBreak(str) {
  return /\n$/.test(str) ? str : str + '\n';
}

const log = {
  ok(msg, verbose) {
    verbose >= 2 && process.stdout.write(colors.white.bgGreen(' OKK ') + msg);
  },
  nochange(msg, verbose) {
    verbose >= 1 && process.stdout.write(colors.white.bgYellow(' NOC ') + msg);
  },
  skip(msg, verbose) {
    verbose >= 1 && process.stdout.write(colors.white.bgYellow(' SKIP ') + msg);
  },
  error(msg, verbose) {
    verbose >= 0 && process.stdout.write(colors.white.bgRed(' ERR ') + msg);
  },
};

function concatAll(arrays) {
  return arrays.reduce(
    (result, array) => (result.push.apply(result, array), result),
    []
  );
}
{'fraction_non_alphanumeric': 0.16684192078513846, 'fraction_numerical': 0.009113214160532773, 'mean_word_length': 3.8620102214650767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3983805', 'n_tokens_mistral': 949, 'n_tokens_neox': 901, 'n_words': 262}