<commit_before>'use strict';

var byline        = require('byline'),
    chalk         = require('chalk'),
    eventStream   = require('event-stream'),
    fs            = require('fs'),
    isBinaryFile  = require('isbinaryfile'),
    minimatch     = require('minimatch'),
    readdirp      = require('readdirp');

var ignoredDirectories  = ['node_modules/**', '.git/**', '.hg/**'],
    filesToScan         = ['**/*.js', 'Makefile', '**/*.sh'],
    scanPath            = process.cwd(),
    fileEncoding        = 'utf8',
    lineLengthLimit     = 1000,
    skipChecks          = [],
    messageChecks       = {
      note: {
        regex:    /(?:^|[^:])\/[/*]\s*NOTE\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✐ NOTE',
        colorer:  chalk.green
      },
      optimize: {
        regex:    /(?:^|[^:])\/[/*]\s*OPTIMIZE\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ↻ OPTIMIZE',
        colorer:  chalk.blue
      },
      todo: {
        regex:    /(?:^|[^:])\/[/*]\s*TODO\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✓ TODO',
        colorer:  chalk.magenta
      },
      hack: {
        regex:    /(?:^|[^:])\/[/*]\s*HACK\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✄ HACK',
        colorer:  chalk.yellow
      },
      xxx: {
        regex:    /(?:^|[^:])\/[/*]\s*XXX\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✗ XXX',
        colorer:  chalk.black.bgYellow
      },
      fixme: {
        regex:    /(?:^|[^:])\/[/*]\s*FIXME\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ☠ FIXME',
        colorer:  chalk.red
      },
      bug: {
        regex:    /(?:^|[^:])\/[/*]\s*BUG\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ☢ BUG',
        colorer:  chalk.white.bgRed
      }
    };

/**
 * Determines whether or not to let the file through. by ensuring that the
 * file name does not match one of the excluded directories, and ensuring it
 * matches one of the file filters.
 *
 * It will also ensure that even if a binary file matches the filter patterns,
 * it will not let it through as searching binary file contents for string
 * matches will never make sense.
 *
 * @param   {String} fileInformation
 *
 * @return  {Boolean}
 */
// TODO: This could be simpler using minimatch negation patterns in one set, instead disparate ones for files and directories.
function fileFilterer (fileInformation) {
  var shouldIgnoreDirectory = false,
      shouldIgnoreFile      = true,
      letTheFileThrough;

  ignoredDirectories.forEach(function (directoryPattern) {
    if (shouldIgnoreDirectory) return;
    shouldIgnoreDirectory = minimatch(fileInformation.path, directoryPattern, { dot: true });
  });

  if (!shouldIgnoreDirectory) {<commit_msg>Update fixme.js<commit_after>'use strict';

var byline        = require('byline'),
    chalk         = require('chalk'),
    eventStream   = require('event-stream'),
    fs            = require('fs'),
    isBinaryFile  = require('isbinaryfile'),
    minimatch     = require('minimatch'),
    readdirp      = require('readdirp');

var ignoredDirectories  = ['node_modules/**', '.git/**', '.hg/**'],
    filesToScan         = ['**/*.js', 'Makefile', '**/*.sh'],
    scanPath            = process.cwd(),
    fileEncoding        = 'utf8',
    lineLengthLimit     = 1000,
    skipChecks          = [],
    messageChecks       = {
      note: {
        regex:    /(?:^|[^:])\/[/*]\s*@?NOTE\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✐ NOTE',
        colorer:  chalk.green
      },
      optimize: {
        regex:    /(?:^|[^:])\/[/*]\s*@?OPTIMIZE\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ↻ OPTIMIZE',
        colorer:  chalk.blue
      },
      todo: {
        regex:    /(?:^|[^:])\/[/*]\s*@?TODO\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✓ TODO',
        colorer:  chalk.magenta
      },
      hack: {
        regex:    /(?:^|[^:])\/[/*]\s*@?HACK\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✄ HACK',
        colorer:  chalk.yellow
      },
      xxx: {
        regex:    /(?:^|[^:])\/[/*]\s*@?XXX\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ✗ XXX',
        colorer:  chalk.black.bgYellow
      },
      fixme: {
        regex:    /(?:^|[^:])\/[/*]\s*@?FIXME\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ☠ FIXME',
        colorer:  chalk.red
      },
      bug: {
        regex:    /(?:^|[^:])\/[/*]\s*@?BUG\b\s*(?:\(([^:]*)\))*\s*:?\s*(.*)/i,
        label:    ' ☢ BUG',
        colorer:  chalk.white.bgRed
      }
    };

/**
 * Determines whether or not to let the file through. by ensuring that the
 * file name does not match one of the excluded directories, and ensuring it
 * matches one of the file filters.
 *
 * It will also ensure that even if a binary file matches the filter patterns,
 * it will not let it through as searching binary file contents for string
 * matches will never make sense.
 *
 * @param   {String} fileInformation
 *
 * @return  {Boolean}
 */
// TODO: This could be simpler using minimatch negation patterns in one set, instead disparate ones for files and directories.
function fileFilterer (fileInformation) {
  var shouldIgnoreDirectory = false,
      shouldIgnoreFile      = true,
      letTheFileThrough;

  ignoredDirectories.forEach(function (directoryPattern) {
    if (shouldIgnoreDirectory) return;
    shouldIgnoreDirectory = minimatch(fileInformation.path, directoryPattern, { dot: true });
  });

  if (!shouldIgnoreDirectory) {
{'fraction_non_alphanumeric': 0.22654253377144942, 'fraction_numerical': 0.0018254837531945967, 'mean_word_length': 2.1434308663224324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5847045', 'n_tokens_mistral': 2013, 'n_tokens_neox': 1892, 'n_words': 399}