<commit_before>/*
 * grunt
 * http://gruntjs.com/
 *
 * Copyright (c) 2012 "Cowboy" <NAME>
 * Licensed under the MIT license.
 * http://benalman.com/about/license/
 */

'use strict';

var grunt = require('../grunt');

// Nodejs libs.
var fs = require('fs');
var path = require('path');
// In Nodejs 0.8.0, existsSync moved from path -> fs.
var existsSync = fs.existsSync || path.existsSync;

// The module to be exported.
var file = module.exports = {};

// External libs.
file.glob = require('glob-whatev');
file.findup = require('../util/findup');
var YAML = require('js-yaml');

// Windows?
var win32 = process.platform === 'win32';

// Normalize \\ paths to / paths.
function unixifyPath(filepath) {
  if (win32) {
    return filepath.replace(/\\/g, '/');
  } else {
    return filepath;
  }
}

// Change the current base path (ie, CWD) to the specified path.
file.setBase = function() {
  var dirpath = path.join.apply(path, arguments);
  process.chdir(dirpath);
};

// Match a filepath or filepaths against one or more wildcard patterns. Returns
// all matching filepaths.
file.match = function(options, patterns, filepaths) {
  if (grunt.util.kindOf(options) !== 'object') {
    filepaths = patterns;
    patterns = options;
    options = {};
  }
  // Return empty set if patterns or filepaths are completely empty.
  if (patterns == null || filepaths == null) { return []; }
  // Normalize patterns and filepaths to arrays.
  if (!Array.isArray(patterns)) { patterns = [patterns]; }
  if (!Array.isArray(filepaths)) { filepaths = [filepaths]; }
  // Return empty set if there are no patterns or filepaths.
  if (patterns.length === 0 || filepaths.length === 0) { return []; }
  // Filepaths to return.
  var result = [];
  // Flattened, Underscore.js-chainable set of patterns.
  grunt.util._(patterns).chain().flatten().map(function(pattern) {
    // Process any templates.
    return grunt.template.process(pattern);
  }).each(function(pattern) {
    // If the first character is ! it should be omitted
    var exclusion = pattern.indexOf('!') === 0;
    // If the pattern is an exclusion, remove the !
    if (exclusion) { pattern = pattern.slice(1); }
    // Find all matching files for this pattern.
    var matches = filepaths.filter(file.glob.minimatch.filter(pattern, options));
    if (exclusion) {
      // If an exclusion, remove matching files.
      result = grunt.util._.difference(result, matches);
    } else {
      // Otherwise add matching files.
      result = grunt.util._.union(result, matches);
    }
  });
  return result;
};

// Match a filepath or filepaths against one or more wildcard patterns. Returns
// true if any of the patterns match.
file.isMatch = function() {
  return file.match.apply(file, arguments).length > 0;
};

// List of changed / deleted file paths, generated by the watch task.
file.watchFiles = {changed: null, deleted: null};

// Return an array of all file paths that match the given wildcard patterns.
file.expand = function() {
  var args = grunt.util.toArray(arguments);
  // If the first argument is an options object, save those options to pass
  // into the file.glob.glob method for minimatch to use.
  var options = grunt.util.kindOf(args[0]) === 'object' ? args.shift() : {};
  // Use the first argument if it's an Array, otherwise convert the arguments
  // object to an array and use that.
  var patterns = Array.isArray(args[0]) ? args[0] : args;
  // Filepaths to return.
  var result = [];
  // Flattened, Underscore.js-chainable set of patterns.
  grunt.util._(patterns).chain().flatten().each(function(pattern) {
    // If pattern is a template, process it accordingly.
    pattern = grunt.template.process(pattern);
    // If the first character is ! it should be omitted
    var exclusion = pattern.indexOf('!') === 0;
    // If the pattern is an exclusion, remove the !
    if (exclusion) { pattern = pattern.slice(1); }
    // Find all matching files for this pattern.
    var matches = file.glob.glob(pattern, options);
    if (exclusion) {
      // If an exclusion, remove matching files.
      result = grunt.util._.difference(result, matches);
    } else {
      // Otherwise add matching files.
      result = result.concat(matches);
    }
  });
  return grunt.util._(result).uniq();
};

// Further filter file.expand.
function expandByType(type) {
  var args = grunt.util.toArray(arguments).slice(1);
  // If the first argument is an options object, grab it.
  var options = grunt.util.kindOf(args[0]) === 'object' ? args[0] : {};
  // Match, then filter filepaths.
  return file.expand.apply(file, args).filter(function(filepath) {
    try {
      // If the file is of the right type and exists, this should work.
      return fs.statSync(path.join(options.cwd, filepath))[type]();<commit_msg>Greatly simplified file.match and file.expand methods now that directives are gone.<commit_after>/*
 * grunt
 * http://gruntjs.com/
 *
 * Copyright (c) 2012 "Cowboy" <NAME>
 * Licensed under the MIT license.
 * http://benalman.com/about/license/
 */

'use strict';

var grunt = require('../grunt');

// Nodejs libs.
var fs = require('fs');
var path = require('path');
// In Nodejs 0.8.0, existsSync moved from path -> fs.
var existsSync = fs.existsSync || path.existsSync;

// The module to be exported.
var file = module.exports = {};

// External libs.
file.glob = require('glob-whatev');
file.findup = require('../util/findup');
var YAML = require('js-yaml');

// Windows?
var win32 = process.platform === 'win32';

// Normalize \\ paths to / paths.
function unixifyPath(filepath) {
  if (win32) {
    return filepath.replace(/\\/g, '/');
  } else {
    return filepath;
  }
}

// Change the current base path (ie, CWD) to the specified path.
file.setBase = function() {
  var dirpath = path.join.apply(path, arguments);
  process.chdir(dirpath);
};

// Process specified wildcard glob patterns or filenames against a
// callback, processing templates, excluding and uniquing files in the
// result set.
var processPatterns = function(patterns, fn) {
  // Filepaths to return.
  var result = [];
  // Flattened, Underscore.js-chainable set of patterns.
  grunt.util._(patterns).chain().flatten().each(function(pattern) {
    // If pattern is a template, process it accordingly.
    pattern = grunt.template.process(pattern);
    // If the first character is ! it should be omitted
    var exclusion = pattern.indexOf('!') === 0;
    // If the pattern is an exclusion, remove the !
    if (exclusion) { pattern = pattern.slice(1); }
    // Find all matching files for this pattern.
    var matches = fn(pattern);
    if (exclusion) {
      // If an exclusion, remove matching files.
      result = grunt.util._.difference(result, matches);
    } else {
      // Otherwise add matching files.
      result = grunt.util._.union(result, matches);
    }
  });
  return result;
};

// Match a filepath or filepaths against one or more wildcard patterns. Returns
// all matching filepaths.
file.match = function(options, patterns, filepaths) {
  if (grunt.util.kindOf(options) !== 'object') {
    filepaths = patterns;
    patterns = options;
    options = {};
  }
  // Return empty set if either patterns or filepaths was omitted.
  if (patterns == null || filepaths == null) { return []; }
  // Normalize patterns and filepaths to arrays.
  if (!Array.isArray(patterns)) { patterns = [patterns]; }
  if (!Array.isArray(filepaths)) { filepaths = [filepaths]; }
  // Return empty set if there are no patterns or filepaths.
  if (patterns.length === 0 || filepaths.length === 0) { return []; }
  // Return all matching filepaths.
  return processPatterns(patterns, function(pattern) {
    return filepaths.filter(file.glob.minimatch.filter(pattern, options));
  });
};

// Match a filepath or filepaths against one or more wildcard patterns. Returns
// true if any of the patterns match.
file.isMatch = function() {
  return file.match.apply(file, arguments).length > 0;
};

// List of changed / deleted file paths, generated by the watch task.
file.watchFiles = {changed: null, deleted: null};

// Return an array of all file paths that match the given wildcard patterns.
file.expand = function() {
  var args = grunt.util.toArray(arguments);
  // If the first argument is an options object, save those options to pass
  // into the file.glob.glob method for minimatch to use.
  var options = grunt.util.kindOf(args[0]) === 'object' ? args.shift() : {};
  // Use the first argument if it's an Array, otherwise convert the arguments
  // object to an array and use that.
  var patterns = Array.isArray(args[0]) ? args[0] : args;
  // Return empty set if there are no patterns or filepaths.
  if (patterns.length === 0) { return []; }
  // Return all matching filepaths.
  return processPatterns(patterns, function(pattern) {
    // Find all matching files for this pattern.
    return file.glob.glob(pattern, options);
  });
};

// Further filter file.expand.
function expandByType(type) {
  var args = grunt.util.toArray(arguments).slice(1);
  // If the first argument is an options object, grab it.
  var options = grunt.util.kindOf(args[0]) === 'object' ? args[0] : {};
  // Match, then filter filepaths.
  return file.expand.apply(file, args).filter(function(filepath) {
    try {
      // If the file is of the right type and exists, this should work.
      return fs.statSync(path.join(options.cwd, filepath))[type]();
{'fraction_non_alphanumeric': 0.13726324749946797, 'fraction_numerical': 0.005426686529048734, 'mean_word_length': 3.2567934782608696, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5065495', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2814, 'n_words': 944}