<commit_before>/*
 * grunt
 * https://github.com/cowboy/grunt
 *
 * Copyright (c) 2012 "Cowboy" <NAME>
 * Licensed under the MIT license.
 * http://benalman.com/about/license/
 */

var grunt = require('../grunt');

// Nodejs libs.
var fs = require('fs');
var path = require('path');

// External libs.
var glob = require('glob-whatev');

// The module to be exported.
var file = module.exports = {};

// Change the current base path (ie, CWD) to the specified path.
file.setBase = function() {
  var dirpath = path.join.apply(path, arguments);
  process.chdir(dirpath);
};

// Return an array of all file paths that match the given wildcard patterns.
file.expand = function() {
  var args = grunt.utils.toArray(arguments);
  // If the first argument is an options object, save those options to pass
  // into the glob.glob method for minimatch to use.
  var options = grunt.utils.kindOf(args[0]) === 'object' ? args.shift() : {};
  // Use the first argument if it's an Array, otherwise convert the arguments
  // object to an array and use that.
  var patterns = Array.isArray(args[0]) ? args[0] : args;
  // Generate a should-be-unique number.
  var uid = +new Date();
  // Return a flattened, uniqued array of matching file paths.
  return grunt.utils._(patterns).chain().flatten().map(function(pattern) {
    // If pattern is a template, process it accordingly.
    pattern = grunt.template.process(pattern);
    // Just return the pattern if it's an internal directive.
    if (grunt.task.getDirectiveParts(pattern)) { return pattern; }
    // Otherwise, expand paths.
    return glob.glob(pattern, options);
  }).flatten().uniq(false, function(filepath) {
    // Only unique file paths, but don't unique <something> directives, in case
    // they are repeated intentionally.
    return grunt.task.getDirectiveParts(filepath) ? ++uid : filepath;
  }).value();
};

// Further filter file.expand.
function expandByType(type) {
  var args = grunt.utils.toArray(arguments).slice(1);
  return file.expand.apply(file, args).filter(function(filepath) {
    // Just return the filepath if it's an internal directive.
    if (grunt.task.getDirectiveParts(filepath)) { return filepath; }
    try {
      return fs.statSync(filepath)[type]();
    } catch(e) {
      throw grunt.task.taskError(e.message, e);
    }
  });
}

// A few type-specific file expansion methods.
file.expandDirs = expandByType.bind(file, 'isDirectory');
file.expandFiles = expandByType.bind(file, 'isFile');
<commit_msg>Exposing glob-whatev as file.glob.<commit_after>/*
 * grunt
 * https://github.com/cowboy/grunt
 *
 * Copyright (c) 2012 "Cowboy" <NAME>
 * Licensed under the MIT license.
 * http://benalman.com/about/license/
 */

var grunt = require('../grunt');

// Nodejs libs.
var fs = require('fs');
var path = require('path');

// The module to be exported.
var file = module.exports = {};

// External libs.
file.glob = require('glob-whatev');

// Change the current base path (ie, CWD) to the specified path.
file.setBase = function() {
  var dirpath = path.join.apply(path, arguments);
  process.chdir(dirpath);
};

// Return an array of all file paths that match the given wildcard patterns.
file.expand = function() {
  var args = grunt.utils.toArray(arguments);
  // If the first argument is an options object, save those options to pass
  // into the file.glob.glob method for minimatch to use.
  var options = grunt.utils.kindOf(args[0]) === 'object' ? args.shift() : {};
  // Use the first argument if it's an Array, otherwise convert the arguments
  // object to an array and use that.
  var patterns = Array.isArray(args[0]) ? args[0] : args;
  // Generate a should-be-unique number.
  var uid = +new Date();
  // Return a flattened, uniqued array of matching file paths.
  return grunt.utils._(patterns).chain().flatten().map(function(pattern) {
    // If pattern is a template, process it accordingly.
    pattern = grunt.template.process(pattern);
    // Just return the pattern if it's an internal directive.
    if (grunt.task.getDirectiveParts(pattern)) { return pattern; }
    // Otherwise, expand paths.
    return file.glob.glob(pattern, options);
  }).flatten().uniq(false, function(filepath) {
    // Only unique file paths, but don't unique <something> directives, in case
    // they are repeated intentionally.
    return grunt.task.getDirectiveParts(filepath) ? ++uid : filepath;
  }).value();
};

// Further filter file.expand.
function expandByType(type) {
  var args = grunt.utils.toArray(arguments).slice(1);
  return file.expand.apply(file, args).filter(function(filepath) {
    // Just return the filepath if it's an internal directive.
    if (grunt.task.getDirectiveParts(filepath)) { return filepath; }
    try {
      return fs.statSync(filepath)[type]();
    } catch(e) {
      throw grunt.task.taskError(e.message, e);
    }
  });
}

// A few type-specific file expansion methods.
file.expandDirs = expandByType.bind(file, 'isDirectory');
file.expandFiles = expandByType.bind(file, 'isFile');

{'fraction_non_alphanumeric': 0.1385782747603834, 'fraction_numerical': 0.003194888178913738, 'mean_word_length': 3.5495004541326067, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180323', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1533, 'n_words': 476}