<commit_before>var async = require('async'),
    _ = require('underscore'),
    glob = require('glob'),
    fs = require('graceful-fs'),
    path = require('path');


/**
 * generate a map of file operations to perform
 * @type {Function}
 * @param {String} tempDirectory the temporary-directory the source files are located in
 * @param {{ target:String|Object }} resource
 * @param {Function} callback( error, [ { source:String, destination:String, isDirectory:Boolean } ])
 */
module.exports = exports = function( tempDirectory, resource, callback ){
    var params = (typeof resource.target === 'string' ?
        mapSimpleTarget :
        mapComplexTarget
        )( tempDirectory, resource );
    //data structure is: [ [sources], [destinations] ]
    generateMap( tempDirectory, params[0], params[1], resource.exclude, callback );
};


var append = function( pre, val ){ return pre + val; };

//ensure there is a trailing slash on the directory
var trailSlash = function( path ){
    return path.lastIndexOf('/') === path.length -1 ? path : path + '/';
};

//stat to find if its a directory
//don't report errors cause these are still the raw globs, non-directories wont exist
var isDirectory = function( absGlob, callback ){
    fs.stat( absGlob, function( err, stat ){
        //if its an Error, IGNORE
        callback( null, stat ? stat.isDirectory() : false );
    });
};

/**
 * @private
 * append * * / * wildcard to directory
 * in order to get individual file lists
 * @param {String} cwd the current working directory
 * @param {Array<String>} pathTargets the targets i.e [ 'src/', '*.json' ]
 * @param {Function} callback( pathGlobs )
 */
 var globifyTargets = function( cwd, pathTargets, callback ){
    var absGlobs = pathTargets.map(_.partial(append,cwd));
    async.mapSeries( absGlobs, isDirectory, function( err, isDirs ){
        //never an error
        pathTargets = isDirs.map(function( isDir, i ){
            return isDir ? trailSlash(pathTargets[i]) + '**/*' : pathTargets[i];
        });
        callback( pathTargets );
    });
};

/**
 * @public
 * get the relative paths for all source files from the target keys
 * @param {String} cwd the current working directory for glob-matching
 * @param {Array} globs the strings of patterns to match
 * @param callback( error, sourceMap:Object )
 */
var getRelativeSources = function( cwd, globs, callback ){
    var globContent = function(k,fn){
        glob(k, { cwd: cwd }, fn);
    };
    globifyTargets( cwd, globs, function( finalGlobs ){
        async.map(_.map(finalGlobs, _.partial(append,cwd)), globContent, function( err, sources ){
            if( err ){
                callback( err );
                return;
            }
            //remove the cwd from the resolved paths, for a relative-path
            sources = _.map(sources, function(sourceList, i ){
                return _.map( sourceList, function( s, j ){
                    return s.slice(cwd.length);
                });
            });
            callback( null, _.object(finalGlobs, sources) );
        });<commit_msg>fix fetch ignoring dotfiles<commit_after>var async = require('async'),
    _ = require('underscore'),
    glob = require('glob'),
    fs = require('graceful-fs'),
    path = require('path');


/**
 * generate a map of file operations to perform
 * @type {Function}
 * @param {String} tempDirectory the temporary-directory the source files are located in
 * @param {{ target:String|Object }} resource
 * @param {Function} callback( error, [ { source:String, destination:String, isDirectory:Boolean } ])
 */
module.exports = exports = function( tempDirectory, resource, callback ){
    var params = (typeof resource.target === 'string' ?
        mapSimpleTarget :
        mapComplexTarget
        )( tempDirectory, resource );
    //data structure is: [ [sources], [destinations] ]
    generateMap( tempDirectory, params[0], params[1], resource.exclude, callback );
};


var append = function( pre, val ){ return pre + val; };

//ensure there is a trailing slash on the directory
var trailSlash = function( path ){
    return path.lastIndexOf('/') === path.length -1 ? path : path + '/';
};

//stat to find if its a directory
//don't report errors cause these are still the raw globs, non-directories wont exist
var isDirectory = function( absGlob, callback ){
    fs.stat( absGlob, function( err, stat ){
        //if its an Error, IGNORE
        callback( null, stat ? stat.isDirectory() : false );
    });
};

/**
 * @private
 * append * * / * wildcard to directory
 * in order to get individual file lists
 * @param {String} cwd the current working directory
 * @param {Array<String>} pathTargets the targets i.e [ 'src/', '*.json' ]
 * @param {Function} callback( pathGlobs )
 */
 var globifyTargets = function( cwd, pathTargets, callback ){
    var absGlobs = pathTargets.map(_.partial(append,cwd));
    async.mapSeries( absGlobs, isDirectory, function( err, isDirs ){
        //never an error
        pathTargets = isDirs.map(function( isDir, i ){
            return isDir ? trailSlash(pathTargets[i]) + '**/*' : pathTargets[i];
        });
        callback( pathTargets );
    });
};

/**
 * @public
 * get the relative paths for all source files from the target keys
 * @param {String} cwd the current working directory for glob-matching
 * @param {Array} globs the strings of patterns to match
 * @param callback( error, sourceMap:Object )
 */
var getRelativeSources = function( cwd, globs, callback ){
    var globContent = function(k,fn){
        glob(k, { cwd: cwd, dot: true }, fn);
    };
    globifyTargets( cwd, globs, function( finalGlobs ){
        async.map(_.map(finalGlobs, _.partial(append,cwd)), globContent, function( err, sources ){
            if( err ){
                callback( err );
                return;
            }
            //remove the cwd from the resolved paths, for a relative-path
            sources = _.map(sources, function(sourceList, i ){
                return _.map( sourceList, function( s, j ){
                    return s.slice(cwd.length);
                });
            });
            callback( null, _.object(finalGlobs, sources) );
        });
{'fraction_non_alphanumeric': 0.13132783438460294, 'fraction_numerical': 0.0009704027171276079, 'mean_word_length': 2.865, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4808560', 'n_tokens_mistral': 1757, 'n_tokens_neox': 1726, 'n_words': 571}