<commit_before>var 	dir = require('node-dir'),
		_ = require("underscore");

//Extend to provide all the functions of node-dir
_.extend(exports, dir);

_.extend(exports, {
	/**
	 * Lists paths to all files in the provided directory.
	 * Will not display the files of subdirectories
	 *
	 * WARNING: advised not to use a path containing a large number
	 * of recursive subdirectories as this does recursive dir functions
	 * @param path the absolute path to directory ( __dirname+'/api/' )
	 * @param callback array of Strings
	 */
	filesLocal: function (path, callback) {
		dir.files(path, function (err, files) {
			if (err) throw err;
			return dir.subdirs(path, function (err, subdirs) {
				if (err) throw err;
				//remove all subdir files
				for (var subdir in subdirs) {
					if(subdirs.hasOwnProperty(subdir)){
						subdirs[subdir] += '\\';//ensure that it only blocks files INSIDE this dir(not similar named files outside)
						files = files.filter(function (file) {
							return (file.substring(0, subdirs[subdir].length) != subdirs[subdir])
						});
					}
				}
				//console.log('results:\n',files);
				callback(files);
			});
		});<commit_msg>possible Async error for file loading<commit_after>var 	async = require('async'),
		dir = require('node-dir'),
		_ = require("underscore");

//Extend to provide all the functions of node-dir
_.extend(exports, dir);

_.extend(exports, {
	/**
	 * Lists paths to all files in the provided directory.
	 * Will not display the files of subdirectories
	 *
	 * WARNING: advised not to use a path containing a large number
	 * of recursive subdirectories as this does recursive dir functions
	 * @param path the absolute path to directory ( __dirname+'/api/' )
	 * @param callback array of Strings
	 */
	filesLocal: function (path, callback) {
		dir.files(path, function (err, files) {
			if (err) throw err;
			return dir.subdirs(path, function (err, subdirs) {
				if (err) throw err;
				//remove all subdir files
				async.eachSeries(subdirs, function (subdir, cb) {
					if (subdir) {
						subdir += '\\';//ensure that it only blocks files INSIDE this dir(not similar named files outside)
						files = files.filter(function (file) {
							return (file.substring(0, subdir.length) != subdir)
						});
					}
					cb();
				}, function done(){
					callback(files);
				});
				//console.log('results:\n',files);
				
			});
		});
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.0008361204013377926, 'mean_word_length': 3.2353982300884954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5611604', 'n_tokens_mistral': 859, 'n_tokens_neox': 710, 'n_words': 246}