<commit_before>// Courtesy of [stygstra](https://gist.github.com/514983)
var fs = require("fs");
var path = require("path");

module.exports = (function() {


	function walk(dirname, options, callbacks) {

		var counter = 0;

		var walkInternal = function(dirname, options, callbacks) {
			callbacks = callbacks || {};

			options = options || {};
			if (typeof callbacks.finished !== 'function') { callbacks.finished = function(){}; }
			if (typeof callbacks.file !== 'function')     { callbacks.file = function(){}; }

			counter += 1;
			fs.readdir(dirname, function(err, relnames) {
				if (err) {
					callbacks.finished(err);
					return;
				}

				if (!relnames.length) {
					counter -= 1;
					if (!counter) {
					  callbacks.finished(null);
					}
					return;
				}

				relnames.forEach(function(relname, index, relnames) {
					var name = path.join(dirname, relname);
					counter += 1;

					fs.stat(name, function(err, stat) {
						if (err) {
							callbacks.finished(err);
							return;
						}

						if (stat.isDirectory()) {
							if (name !== '.svn' && name !== '.git') {
								walkInternal(name, options, callbacks);
							}
						}
						else if (!options.mask || name.match(options.mask)) {
							callbacks.file(name);
						}

						counter -= 1;
						if (index === relnames.length - 1) counter -= 1;

						if (counter === 0) {
							callbacks.finished(null);
						}
					});
				});
			});
		};

		walkInternal(dirname, options, callbacks);
	}

	return walk;
})();<commit_msg>Update lib/walk.js to accept array of directories. #136<commit_after>// Courtesy of [stygstra](https://gist.github.com/514983)
var fs = require("fs");
var path = require("path");

module.exports = (function() {


	function walk(directories, options, callbacks) {

		var filesRemaining = 0,
			loopsRemaining = 0;

		var walkInternal = function(directories, options, callbacks) {
			callbacks = callbacks || {};
			options = options || {};
			if (typeof callbacks.finished !== 'function') { callbacks.finished = function(){}; }
			if (typeof callbacks.file !== 'function')     { callbacks.file = function(){}; }

			if (!Array.isArray(directories)) {
				directories = [directories];
			}

			// Loop through all the given directories.
			loopsRemaining += directories.length;
			directories.forEach(function(directory, index, directories) {
				loopsRemaining -= 1;

				// Start an asynchronous search of the file system.
				filesRemaining += 1;
				fs.readdir(directory, function(err, relnames) {
					if (err) {
						callbacks.finished(err);
						return;
					}

					// About to start looping through the directory contents.
					loopsRemaining += relnames.length;
					// The fs.readdir() callback has returned.
					filesRemaining -= 1;

					// If there is no more file system to search, call .finished().
					if (filesRemaining === 0 && loopsRemaining === 0) {
					  callbacks.finished(null);
					}

					// Otherwise, if readdir() has results, loop through them.
					relnames.forEach(function(relname, index, relnames) {
						loopsRemaining -= 1;
						var name = path.join(directory, relname);

						// Start an asynchronous stat of this file system item.
						filesRemaining += 1;
						fs.stat(name, function(err, stat) {
							if (err) {
								callbacks.finished(err);
								return;
							}

							if (stat.isDirectory()) {
								if (name !== '.svn' && name !== '.git') {
									walkInternal(name, options, callbacks);
								}
							}
							else if (!options.mask || name.match(options.mask)) {
								callbacks.file(name);
							}

							// The fs.stat() callback has returned.
							filesRemaining -= 1;

							// If there is no more file system to search, call .finished().
							if (filesRemaining === 0 && loopsRemaining === 0) {
								callbacks.finished(null);
							}
						});
					});
				});
			});
		};

		walkInternal(directories, options, callbacks);
	}

	return walk;
})();
{'fraction_non_alphanumeric': 0.14991136996708027, 'fraction_numerical': 0.008863003291972652, 'mean_word_length': 2.5236396074933096, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7002658', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1180, 'n_words': 301}