<commit_before>////////////////////////////////////////////////////////////////////////////////
// Directory Walker
//
// Walks directories recursively and executes handlers on each object.
// Only synchronous walking is supported because async can only be done
// with CPS (continuation-passing style), which is complicated and adds
// restrictions to the callbacks.
////////////////////////////////////////////////////////////////////////////////
/*global require, exports, process, console */
var	$fs = require('fs'),

// - dirHandler: called for each directory
// - fileHandler: called for each file
// - options:
//   - filter: regex for filtering file names
walker = function (dirHandler, fileHandler, options) {
	// defaults
	dirHandler = dirHandler || function () { };
	fileHandler = fileHandler || function () { };
	options = options || { filter: new RegExp('^.+\\.(wmv|avi|mov|mp4)$', 'ig'), hidden: false };

	var self = {
		// synchronous directory walking
		// when the function returns, the whole process has completed
		// - root: root path
		// - maxDepth: maximum depth
		walkSync: function (root, maxDepth) {
			var	fileCount = 0,
					hasMaxDepth = typeof maxDepth !== 'undefined';
			
			// recursive inner function
			// - relative: path relative to root
			console.log("WALKER - walking path: " + root + (hasMaxDepth ?  ", max depth: " + maxDepth : ""));
			(function walk(relative, depth) {
				var	i,
						files,
						path = root + relative,
						stats = $fs.lstatSync(path);
	
				if (stats.isDirectory()) {
					// calling directory callback
					dirHandler(relative, stats);
					
					// listing directories
					files = $fs.readdirSync(path);
					for (i = 0; i < files.length; i++) {
						// optionally excluding hidden files
						if ((options.hidden || files[i][0] !== '.') &&
							(!hasMaxDepth || depth < maxDepth)) {
							walk(relative + '/' + files[i], depth + 1);
						}
					}
				} else if (stats.isFile()) {
					// calling file handler
					if (path.match(options.filter)) {
						if (fileCount++ % 10 === 0) {
							process.stdout.write(".");
						}
						fileHandler(relative, stats);
					}
				}
			})('', 0);
			process.stdout.write("\n");
			
			return self;
		}
	};
	
	return self;
};

// exports
exports.walker = walker;
<commit_msg>directory walker progress indication improved<commit_after>////////////////////////////////////////////////////////////////////////////////
// Directory Walker
//
// Walks directories recursively and executes handlers on each object.
// Only synchronous walking is supported because async can only be done
// with CPS (continuation-passing style), which is complicated and adds
// restrictions to the callbacks.
////////////////////////////////////////////////////////////////////////////////
/*global require, exports, process, console */
var	$fs = require('fs'),

// - dirHandler: called for each directory
// - fileHandler: called for each file
// - options:
//   - filter: regex for filtering file names
walker = function (dirHandler, fileHandler, options) {
	// defaults
	dirHandler = dirHandler || function () { };
	fileHandler = fileHandler || function () { };
	options = options || { filter: new RegExp('^.+\\.(wmv|avi|mov|mp4)$', 'ig'), hidden: false };

	var self = {
		// synchronous directory walking
		// when the function returns, the whole process has completed
		// - root: root path
		// - maxDepth: maximum depth
		walkSync: function (root, maxDepth) {
			var	fileCount = 0,
					hasMaxDepth = typeof maxDepth !== 'undefined';
			
			// recursive inner function
			// - relative: path relative to root
			console.log("WALKER - walking path: " + root + (hasMaxDepth ?  ", max depth: " + maxDepth : ""));
			
			(function walk(relative, depth) {
				if (fileCount++ % 10 === 0) {
					process.stdout.write(".");
				}

				var	i,
						files,
						path = root + relative,
						stats = $fs.lstatSync(path);
	
				if (stats.isDirectory()) {
					// calling directory callback
					dirHandler(relative, stats);
					
					// listing directories
					files = $fs.readdirSync(path);
					for (i = 0; i < files.length; i++) {
						// optionally excluding hidden files
						if ((options.hidden || files[i][0] !== '.') &&
							(!hasMaxDepth || depth < maxDepth)) {
							walk(relative + '/' + files[i], depth + 1);
						}
					}
				} else if (stats.isFile()) {
					// calling file handler
					if (path.match(options.filter)) {
						fileHandler(relative, stats);
					}
				}
			})('', 0);
			
			if (fileCount > 10) {
				process.stdout.write("\n");
			}
			
			return self;
		}
	};
	
	return self;
};

// exports
exports.walker = walker;

{'fraction_non_alphanumeric': 0.20528804815133275, 'fraction_numerical': 0.004299226139294927, 'mean_word_length': 2.095808383233533, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '893841', 'n_tokens_mistral': 1611, 'n_tokens_neox': 1254, 'n_words': 395}