<commit_before>var fs = require('fs'),
	p = require('path'),
	u = require('util');

function doFilter(filters,path,stat){
	return filters.every((f)=>{
		if(u.isFunction(f))
			return f(path,stat);
		if(u.isRegExp(f))
			return f.test(path);
		return true;
	});
}	

function rreaddirSync(path, pfilters, plist) {
  var list = plist || [], filters = u.isArray(pfilters) ? pfilters : [pfilters],
	files = fs.readdirSync(path);

	files.forEach(function (file) {
		var subPath = p.join(path, file),
			stats = fs.lstatSync(subPath);
		doFilter(filters,subPath,stats) && list.push(subPath);
		stats.isDirectory() && rreaddirSync(subPath, filters, list);
	});

  return list;
}

rreaddirSync.ONLY_DIR = (p,s)=>s.isDirectory();
rreaddirSync.ONLY_FILE = (p,s)=>s.isFile();

module.exports = rreaddirSync;<commit_msg>Added depth feature and subbdir exclusion<commit_after>var fs = require('fs'),
	p = require('path'),
	u = require('util');

function doFilter(filters,path,stat){
	return filters.every((f)=>{
		(u.isRegExp(f)) && console.log("Path: " + path + " RegExp: " + f + " RES: " + f.test(path));
		return (u.isFunction(f))
			? f(path,stat) :
		(u.isRegExp(f)) 
			? f.test(path) :
		true;
	});
}	

function rreaddirSync(path, pfilters, depth, plist, pcurrDepth) {
  var list = plist || [], filters = u.isArray(pfilters) ? pfilters : [pfilters],
	files = fs.readdirSync(path), currDepth = u.isNumber(pcurrDepth) ? pcurrDepth : 1;

	files.forEach(function (file) {
		var subPath = p.join(path, file),
			stats = fs.lstatSync(subPath);
		if(doFilter(filters,subPath,stats)){
			list.push(subPath);
			(!u.isNumber(depth) || currDepth < depth) && 
			stats.isDirectory() && rreaddirSync(subPath, filters, depth, list, currDepth + 1);
		}
	});
  return list;
}

var exp = (p,f,l,d)=>rreaddirSync(p,f,d,l);

exp.ONLY_DIR = (p,s)=>s.isDirectory();
exp.ONLY_FILE = (p,s)=>s.isFile();

module.exports = exp;
{'fraction_non_alphanumeric': 0.19145569620253164, 'fraction_numerical': 0.0010548523206751054, 'mean_word_length': 4.76595744680851, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3029474', 'n_tokens_mistral': 768, 'n_tokens_neox': 732, 'n_words': 132}