<commit_before>
var _ = require("underscore"),
    commondir = require("commondir"), 
    fsHelper = require("./fs-helper"), 
    path = require("path"),
    fs = require("fs"),
    buildConfig = require("../lib/build-configurations");

var argv = {},
    dryRun = false;



function Fileset (directory, files, directories) {
	this.directory = directory;
	this.fileMap = files || {};
	this.directoryMap = directories || {};
}

Fileset.prototype.crawl = function (includeModules) {
    var self = this,
        directory = self.directory,
        files = require("findit").findSync(directory),
        fileMap = self.fileMap = {},
        directoryMap = self.directoryMap = {};
    
    includeModules = !!includeModules;
    
    _.each(files, function (filepath) {
        var filename = path.basename(filepath),
            relative = path.relative(self.directory, filepath);
        
        // TODO make this obey .gitignore
        if ((!includeModules && relative.indexOf("node_modules") >= 0) || relative.indexOf(".git") >= 0) {
            return;
        }
        
        if (!fs.existsSync(filepath)) {
            return;
        }
        
        if (fsHelper.isDirectory(filepath)) {
            directoryMap[relative] = filepath;
            return;
        }
        
        var fileObject = buildConfig.platformSpecificity(argv, relative);
        <commit_msg>Don't show .svn files.<commit_after>
var _ = require("underscore"),
    commondir = require("commondir"), 
    fsHelper = require("./fs-helper"), 
    path = require("path"),
    fs = require("fs"),
    buildConfig = require("../lib/build-configurations");

var argv = {},
    dryRun = false;



function Fileset (directory, files, directories) {
	this.directory = directory;
	this.fileMap = files || {};
	this.directoryMap = directories || {};
}

Fileset.prototype.crawl = function (includeModules) {
    var self = this,
        directory = self.directory,
        files = require("findit").findSync(directory),
        fileMap = self.fileMap = {},
        directoryMap = self.directoryMap = {};
    
    includeModules = !!includeModules;
    
    _.each(files, function (filepath) {
        var filename = path.basename(filepath),
            relative = path.relative(self.directory, filepath);
        
        // TODO make this obey .gitignore
        if ((!includeModules && relative.indexOf("node_modules") >= 0) || relative.indexOf(".git") >= 0 || relative.indexOf(".svn") >= 0) {
            return;
        }
        
        if (!fs.existsSync(filepath)) {
            return;
        }
        
        if (fsHelper.isDirectory(filepath)) {
            directoryMap[relative] = filepath;
            return;
        }
        
        var fileObject = buildConfig.platformSpecificity(argv, relative);
        
{'fraction_non_alphanumeric': 0.1381156316916488, 'fraction_numerical': 0.0017844396859386152, 'mean_word_length': 2.3054245283018866, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6489544', 'n_tokens_mistral': 791, 'n_tokens_neox': 774, 'n_words': 159}