<commit_before>var assert = require("assert");
var path = require("path");
var fs = require("fs");
var spawn = require("child_process").spawn;
var Q = require("q");
var EventEmitter = require("events").EventEmitter;
var inherits = require("util").inherits;
var util = require("./util");

function Watcher(sourceDir, persistent) {
    var self = this;
    assert.ok(self instanceof Watcher);
    assert.ok(self instanceof EventEmitter);

    EventEmitter.call(this);

    Object.defineProperties(self, {
        sourceDir: { value: sourceDir },
        sourceCache: { value: {} },
        watched: { value: {} },
        persistent: { value: !!persistent }
    });
}

inherits(Watcher, EventEmitter);
var Wp = Watcher.prototype;

Wp.readFileP = function(relativePath) {
    relativePath = path.normalize(relativePath);

    var cache = this.sourceCache;
    if (cache.hasOwnProperty(relativePath))
        return cache[relativePath];

    return cache[relativePath] = this.noCacheReadFileP(relativePath);
};

Wp.noCacheReadFileP = function(relativePath) {
    relativePath = path.normalize(relativePath);
    var fullPath = path.join(this.sourceDir, relativePath);
    var promise = util.readFileP(fullPath);
    this.sourceCache[relativePath] = promise;
    return promise;
};

Wp.watch = function(relativePath) {
    var self = this;
    var watched = self.watched;

    relativePath = path.normalize(relativePath);
    if (!watched.hasOwnProperty(relativePath)) {
        var fullPath = path.join(self.sourceDir, relativePath);
        var options = { persistent: self.persistent };

        function handler(event) {
            var oldP = self.readFileP(relativePath);
            var newP = self.noCacheReadFileP(relativePath);

            Q.all([oldP, newP]).spread(function(oldData, newData) {
                if (oldData !== newData)
                    self.emit("changed", relativePath);
            }).done();
        }

        try {
            fs.watch(fullPath, options, handler);
            // Don't mark the file as watched if fs.watch threw an exception.
            watched[relativePath] = true;
        } catch (e) {
            util.log.err(
                "unable to watch file " + relativePath +
                " (" + e + ")",
                "yellow");
        }
    }
};

function run(cmd, args) {
    return spawn(cmd, args, {
        stdio: "pipe",
        env: process.env
    });
}

var grepExtensionDef = Q.defer();
var grepExtensionP = grepExtensionDef.promise;

run("grep", [
    "--quiet",
    "--perl-regexp",
    "spawn",
    __filename
]).on("close", function(code) {
    if (code === 0 || code === 1) {
        grepExtensionDef.resolve("--perl-regexp");
    } else {
        grepExtensionDef.resolve("--extended-regexp");
    }
});

Wp.grepP = function(pattern) {
    var watcher = this;
    return grepExtensionP.then(function(extension) {
        return grepP.call(watcher, pattern, extension);
    });
};

function grepP(pattern, grepExtension) {
    var self = this;
    var sourceDir = self.sourceDir;

    var grep = run("grep", [
        "--recursive",
        "--only-matching",
        "--null", // separate file from match with \0 instead of :
        grepExtension,
        pattern,
        sourceDir
    ]);

    var outs = [];
    var errs = [];
    var closed = false;

    grep.stdout.on("data", function(data) {
        assert.ok(!closed);
        outs.push(data);
    });

    grep.stderr.on("data", function(data) {
        assert.ok(!closed);
        errs.push(data);
    });

    var deferred = Q.defer();
    var promise = deferred.promise;

    grep.on("close", function(code) {
        assert.ok(!closed);
        closed = true;

        switch (code) {
        default:
            if (errs.length > 0) {
                util.log.err(errs.join(""));
            }

            // intentionally fall through

        case 0: case 1: // 1 means no results
            deferred.resolve(outs.join(""));
        }
    });

    return promise.then(function(out) {
        var pathToMatch = {};

        out.split("\n").forEach(function(line) {
            if ((line = line.trim())) {
                var splat = line.split("\0"); // see --null above
                var relPath = path.relative(sourceDir, splat.shift());

                // Only record the first match in any particular file.
                if (pathToMatch.hasOwnProperty(relPath))
                    return;

                pathToMatch[relPath] = splat.join("\0");

                // TODO Watch the whole directory tree to catch new files
                // that match the grepped-for pattern.
                self.watch(relPath);
            }
        });

        return pathToMatch;
    });
};

exports.Watcher = Watcher;
<commit_msg>Make the Watcher actually watch stuff again.<commit_after>var assert = require("assert");
var path = require("path");
var fs = require("fs");
var spawn = require("child_process").spawn;
var Q = require("q");
var EventEmitter = require("events").EventEmitter;
var inherits = require("util").inherits;
var util = require("./util");

function Watcher(sourceDir, persistent) {
    var self = this;
    assert.ok(self instanceof Watcher);
    assert.ok(self instanceof EventEmitter);

    EventEmitter.call(this);

    Object.defineProperties(self, {
        sourceDir: { value: sourceDir },
        sourceCache: { value: {} },
        watched: { value: {} },
        persistent: { value: !!persistent }
    });
}

inherits(Watcher, EventEmitter);
var Wp = Watcher.prototype;

Wp.readFileP = function(relativePath) {
    relativePath = path.normalize(relativePath);

    var cache = this.sourceCache;
    if (cache.hasOwnProperty(relativePath))
        return cache[relativePath];

    return cache[relativePath] = this.noCacheReadFileP(relativePath);
};

Wp.noCacheReadFileP = function(relativePath) {
    relativePath = path.normalize(relativePath);
    var fullPath = path.join(this.sourceDir, relativePath);
    var promise = util.readFileP(fullPath);
    this.sourceCache[relativePath] = promise;
    this.watch(relativePath);
    return promise;
};

Wp.watch = function(relativePath) {
    var self = this;
    var watched = self.watched;

    relativePath = path.normalize(relativePath);
    if (!watched.hasOwnProperty(relativePath)) {
        var fullPath = path.join(self.sourceDir, relativePath);
        var options = { persistent: self.persistent };

        function handler(event) {
            var oldP = self.readFileP(relativePath);
            var newP = self.noCacheReadFileP(relativePath);

            Q.all([oldP, newP]).spread(function(oldData, newData) {
                if (oldData !== newData)
                    self.emit("changed", relativePath);
            }).done();
        }

        try {
            fs.watch(fullPath, options, handler);
            // Don't mark the file as watched if fs.watch threw an exception.
            watched[relativePath] = true;
        } catch (e) {
            util.log.err(
                "unable to watch file " + relativePath +
                " (" + e + ")",
                "yellow");
        }
    }
};

function run(cmd, args) {
    return spawn(cmd, args, {
        stdio: "pipe",
        env: process.env
    });
}

var grepExtensionDef = Q.defer();
var grepExtensionP = grepExtensionDef.promise;

run("grep", [
    "--quiet",
    "--perl-regexp",
    "spawn",
    __filename
]).on("close", function(code) {
    if (code === 0 || code === 1) {
        grepExtensionDef.resolve("--perl-regexp");
    } else {
        grepExtensionDef.resolve("--extended-regexp");
    }
});

Wp.grepP = function(pattern) {
    var watcher = this;
    return grepExtensionP.then(function(extension) {
        return grepP.call(watcher, pattern, extension);
    });
};

function grepP(pattern, grepExtension) {
    var self = this;
    var sourceDir = self.sourceDir;

    var grep = run("grep", [
        "--recursive",
        "--only-matching",
        "--null", // separate file from match with \0 instead of :
        grepExtension,
        pattern,
        sourceDir
    ]);

    var outs = [];
    var errs = [];
    var closed = false;

    grep.stdout.on("data", function(data) {
        assert.ok(!closed);
        outs.push(data);
    });

    grep.stderr.on("data", function(data) {
        assert.ok(!closed);
        errs.push(data);
    });

    var deferred = Q.defer();
    var promise = deferred.promise;

    grep.on("close", function(code) {
        assert.ok(!closed);
        closed = true;

        switch (code) {
        default:
            if (errs.length > 0) {
                util.log.err(errs.join(""));
            }

            // intentionally fall through

        case 0: case 1: // 1 means no results
            deferred.resolve(outs.join(""));
        }
    });

    return promise.then(function(out) {
        var pathToMatch = {};

        out.split("\n").forEach(function(line) {
            if ((line = line.trim())) {
                var splat = line.split("\0"); // see --null above
                var relPath = path.relative(sourceDir, splat.shift());

                // Only record the first match in any particular file.
                if (pathToMatch.hasOwnProperty(relPath))
                    return;

                pathToMatch[relPath] = splat.join("\0");

                // TODO Watch the whole directory tree to catch new files
                // that match the grepped-for pattern.
                self.watch(relPath);
            }
        });

        return pathToMatch;
    });
};

exports.Watcher = Watcher;

{'fraction_non_alphanumeric': 0.13723038449515473, 'fraction_numerical': 0.001875586120662707, 'mean_word_length': 2.1593153390388413, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5166174', 'n_tokens_mistral': 2863, 'n_tokens_neox': 2754, 'n_words': 629}