<commit_before>var through = require('through');
var copy = require('shallow-copy');
var browserify = require('browserify');
var fs = require('fs');
var chokidar = require('chokidar');

module.exports = watchify;
watchify.browserify = browserify;

function watchify (opts) {
    if (!opts) opts = {};
    var b = typeof opts.bundle === 'function' ? opts : browserify(opts);
    var cache = {};
    var pkgcache = {};
    var watching = {};
    var pending = false;
    var queuedCloses = {};
    var queuedDeps = {};
    var changingDeps = {};
    var first = true;
    
    if (opts.cache) {
        cache = opts.cache;
        delete opts.cache;
        first = false;
    }
    
    if (opts.pkgcache) {
        pkgcache = opts.pkgcache;
        delete opts.pkgcache;
    }
    
    b.on('package', function (file, pkg) {
        pkgcache[file] = pkg;
    });
    
    b.on('dep', function (dep) {
        queuedDeps[dep.id] = copy(dep);
    });
    
    var fwatchers = {};
    var fwatcherFiles = {};
    b.on('bundle', function (bundle) {
        bundle.on('transform', function (tr, mfile) {
            if (!fwatchers[mfile]) fwatchers[mfile] = [];
            if (!fwatcherFiles[mfile]) fwatcherFiles[mfile] = [];

            tr.on('file', function (file) {
                if (!fwatchers[mfile]) return;
                if (fwatchers[mfile].indexOf(file) >= 0) return;
                
                var w = chokidar.watch(file, {persistent: true});
                w.on('error', b.emit.bind(b, 'error'));
                w.on('change', function () {
                    invalidate(mfile);
                });
                fwatchers[mfile].push(w);
                fwatcherFiles[mfile].push(file);
            });
        });
    });
    
    var watchers = {};
    function addDep (dep) {
        if (watching[dep.id]) return;
        watching[dep.id] = true;
        cache[dep.id] = dep;
        
        var watcher = chokidar.watch(dep.id, {persistent: true});
        watchers[dep.id] = watcher;
        watcher.on('error', b.emit.bind(b, 'error'));
        watcher.on('change', function () {
            invalidate(dep.id);
        });
    }
    
    function invalidate (id) {
        delete cache[id];
        if (fwatchers[id]) {
            fwatchers[id].forEach(function (w) {
                queuedCloses[id + '-' + id] = w;
            });
            delete fwatchers[id];
            delete fwatcherFiles[id];
        }
        queuedCloses[id] = watchers[id];
        changingDeps[id] = true
        
        // wait for the disk/editor to quiet down first:
        if (!pending) setTimeout(function () {
            pending = false;
            b.emit('update', Object.keys(changingDeps));
            changingDeps = {};
        
        }, opts.delay || 600);
        pending = true;
    }
    
    var bundle = b.bundle.bind(b);
    b.close = function () {
        Object.keys(fwatchers).forEach(function (id) {
            fwatchers[id].forEach(function (w) { w.close() });
        });
        Object.keys(watchers).forEach(function (id) {
            watchers[id].close();
        });
    };
    
    b.bundle = function (opts_, cb) {
        if (b._pending) return bundle(opts_, cb);
        
        if (typeof opts_ === 'function') {
            cb = opts_;
            opts_ = {};
        }
        if (!opts_) opts_ = {};
        if (!first) opts_.cache = cache;
        opts_.includePackage = true;
        opts_.packageCache = pkgcache;
        
        // we only want to mess with the listeners if the bundle was created
        // successfully, e.g. on the 'end' event.
        var outStream = bundle(opts_, cb);
        outStream.on('error', function (err) {
            var updated = false;
            b.once('update', function () { updated = true });
            
            if (err.type === 'not found') {
                (function f () {
                    if (updated) return;
                    fs.exists(err.filename, function (ex) {
                        if (ex) b.emit('update', [ err.filename ])
                        else setTimeout(f, opts.delay || 600)
                    });
                })();
            }
        });
        outStream.on('end', end);
        function end () {
            first = false;
            var depId;
            for (depId in queuedCloses) {
                queuedCloses[depId].close();
                watching[depId] = false;
            }
            queuedCloses = {};
            for (depId in queuedDeps) {
                addDep(queuedDeps[depId]);
            }
            queuedDeps = {};
        }
        return outStream;
    };
    
    return b;
}
<commit_msg>only watch once for changes on transform files<commit_after>var through = require('through');
var copy = require('shallow-copy');
var browserify = require('browserify');
var fs = require('fs');
var chokidar = require('chokidar');

module.exports = watchify;
watchify.browserify = browserify;

function watchify (opts) {
    if (!opts) opts = {};
    var b = typeof opts.bundle === 'function' ? opts : browserify(opts);
    var cache = {};
    var pkgcache = {};
    var watching = {};
    var pending = false;
    var queuedCloses = {};
    var queuedDeps = {};
    var changingDeps = {};
    var first = true;
    
    if (opts.cache) {
        cache = opts.cache;
        delete opts.cache;
        first = false;
    }
    
    if (opts.pkgcache) {
        pkgcache = opts.pkgcache;
        delete opts.pkgcache;
    }
    
    b.on('package', function (file, pkg) {
        pkgcache[file] = pkg;
    });
    
    b.on('dep', function (dep) {
        queuedDeps[dep.id] = copy(dep);
    });
    
    var fwatchers = {};
    var fwatcherFiles = {};
    b.on('bundle', function (bundle) {
        bundle.on('transform', function (tr, mfile) {
            if (!fwatchers[mfile]) fwatchers[mfile] = [];
            if (!fwatcherFiles[mfile]) fwatcherFiles[mfile] = [];

            tr.on('file', function (file) {
                if (!fwatchers[mfile]) return;
                if (fwatcherFiles[mfile].indexOf(file) >= 0) return;
                
                var w = chokidar.watch(file, {persistent: true});
                w.on('error', b.emit.bind(b, 'error'));
                w.on('change', function () {
                    invalidate(mfile);
                });
                fwatchers[mfile].push(w);
                fwatcherFiles[mfile].push(file);
            });
        });
    });
    
    var watchers = {};
    function addDep (dep) {
        if (watching[dep.id]) return;
        watching[dep.id] = true;
        cache[dep.id] = dep;
        
        var watcher = chokidar.watch(dep.id, {persistent: true});
        watchers[dep.id] = watcher;
        watcher.on('error', b.emit.bind(b, 'error'));
        watcher.on('change', function () {
            invalidate(dep.id);
        });
    }
    
    function invalidate (id) {
        delete cache[id];
        if (fwatchers[id]) {
            fwatchers[id].forEach(function (w) {
                queuedCloses[id + '-' + id] = w;
            });
            delete fwatchers[id];
            delete fwatcherFiles[id];
        }
        queuedCloses[id] = watchers[id];
        changingDeps[id] = true
        
        // wait for the disk/editor to quiet down first:
        if (!pending) setTimeout(function () {
            pending = false;
            b.emit('update', Object.keys(changingDeps));
            changingDeps = {};
        
        }, opts.delay || 600);
        pending = true;
    }
    
    var bundle = b.bundle.bind(b);
    b.close = function () {
        Object.keys(fwatchers).forEach(function (id) {
            fwatchers[id].forEach(function (w) { w.close() });
        });
        Object.keys(watchers).forEach(function (id) {
            watchers[id].close();
        });
    };
    
    b.bundle = function (opts_, cb) {
        if (b._pending) return bundle(opts_, cb);
        
        if (typeof opts_ === 'function') {
            cb = opts_;
            opts_ = {};
        }
        if (!opts_) opts_ = {};
        if (!first) opts_.cache = cache;
        opts_.includePackage = true;
        opts_.packageCache = pkgcache;
        
        // we only want to mess with the listeners if the bundle was created
        // successfully, e.g. on the 'end' event.
        var outStream = bundle(opts_, cb);
        outStream.on('error', function (err) {
            var updated = false;
            b.once('update', function () { updated = true });
            
            if (err.type === 'not found') {
                (function f () {
                    if (updated) return;
                    fs.exists(err.filename, function (ex) {
                        if (ex) b.emit('update', [ err.filename ])
                        else setTimeout(f, opts.delay || 600)
                    });
                })();
            }
        });
        outStream.on('end', end);
        function end () {
            first = false;
            var depId;
            for (depId in queuedCloses) {
                queuedCloses[depId].close();
                watching[depId] = false;
            }
            queuedCloses = {};
            for (depId in queuedDeps) {
                addDep(queuedDeps[depId]);
            }
            queuedDeps = {};
        }
        return outStream;
    };
    
    return b;
}

{'fraction_non_alphanumeric': 0.14177754124534328, 'fraction_numerical': 0.0014901543374135177, 'mean_word_length': 1.5449620801733477, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5403934', 'n_tokens_mistral': 2811, 'n_tokens_neox': 2750, 'n_words': 605}