<commit_before>import * as sw from "../../utils/simpleWorker";
import * as contract from "./fileListingContract";

import * as glob from "glob";
import chokidar = require('chokidar');
import {debounce} from "../../../common/utils";
import path = require('path');
import {TypedEvent}  from "../../../common/events";

let listing = new TypedEvent<{relativeFilePaths: string[]}>();

namespace Worker {
    export var echo: typeof contract.worker.echo = (q) => {
        return master.increment(q).then((res) => {
            return {
                text: q.text,
                num: res.num
            };
        });
    }

    export var fileList: typeof contract.worker.fileList = (q) => {
        return listing.current();
    }

    var directoryUnderWatch: string;

    export var setupWatch: typeof contract.worker.setupWatch = (q) => {
        directoryUnderWatch = q.directory;

        var sendNewFileList = debounce((function () {
            var mg = new glob.Glob('**', { cwd: q.directory }, (e, newList) => {
                if (e) {
                    console.error('Globbing error:', e);
                }

                /** Filter out directories */
                newList = newList.filter(nl=> {
                    let p = path.resolve(q.directory,nl);
                    return mg.cache[p] && mg.cache[p] == 'FILE';
                });

                listing.emit({ relativeFilePaths: newList });
            });
        }),500);

        sendNewFileList();

        let watcher = chokidar.watch(directoryUnderWatch);

        // Just the ones that impact file listing
        // https://github.com/paulmillr/chokidar#methods--events
        watcher.on('add', sendNewFileList);
        watcher.on('addDir', sendNewFileList);
        watcher.on('unlink', sendNewFileList);
        watcher.on('unlinkDir', sendNewFileList);

        // Just for changes
        watcher.on('change', (path) => {
            master.fileChanged({ filePath: path });
        });

        return Promise.resolve({});
    }
}

// Ensure that the namespace follows the contract
var _checkTypes: typeof contract.worker = Worker;
// run worker<commit_msg>fix(watcher) ignore initial, should definitely decrease workload<commit_after>import * as sw from "../../utils/simpleWorker";
import * as contract from "./fileListingContract";

import * as glob from "glob";
import chokidar = require('chokidar');
import {debounce} from "../../../common/utils";
import path = require('path');
import {TypedEvent}  from "../../../common/events";

let listing = new TypedEvent<{relativeFilePaths: string[]}>();

namespace Worker {
    export var echo: typeof contract.worker.echo = (q) => {
        return master.increment(q).then((res) => {
            return {
                text: q.text,
                num: res.num
            };
        });
    }

    export var fileList: typeof contract.worker.fileList = (q) => {
        return listing.current();
    }

    var directoryUnderWatch: string;

    export var setupWatch: typeof contract.worker.setupWatch = (q) => {
        directoryUnderWatch = q.directory;

        var sendNewFileList = debounce((function () {
            var mg = new glob.Glob('**', { cwd: q.directory }, (e, newList) => {
                if (e) {
                    console.error('Globbing error:', e);
                }

                /** Filter out directories */
                newList = newList.filter(nl=> {
                    let p = path.resolve(q.directory,nl);
                    return mg.cache[p] && mg.cache[p] == 'FILE';
                });

                listing.emit({ relativeFilePaths: newList });
            });
        }),500);

        sendNewFileList();

        let watcher = chokidar.watch(directoryUnderWatch, { ignoreInitial: true });

        // Just the ones that impact file listing
        // https://github.com/paulmillr/chokidar#methods--events
        watcher.on('add', sendNewFileList);
        watcher.on('addDir', sendNewFileList);
        watcher.on('unlink', sendNewFileList);
        watcher.on('unlinkDir', sendNewFileList);

        // Just for changes
        watcher.on('change', (path) => {
            master.fileChanged({ filePath: path });
        });

        return Promise.resolve({});
    }
}

// Ensure that the namespace follows the contract
var _checkTypes: typeof contract.worker = Worker;
// run worker
{'fraction_non_alphanumeric': 0.1430853491556367, 'fraction_numerical': 0.0013692377909630307, 'mean_word_length': 2.0458651841556637, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2136181', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1195, 'n_words': 295}