<commit_before>import * as sw from "../../utils/simpleWorker";
import * as contract from "./fileListingContract";
import * as fs from "fs";
import * as fsu from "../../utils/fsu";
import * as utils from "../../../common/utils";

import * as glob from "glob";
import chokidar = require('chokidar');
import {throttle} from "../../../common/utils";
import path = require('path');
import {TypedEvent}  from "../../../common/events";
import * as types from "../../../common/types";

/** A Map for faster live calculations */
type LiveList = { [filePath: string]: types.FilePathType };

namespace Worker {
    export var echo: typeof contract.worker.echo = (q) => {
        return master.increment(q).then((res) => {
            return {
                text: q.text,
                num: res.num
            };
        });
    }

    // export var fileList: typeof contract.worker.fileList = (q) => {
    //     return listing.current();
    // }

    var directoryUnderWatch: string;

    export var setupWatch: typeof contract.worker.setupWatch = (q) => {
        directoryUnderWatch = q.directory;

        let completed = false;
        let liveList: LiveList = {};

        // Utility to send new file list
        var sendNewFileList = () => {
            let filePaths = Object.keys(liveList)
                .filter(x=>
                    // Remove .git we have no use for that here
                    !x.includes('/.git/')
                    // MAC
                    && x !== '.DS_Store'
                )
                // sort
                .sort((a, b) => {
                    // sub dir wins!
                    if (b.startsWith(a)){
                        return -1;
                    }
                    if (a.startsWith(b)){
                        return 1;
                    }

                    // The next sorts are slow and only done after initial listing!
                    if (!completed) {
                        return a.length - b.length;
                    }

                    // sort by name
                    return a.toLowerCase().localeCompare(b.toLowerCase());
                })

                // Convert ot file path type
                .map(filePath => {
                    let type = liveList[filePath];
                    return {filePath, type};
                });

            master.fileListUpdated({
                filePaths,
                completed
            });
        };

        /**
         * Slower version for
         * - initial partial serach
         * - later updates which might be called a lot because of some directory of files removed
         */
         let sendNewFileListThrottled = throttle(sendNewFileList, 500);

         /**
          * Utility function to get the listing from a directory
          */
         const getListing = (dirPath: string): Promise<types.FilePath[]> => {
             return new Promise((resolve)=>{
                 let mg = new glob.Glob('**', { cwd: dirPath }, (e, globResult) => {
                     if (e) {
                         console.error('Globbing error:', e);
                     }

                     let list = globResult.map(nl=> {
                         let p = path.resolve(q.directory,nl);
                         let type = mg.cache[p] && mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                         return {
                             filePath: fsu.consistentPath(p),
                             type,
                         }
                     });

                     resolve(list);
                 });
             });
         }

        // create initial list using 10x faster glob.Glob!
        (function () {
           var mg = new glob.Glob('**', { cwd: q.directory }, (e, newList) => {
               if (e) {
                   console.error('Globbing error:', e);
               }

               let list = newList.map(nl=> {
                   let p = path.resolve(q.directory,nl);
                   let type = mg.cache[p] && mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                   return {
                       filePath: fsu.consistentPath(p),
                       type,
                   }
               });

               // Initial search complete!
               completed = true;
               list.forEach(entry => liveList[entry.filePath] = entry.type);
               sendNewFileList();
           });
           /** Still send the listing while globbing so user gets immediate feedback */
           mg.on('match',(match)=>{
               let p = path.resolve(q.directory,match);
               if (mg.cache[p]){
                  liveList[fsu.consistentPath(p)] = mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                  sendNewFileListThrottled();
               }
           });
       })();


        function fileAdded(filePath: string, stat: fs.Stats) {
            filePath = fsu.consistentPath(filePath);

            // Only send if we don't know about this already (because of faster initial scan)
            if (!liveList[filePath]) {
                let type = stat && stat.isDirectory() ? types.FilePathType.Dir : types.FilePathType.File;
                liveList[filePath] = type;
                sendNewFileListThrottled();
            }
        }

        function dirAdded(dirPath: string) {
            dirPath = fsu.consistentPath(dirPath);

            /**
             * - glob the folder
             * - send the folder throttled
             */
            getListing(dirPath).then(res=>{
                res.forEach(fpDetails => {
                    if (!liveList[fpDetails.filePath]) {
                        let type = fpDetails.type
                        liveList[fpDetails.filePath] = type;
                    }
                });
                sendNewFileListThrottled();
            }).catch(res=>{
                console.error('[FLW] DirPath listing failed:', dirPath, res);
            });
        }<commit_msg>WIP file worker : avoid needless stat<commit_after>import * as sw from "../../utils/simpleWorker";
import * as contract from "./fileListingContract";
import * as fs from "fs";
import * as fsu from "../../utils/fsu";
import * as utils from "../../../common/utils";

import * as glob from "glob";
import chokidar = require('chokidar');
import {throttle} from "../../../common/utils";
import path = require('path');
import {TypedEvent}  from "../../../common/events";
import * as types from "../../../common/types";

/** A Map for faster live calculations */
type LiveList = { [filePath: string]: types.FilePathType };

namespace Worker {
    export var echo: typeof contract.worker.echo = (q) => {
        return master.increment(q).then((res) => {
            return {
                text: q.text,
                num: res.num
            };
        });
    }

    // export var fileList: typeof contract.worker.fileList = (q) => {
    //     return listing.current();
    // }

    var directoryUnderWatch: string;

    export var setupWatch: typeof contract.worker.setupWatch = (q) => {
        directoryUnderWatch = q.directory;

        let completed = false;
        let liveList: LiveList = {};

        // Utility to send new file list
        var sendNewFileList = () => {
            let filePaths = Object.keys(liveList)
                .filter(x=>
                    // Remove .git we have no use for that here
                    !x.includes('/.git/')
                    // MAC
                    && x !== '.DS_Store'
                )
                // sort
                .sort((a, b) => {
                    // sub dir wins!
                    if (b.startsWith(a)){
                        return -1;
                    }
                    if (a.startsWith(b)){
                        return 1;
                    }

                    // The next sorts are slow and only done after initial listing!
                    if (!completed) {
                        return a.length - b.length;
                    }

                    // sort by name
                    return a.toLowerCase().localeCompare(b.toLowerCase());
                })

                // Convert ot file path type
                .map(filePath => {
                    let type = liveList[filePath];
                    return {filePath, type};
                });

            master.fileListUpdated({
                filePaths,
                completed
            });
        };

        /**
         * Slower version for
         * - initial partial serach
         * - later updates which might be called a lot because of some directory of files removed
         */
         let sendNewFileListThrottled = throttle(sendNewFileList, 500);

         /**
          * Utility function to get the listing from a directory
          */
         const getListing = (dirPath: string): Promise<types.FilePath[]> => {
             return new Promise((resolve)=>{
                 let mg = new glob.Glob('**', { cwd: dirPath }, (e, globResult) => {
                     if (e) {
                         console.error('Globbing error:', e);
                     }

                     let list = globResult.map(nl=> {
                         let p = path.resolve(q.directory,nl);
                         let type = mg.cache[p] && mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                         return {
                             filePath: fsu.consistentPath(p),
                             type,
                         }
                     });

                     resolve(list);
                 });
             });
         }

        // create initial list using 10x faster glob.Glob!
        (function () {
           var mg = new glob.Glob('**', { cwd: q.directory }, (e, newList) => {
               if (e) {
                   console.error('Globbing error:', e);
               }

               let list = newList.map(nl=> {
                   let p = path.resolve(q.directory,nl);
                   let type = mg.cache[p] && mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                   return {
                       filePath: fsu.consistentPath(p),
                       type,
                   }
               });

               // Initial search complete!
               completed = true;
               list.forEach(entry => liveList[entry.filePath] = entry.type);
               sendNewFileList();
           });
           /** Still send the listing while globbing so user gets immediate feedback */
           mg.on('match',(match)=>{
               let p = path.resolve(q.directory,match);
               if (mg.cache[p]){
                  liveList[fsu.consistentPath(p)] = mg.cache[p] == 'FILE' ? types.FilePathType.File : types.FilePathType.Dir;
                  sendNewFileListThrottled();
               }
           });
       })();


        function fileAdded(filePath: string) {
            filePath = fsu.consistentPath(filePath);

            // Only send if we don't know about this already (because of faster initial scan)
            if (!liveList[filePath]) {
                let type = types.FilePathType.File;
                liveList[filePath] = type;
                sendNewFileListThrottled();
            }
        }

        function dirAdded(dirPath: string) {
            dirPath = fsu.consistentPath(dirPath);

            /**
             * - glob the folder
             * - send the folder throttled
             */
            getListing(dirPath).then(res=>{
                res.forEach(fpDetails => {
                    if (!liveList[fpDetails.filePath]) {
                        let type = fpDetails.type
                        liveList[fpDetails.filePath] = type;
                    }
                });
                sendNewFileListThrottled();
            }).catch(res=>{
                console.error('[FLW] DirPath listing failed:', dirPath, res);
            });
        }
{'fraction_non_alphanumeric': 0.11908308507116849, 'fraction_numerical': 0.0011585567692816948, 'mean_word_length': 1.204889618682722, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2885830', 'n_tokens_mistral': 3102, 'n_tokens_neox': 2932, 'n_words': 758}