<commit_before>import * as fs from "fs";
import * as path from "path";

import * as bslglobals from "./features/bslGlobals";

let exec = require("child-process-promise").exec;
let iconv = require("iconv-lite");
let loki = require("lokijs");
let Parser = require("onec-syntaxparser");
let FileQueue = require("filequeue");
let fq = new FileQueue(500);

import * as vscode from "vscode";

export class Global {
    exec: string;
    cache: any;
    db: any;
    dblocal: any;
    dbcalls: any;
    globalfunctions: any;
    globalvariables: any;
    keywords: any;
    toreplaced: any;
    methodForDescription: any = undefined;
    private cacheUpdates: boolean;
    private pathSeparator: string;

    getCacheLocal(filename: string, word: string, source, update: boolean = false, allToEnd: boolean = true, fromFirst: boolean = true) {
        let suffix = allToEnd  ? "" : "$";
        let prefix = fromFirst ? "^" : "";
        let querystring = {"name": {"$regex": new RegExp(prefix + word + suffix, "i")}};
        let entries = new Parser().parse(source).getMethodsTable().find(querystring);
        return entries;
    }

    getRefsLocal(filename: string, source: string) {
        let parsesModule = new Parser().parse(source);
        let entries = parsesModule.getMethodsTable().find();
        let count = 0;
        let collection = this.cache.addCollection(filename);
        this.updateReferenceCalls(collection, parsesModule.context.CallsPosition, "GlobalModuleText", filename);
        for (let y = 0; y < entries.length; ++y) {
            let item = entries[y];
            this.updateReferenceCalls(collection, item._method.CallsPosition, item, filename);
        }
        return collection;
    }

    getReplaceMetadata () {
        return {
            "AccountingRegisters": "РегистрыБухгалтерии",
            "AccumulationRegisters": "РегистрыНакопления",
            "BusinessProcesses": "БизнесПроцессы",
            "CalculationRegisters": "РегистрыРасчета",
            "ChartsOfAccounts": "ПланыСчетов",
            "ChartsOfCalculationTypes": "ПланыВидовРасчета",
            "ChartsOfCharacteristicTypes": "ПланыВидовХарактеристик",
            "Constants": "Константы",
            "Catalogs": "Справочники",
            "DataProcessors": "Обработки",
            "DocumentJournals": "ЖурналыДокументов",
            "Documents": "Документы",
            "Enums": "Перечисления",
            "ExchangePlans": "ПланыОбмена",
            "InformationRegisters": "РегистрыСведений",
            "Reports": "Отчеты",
            "SettingsStorages": "ХранилищаНастроек",
            "Tasks": "Задачи"
        };
    }

    getModuleForPath(fullpath: string, rootPath: string): any {
        let splitsymbol = "/";
        let moduleArray: Array<string> = fullpath.substr(rootPath.length + (rootPath.slice(-1) === "\\" ? 0 : 1)).split(splitsymbol);
        let moduleStr: string = "";
            let hierarchy = moduleArray.length;
            if (hierarchy > 3) {
                if (moduleArray[hierarchy - 4].startsWith("CommonModules")) {
                    moduleStr = moduleArray[hierarchy - 3];
                } else if (hierarchy > 3 && this.toreplaced[moduleArray[hierarchy - 4]] !== undefined) {
                    moduleStr = this.toreplaced[moduleArray[hierarchy - 4]] + "." + moduleArray[hierarchy - 3];
                }
            }

        return moduleStr;
    }

    addtocachefiles(files: Array<vscode.Uri>, isbsl: boolean = false, rootPath: any = null): any {
        if (!rootPath) {
            rootPath = vscode.workspace.rootPath;
        }
        let filesLength = files.length;
        for (let i = 0; i < filesLength; ++i) {
            let fullpath = files[i].toString();
            fullpath = decodeURIComponent(fullpath);
            if (fullpath.startsWith("file:")) {
                if (process.platform === "win32") {
                    fullpath = fullpath.substr(8);
                } else {
                    fullpath = fullpath.substr(7);
                }
            }
            fq.readFile(fullpath, {encoding: "utf8"}, (err, source) => {
                if (err) {
                    throw err;
                }
                let moduleStr = "";
                if (fullpath.endsWith(".bsl")) {
                    moduleStr = this.getModuleForPath(fullpath, rootPath);
                }
                source = source.replace(/\r/g, "\r\n");
                let parsesModule = new Parser().parse(source);
                let entries = parsesModule.getMethodsTable().find();
                if (i % 100) {
                    vscode.window.setStatusBarMessage("Обновляем кэш файла № " + i + " из " + filesLength, 2000);
                }
                this.updateReferenceCalls(this.dbcalls, parsesModule.context.CallsPosition, "GlobalModuleText", fullpath);
                for (let y = 0; y < entries.length; ++y) {
                    let item = entries[y];
                    this.updateReferenceCalls(this.dbcalls, item._method.CallsPosition, item, fullpath);
                    item["filename"] = fullpath;
                    let newItem: MethodValue = {
                        "name": String(item.name),
                        "isproc": Boolean(item.isproc),
                        "line": item.line,
                        "endline": item.endline,
                        "context": item.context,
                        "_method": item._method,
                        "filename": fullpath,
                        "module": moduleStr,
                        "description": item.description
                    };
                    this.db.insert(newItem);
                }
                if (i === filesLength - 1) {
                    vscode.window.setStatusBarMessage("Обновление кэша завершено", 3000);
                    this.cacheUpdates = true;
                }
            });
        }
    }

    updateCache(): any {
        console.log("update cache");
        vscode.window.setStatusBarMessage("Запущено заполнение кеша", 3000);
        let configuration = vscode.workspace.getConfiguration("language-1c-bsl");
        let basePath: string = String(configuration.get("rootPath"));
        let rootPath = vscode.workspace.rootPath;
        if (rootPath) {
            rootPath = path.join(vscode.workspace.rootPath, basePath);
            this.db = this.cache.addCollection("ValueTable");
            this.dbcalls = this.cache.addCollection("Calls");

            let self = this;
            let files = vscode.workspace.findFiles(basePath !== "" ? basePath.substr(2) + "/**" : "" + "**/*.bsl", "", 1000);
            files.then((value) => {
                this.addtocachefiles(value, false, rootPath);
            }, (reason) => {
                console.log(reason);
            });
            files = vscode.workspace.findFiles(basePath !== "" ? basePath.substr(2) + "/**" : "" + "**/*.os", "", 10000);
            files.then((value) => {
                this.addtocachefiles(value, true, rootPath);
            }, (reason) => {
                console.log(reason);
            });
        }
    };

    customUpdateCache(source: string, filename: string) {
        if (!this.cacheUpdates) {
            return;
        }
        let configuration = vscode.workspace.getConfiguration("language-1c-bsl");
        let basePath: string = String(configuration.get("rootPath"));
        let rootPath = path.join(vscode.workspace.rootPath, basePath);<commit_msg>Убрано ограничение на количество файлов поиска<commit_after>import * as fs from "fs";
import * as path from "path";

import * as bslglobals from "./features/bslGlobals";

let exec = require("child-process-promise").exec;
let iconv = require("iconv-lite");
let loki = require("lokijs");
let Parser = require("onec-syntaxparser");
let FileQueue = require("filequeue");
let fq = new FileQueue(500);

import * as vscode from "vscode";

export class Global {
    exec: string;
    cache: any;
    db: any;
    dblocal: any;
    dbcalls: any;
    globalfunctions: any;
    globalvariables: any;
    keywords: any;
    toreplaced: any;
    methodForDescription: any = undefined;
    private cacheUpdates: boolean;
    private pathSeparator: string;

    getCacheLocal(filename: string, word: string, source, update: boolean = false, allToEnd: boolean = true, fromFirst: boolean = true) {
        let suffix = allToEnd  ? "" : "$";
        let prefix = fromFirst ? "^" : "";
        let querystring = {"name": {"$regex": new RegExp(prefix + word + suffix, "i")}};
        let entries = new Parser().parse(source).getMethodsTable().find(querystring);
        return entries;
    }

    getRefsLocal(filename: string, source: string) {
        let parsesModule = new Parser().parse(source);
        let entries = parsesModule.getMethodsTable().find();
        let count = 0;
        let collection = this.cache.addCollection(filename);
        this.updateReferenceCalls(collection, parsesModule.context.CallsPosition, "GlobalModuleText", filename);
        for (let y = 0; y < entries.length; ++y) {
            let item = entries[y];
            this.updateReferenceCalls(collection, item._method.CallsPosition, item, filename);
        }
        return collection;
    }

    getReplaceMetadata () {
        return {
            "AccountingRegisters": "РегистрыБухгалтерии",
            "AccumulationRegisters": "РегистрыНакопления",
            "BusinessProcesses": "БизнесПроцессы",
            "CalculationRegisters": "РегистрыРасчета",
            "ChartsOfAccounts": "ПланыСчетов",
            "ChartsOfCalculationTypes": "ПланыВидовРасчета",
            "ChartsOfCharacteristicTypes": "ПланыВидовХарактеристик",
            "Constants": "Константы",
            "Catalogs": "Справочники",
            "DataProcessors": "Обработки",
            "DocumentJournals": "ЖурналыДокументов",
            "Documents": "Документы",
            "Enums": "Перечисления",
            "ExchangePlans": "ПланыОбмена",
            "InformationRegisters": "РегистрыСведений",
            "Reports": "Отчеты",
            "SettingsStorages": "ХранилищаНастроек",
            "Tasks": "Задачи"
        };
    }

    getModuleForPath(fullpath: string, rootPath: string): any {
        let splitsymbol = "/";
        let moduleArray: Array<string> = fullpath.substr(rootPath.length + (rootPath.slice(-1) === "\\" ? 0 : 1)).split(splitsymbol);
        let moduleStr: string = "";
            let hierarchy = moduleArray.length;
            if (hierarchy > 3) {
                if (moduleArray[hierarchy - 4].startsWith("CommonModules")) {
                    moduleStr = moduleArray[hierarchy - 3];
                } else if (hierarchy > 3 && this.toreplaced[moduleArray[hierarchy - 4]] !== undefined) {
                    moduleStr = this.toreplaced[moduleArray[hierarchy - 4]] + "." + moduleArray[hierarchy - 3];
                }
            }

        return moduleStr;
    }

    addtocachefiles(files: Array<vscode.Uri>, isbsl: boolean = false, rootPath: any = null): any {
        if (!rootPath) {
            rootPath = vscode.workspace.rootPath;
        }
        let filesLength = files.length;
        for (let i = 0; i < filesLength; ++i) {
            let fullpath = files[i].toString();
            fullpath = decodeURIComponent(fullpath);
            if (fullpath.startsWith("file:")) {
                if (process.platform === "win32") {
                    fullpath = fullpath.substr(8);
                } else {
                    fullpath = fullpath.substr(7);
                }
            }
            fq.readFile(fullpath, {encoding: "utf8"}, (err, source) => {
                if (err) {
                    throw err;
                }
                let moduleStr = "";
                if (fullpath.endsWith(".bsl")) {
                    moduleStr = this.getModuleForPath(fullpath, rootPath);
                }
                source = source.replace(/\r/g, "\r\n");
                let parsesModule = new Parser().parse(source);
                let entries = parsesModule.getMethodsTable().find();
                if (i % 100) {
                    vscode.window.setStatusBarMessage("Обновляем кэш файла № " + i + " из " + filesLength, 2000);
                }
                this.updateReferenceCalls(this.dbcalls, parsesModule.context.CallsPosition, "GlobalModuleText", fullpath);
                for (let y = 0; y < entries.length; ++y) {
                    let item = entries[y];
                    this.updateReferenceCalls(this.dbcalls, item._method.CallsPosition, item, fullpath);
                    item["filename"] = fullpath;
                    let newItem: MethodValue = {
                        "name": String(item.name),
                        "isproc": Boolean(item.isproc),
                        "line": item.line,
                        "endline": item.endline,
                        "context": item.context,
                        "_method": item._method,
                        "filename": fullpath,
                        "module": moduleStr,
                        "description": item.description
                    };
                    this.db.insert(newItem);
                }
                if (i === filesLength - 1) {
                    vscode.window.setStatusBarMessage("Обновление кэша завершено", 3000);
                    this.cacheUpdates = true;
                }
            });
        }
    }

    updateCache(): any {
        console.log("update cache");
        vscode.window.setStatusBarMessage("Запущено заполнение кеша", 3000);
        let configuration = vscode.workspace.getConfiguration("language-1c-bsl");
        let basePath: string = String(configuration.get("rootPath"));
        let rootPath = vscode.workspace.rootPath;
        if (rootPath) {
            rootPath = path.join(vscode.workspace.rootPath, basePath);
            this.db = this.cache.addCollection("ValueTable");
            this.dbcalls = this.cache.addCollection("Calls");

            let self = this;
            let files = vscode.workspace.findFiles(basePath !== "" ? basePath.substr(2) + "/**" : "" + "**/*.bsl", "");
            files.then((value) => {
                this.addtocachefiles(value, false, rootPath);
            }, (reason) => {
                console.log(reason);
            });
            files = vscode.workspace.findFiles(basePath !== "" ? basePath.substr(2) + "/**" : "" + "**/*.os", "");
            files.then((value) => {
                this.addtocachefiles(value, true, rootPath);
            }, (reason) => {
                console.log(reason);
            });
        }
    };

    customUpdateCache(source: string, filename: string) {
        if (!this.cacheUpdates) {
            return;
        }
        let configuration = vscode.workspace.getConfiguration("language-1c-bsl");
        let basePath: string = String(configuration.get("rootPath"));
        let rootPath = path.join(vscode.workspace.rootPath, basePath);
{'fraction_non_alphanumeric': 0.12941961307538358, 'fraction_numerical': 0.006204136090727151, 'mean_word_length': 1.9097437888198758, 'pattern_counts': {'":': 58, '<': 13, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6835168', 'n_tokens_mistral': 4447, 'n_tokens_neox': 4302, 'n_words': 842}