<commit_before>const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
const Mainloop = imports.mainloop;
const Utils = imports.utils;

/* Module listing */

let getModuleAtPath = function(path) {
    if (!GLib.file_test(path, GLib.FileTest.IS_DIR))
        return null;

    let manifestPath = path + '/component.json';
    if (!GLib.file_test(manifestPath, GLib.FileTest.IS_REGULAR))
        return null;

    let file = Gio.File.new_for_path(manifestPath);
    let [, manifestContent] = file.load_contents(null);
    let manifest;

    try {
        manifest = JSON.parse('' + manifestContent);
        manifest.path = path;
    } catch (e) {
        return null;
    }

    /* Quick sanity check */
    if (!manifest.name ||
        !manifest.noflo)
        return null;

    return manifest;
};

let getModulesInPath = function(path) {
    let modules = {};

    if (!GLib.file_test(path, GLib.FileTest.IS_DIR))
        return modules;

    Utils.forEachInDirectory(Gio.File.new_for_path(path), function(child) {
        let module = getModuleAtPath(child.get_path());

        if (module)
            modules[module.name] = module;
    });

    return modules;
};

let getModules = function(paths) {
    let modules = {};
    for (let i in paths) {
        Utils.mergeProps(modules, getModulesInPath(paths[i]));
    }
    return modules;
};

/* Component loader to be injected into the runtime base */

let ComponentLoader = function(options) {
    let self = this;

    /* Ensure with have valid options */
    self.options = options;
    if (!self.options)
        self.options = { paths: [ '.' ] };
    if (!self.options.path)
        self.options.path = ['.'];

    self.components = {};
    self.listCallbacks = [];

    let logFunc = function(name) {
        log('calling : ' +  name);
    };

    let normalizeName = function(name) {
        return name.replace('noflo-', '');
    };

    let removeExtension = function(path) {
        let ret = path.replace(/\.coffee$/, '');
        return ret.replace(/\.js$/, '');
    };

    let generateComponentInstance = function(path) {
        return function(metadata) {
            let implementation = require(path);
            let instance = implementation.getComponent(metadata);
            return instance;
        };
    };

    let generateComponentCodeLoader = function(path) {
        return function() {
            let file = Gio.File.new_for_path(path);
            let [, code] = file.load_contents(null);
            return '' + code;
        };
    };

    let getComponentFullPath = function(module, component) {
        return module.path + '/' + module.noflo.components[component];
    };

    let getComponentRequirePath = function(module, component) {
        return removeExtension(getComponentFullPath(module, component));
    };

    self.listComponents = function(callback) {
        // List is built already
        if (self.builtList) {
            callback(self.components);
            return;
        }

        self.listCallbacks.push(callback);

        // Building list
        if (self.buildingList)
            return;

        self.buildingList = true;

        Mainloop.timeout_add(0, Lang.bind(this, function() {
            let modules = getModules(self.options.paths);
            self.components = {};
            for (let i in modules) {
                let module = modules[i];
                for (let j in module.noflo.components) {
                    let path = normalizeName(module.name + '/' + j);
                    let requirePath = getComponentRequirePath(module, j);
                    let fullPath = getComponentFullPath(module, j);
                    self.components[path] = {
                        module: module,
                        moduleName: normalizeName(module.name),
                        name: j,
                        create: generateComponentInstance(requirePath),
                        getCode: generateComponentCodeLoader(fullPath),
                        language: Utils.guessLanguageFromFilename(fullPath),
                        path: fullPath
                    };
                }

                if (module.noflo.loader) {
                    let path = module.path + '/' + module.noflo.loader;
                    path = removeExtension(path);
                    let loader = require(path);
                    loader(self);
                }
            }

            let callbacks = self.listCallbacks;
            self.listCallbacks = [];
            self.buildingList = false;
            self.builtList = true;
            for (let i in callbacks)
                callbacks[i](self.components);

            return false;
        }));
    };

    self.load = function(name, callback, delayed, metadata) {
        if (!self.components[name])
            throw new Error('Component ' + name + ' not available');

        Mainloop.timeout_add(0, Lang.bind(this, function() {<commit_msg>runtime: ComponentLoader: make loading more readable<commit_after>const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
const Mainloop = imports.mainloop;
const Utils = imports.utils;

/* Module listing */

let getModuleAtPath = function(path) {
    if (!GLib.file_test(path, GLib.FileTest.IS_DIR))
        return null;

    let manifestPath = path + '/component.json';
    if (!GLib.file_test(manifestPath, GLib.FileTest.IS_REGULAR))
        return null;

    let file = Gio.File.new_for_path(manifestPath);
    let [, manifestContent] = file.load_contents(null);
    let manifest;

    try {
        manifest = JSON.parse('' + manifestContent);
        manifest.path = path;
    } catch (e) {
        return null;
    }

    /* Quick sanity check */
    if (!manifest.name ||
        !manifest.noflo)
        return null;

    return manifest;
};

let getModulesInPath = function(path) {
    let modules = {};

    if (!GLib.file_test(path, GLib.FileTest.IS_DIR))
        return modules;

    Utils.forEachInDirectory(Gio.File.new_for_path(path), function(child) {
        let module = getModuleAtPath(child.get_path());

        if (module)
            modules[module.name] = module;
    });

    return modules;
};

let getModules = function(paths) {
    let modules = {};
    for (let i in paths) {
        Utils.mergeProps(modules, getModulesInPath(paths[i]));
    }
    return modules;
};

/* Component loader to be injected into the runtime base */

let ComponentLoader = function(options) {
    let self = this;

    /* Ensure with have valid options */
    self.options = options;
    if (!self.options)
        self.options = { paths: [ '.' ] };
    if (!self.options.path)
        self.options.path = ['.'];

    self.components = {};
    self.listCallbacks = [];

    let logFunc = function(name) {
        log('calling : ' +  name);
    };

    let normalizeName = function(name) {
        return name.replace('noflo-', '');
    };

    let removeExtension = function(path) {
        let ret = path.replace(/\.coffee$/, '');
        return ret.replace(/\.js$/, '');
    };

    let generateComponentInstance = function(path) {
        return function(metadata) {
            let implementation = require(path);
            let instance = implementation.getComponent(metadata);
            return instance;
        };
    };

    let generateComponentCodeLoader = function(path) {
        return function() {
            let file = Gio.File.new_for_path(path);
            let [, code] = file.load_contents(null);
            return '' + code;
        };
    };

    let getComponentFullPath = function(module, component) {
        return module.path + '/' + module.noflo.components[component];
    };

    let getComponentRequirePath = function(module, component) {
        return removeExtension(getComponentFullPath(module, component));
    };

    self.listComponents = function(callback) {
        // List is built already
        if (self.builtList) {
            callback(self.components);
            return;
        }

        self.listCallbacks.push(callback);

        // Building list
        if (self.buildingList)
            return;

        self.buildingList = true;

        Mainloop.timeout_add(0, Lang.bind(this, function() {
            let modules = getModules(self.options.paths);
            self.components = {};
            for (let moduleName in modules) {
                let module = modules[moduleName];
                for (let componentName in module.noflo.components) {
                    let path = normalizeName(moduleName + '/' + componentName);
                    let requirePath = getComponentRequirePath(module, componentName);
                    let fullPath = getComponentFullPath(module, componentName);
                    self.components[path] = {
                        module: module,
                        moduleName: normalizeName(moduleName),
                        name: componentName,
                        create: generateComponentInstance(requirePath),
                        getCode: generateComponentCodeLoader(fullPath),
                        language: Utils.guessLanguageFromFilename(fullPath),
                        path: fullPath
                    };
                }

                if (module.noflo.loader) {
                    let path = module.path + '/' + module.noflo.loader;
                    path = removeExtension(path);
                    let loader = require(path);
                    loader(self);
                }
            }

            let callbacks = self.listCallbacks;
            self.listCallbacks = [];
            self.buildingList = false;
            self.builtList = true;
            for (let i in callbacks)
                callbacks[i](self.components);

            return false;
        }));
    };

    self.load = function(name, callback, delayed, metadata) {
        if (!self.components[name])
            throw new Error('Component ' + name + ' not available');

        Mainloop.timeout_add(0, Lang.bind(this, function() {
{'fraction_non_alphanumeric': 0.11452430659111244, 'fraction_numerical': 0.00039765384233025154, 'mean_word_length': 1.895797351755901, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1643700', 'n_tokens_mistral': 2779, 'n_tokens_neox': 2665, 'n_words': 595}