<commit_before>'use strict';

var path = require('path');
var existsSync = require('../../utils/file_utils').existsSync;
var fs = require('fs');
var path = require('path');

/*
 * This module defines the job execution context on the worker nodes.
 */

module.exports = function(context) {
    var opPath = '';
    var config = context.sysconfig.teraslice;
    if (config.ops_directory) opPath = config.ops_directory;

    var types = ['readers', 'processors'];

    function walk(rootDir, callback) {
        fs.readdirSync(rootDir).forEach(function(filename) {
            var filePath = path.join(rootDir, filename);
            if (fs.statSync(filePath).isDirectory()) {
                walk(filePath, callback);
            } else {
                callback(filePath, rootDir, filename);
            }
        });
    }


    function findOp(name, assetsPath, jobAssets) {
        var filePath;
        var codeName;

        if (!name.match(/.js/)) {
            codeName = `${name}.js`;
        }

        function filterFile(filepath, rootdir, filename) {
            if (filename === name || filename === codeName) {
                filePath = filepath;
            }
        }

        types.forEach(function(type) {
            var pathType = path.resolve(__dirname, '../../' + type);
            walk(pathType, filterFile)
        });

        if (opPath && existsSync(opPath)) {
            types.forEach(function(type) {
                var pathType = path.resolve(opPath + '/' + type);
                if (existsSync(pathType)) {
                    walk(pathType, filterFile)
                }
            });
        }

        if (assetsPath && existsSync(assetsPath)) {
            jobAssets.forEach(function(assetID) {
                var assetOpPath = `${assetsPath}/${assetID}`;
                if (existsSync(assetOpPath)) {
                    walk(assetOpPath, filterFile)<commit_msg>fixed asset loading bug resolves #479 (#480)<commit_after>'use strict';

var path = require('path');
var existsSync = require('../../utils/file_utils').existsSync;
var fs = require('fs');

/*
 * This module defines the job execution context on the worker nodes.
 */

module.exports = function(context) {
    var opPath = '';
    var config = context.sysconfig.teraslice;
    if (config.ops_directory) opPath = config.ops_directory;

    var types = ['readers', 'processors'];


    function findOp(name, assetsPath, jobAssets) {
        var filePath;
        var codeName;

        if (!name.match(/.js/)) {
            codeName = `${name}.js`;
        }

        function findCode(rootDir) {
            fs.readdirSync(rootDir).forEach(function(filename) {
                var nextPath = path.join(rootDir, filename);

                //if name is same as filename/dir then we found it
                if (filename === name || filename === codeName) {
                    filePath = nextPath;
                }

                if (!filePath && filename !== 'node_modules' && fs.statSync(nextPath).isDirectory()) {
                    findCode(nextPath);
                }
            });
        }

        types.forEach(function(type) {
            var pathType = path.resolve(`${__dirname}/../../${type}`);
            if (!filePath) findCode(pathType)
        });

        // if found, don't do extra searches
        if (filePath) return filePath;

        if (assetsPath && existsSync(assetsPath)) {
            jobAssets.forEach(function(assetID) {
                var assetOpPath = `${assetsPath}/${assetID}`;
                //if the path is not found yet and the opPath exists
                if (!filePath && existsSync(assetOpPath)) {
                    findCode(assetOpPath);
                }
            });
        }

        // if found, don't do extra searches
        if (filePath) return filePath;

        if (opPath && existsSync(opPath)) {
            types.forEach(function(type) {
                var pathType = path.resolve(opPath + '/' + type);
                if (!filePath && existsSync(pathType)) {
                    findCode(pathType)
{'fraction_non_alphanumeric': 0.12999754721609028, 'fraction_numerical': 0.0014716703458425313, 'mean_word_length': 1.83785664578984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5799198', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1085, 'n_words': 254}