<commit_before>'use strict';

var fs = require('fs');
var path = require('path');

module.exports = {
  loadPriority:  0,
  initialize: function(api, next){

    if(!api.utils){ api.utils = {}; }

    ////////////////////////////////////////////////////////////////////////////
    // count the number of elements in a hash
    api.utils.hashLength = function(obj){
      var size = 0;
      var key;
      for(key in obj){
        if(obj.hasOwnProperty(key)){ size++; }
      }
      return size;
    };

    ////////////////////////////////////////////////////////////////////////////
    // merge two hashes recursively
    api.utils.hashMerge = function(a, b, arg){
      var c = {};
      var i;
      var response;

      for(i in a){
        if(api.utils.isPlainObject(a[i]) && Object.keys(a[i]).length > 0){
          c[i] = api.utils.hashMerge(c[i], a[i], arg);
        }else{
          if(typeof a[i] === 'function'){
            response = a[i](arg);
            if(api.utils.isPlainObject(response)){
              c[i] = api.utils.hashMerge(c[i], response, arg);
            }else{
              c[i] = response;
            }
          }else{
            c[i] = a[i];
          }
        }
      }
      for(i in b){
        if(api.utils.isPlainObject(b[i]) && Object.keys(b[i]).length > 0){
          c[i] = api.utils.hashMerge(c[i], b[i], arg);
        }else{
          if(typeof b[i] === 'function'){
            response = b[i](arg);
            if(api.utils.isPlainObject(response)){
              c[i] = api.utils.hashMerge(c[i], response, arg);
            }else{
              c[i] = response;
            }
          }else{
            c[i] = b[i];
          }
        }
      }
      return c;
    };

    api.utils.isPlainObject = function(o){
      var safeTypes     = [Boolean, Number, String, Function, Array, Date, RegExp, Buffer];
      var safeInstances = ['boolean', 'number', 'string', 'function'];
      var expandPreventMatchKey = '_toExpand'; // set `_toExpand = false` within an object if you don't want to expand it
      var i;

      if(!o){ return false; }
      if((o instanceof Object) === false){ return false; }
      for(i in safeTypes){
        if(o instanceof safeTypes[i]){ return false; }
      }
      for(i in safeInstances){
        if(typeof o === safeInstances[i]){ return false; }
      }
      if(o[expandPreventMatchKey] === false){ return false; }
      return (o.toString() === '[object Object]');
    };

    ////////////////////////////////////////////////////////////////////////////
    // string to hash
    // http://stackoverflow.com/questions/6393943/convert-javascript-string-in-dot-notation-into-an-object-reference
    api.utils.stringToHash = function(path, object){
      if(!object){ object = api; }
      function _index(obj, i){ return obj[i]; }
      return path.split('.').reduce(_index, object);
    };

    ////////////////////////////////////////////////////////////////////////////
    // unique-ify an array
    api.utils.arrayUniqueify = function(arr){
      var a = [];
      for(var i = 0; i < arr.length; i++){
        for(var j = i + 1; j < arr.length; j++){
          if(arr[i] === arr[j]){ j = ++i; }
        }
        a.push(arr[i]);
      }
      return a;
    };

    ////////////////////////////////////////////////////////////////////////////
    // get all .js files in a directory
    api.utils.recursiveDirectoryGlob = function(dir, extension, followLinkFiles){
      var results = [];

      if(!extension){ extension = '.js'; }
      if(!followLinkFiles){ followLinkFiles = true; }

      extension = extension.replace('.', '');
      if(dir[dir.length - 1] !== path.sep){ dir += path.sep; }

      if(fs.existsSync(dir)){
        fs.readdirSync(dir).forEach(function(file){
          var fullFilePath = path.normalize(dir + file);
          if(file[0] !== '.'){ // ignore 'system' files
            var stats = fs.statSync(fullFilePath);
            var child;
            if(stats.isDirectory()){
              child = api.utils.recursiveDirectoryGlob(fullFilePath, extension, followLinkFiles);
              child.forEach(function(c){ results.push(c); });
            }else if(stats.isSymbolicLink()){
              var realPath = fs.readlinkSync(fullFilePath);
              child = api.utils.recursiveDirectoryGlob(realPath, extension, followLinkFiles);
              child.forEach(function(c){ results.push(c); });
            }else if(stats.isFile()){
              var fileParts = file.split('.');
              var ext = fileParts[(fileParts.length - 1)];
              // real file match
              if(ext === extension){ results.push(fullFilePath); }
              // linkfile traversal
              if(ext === 'link' && followLinkFiles === true){
                var linkedPath = api.utils.sourceRelativeLinkPath(fullFilePath, api.config.general.paths.plugin);
                if(linkedPath){
                  child = api.utils.recursiveDirectoryGlob(linkedPath, extension, followLinkFiles);
                  child.forEach(function(c){ results.push(c); });
                }else{
                  api.log(['cannot find linked refrence to `%s`', file], 'warning');
                }
              }
            }
          }
        });
      }

      return results.sort();
    };

    api.utils.sourceRelativeLinkPath = function(linkfile, pluginPaths){
      var type = fs.readFileSync(linkfile).toString();
      var pathParts = linkfile.split(path.sep);
      var name = pathParts[(pathParts.length - 1)].split('.')[0];
      var pathsToTry = pluginPaths.slice(0);
      var pluginRoot;

      // TODO: always also try the local destination's `node_modules` to allow for nested plugins
      // This might be a security risk without requiring explicit sourcing

      pathsToTry.forEach(function(pluginPath){
        var pluginPathAttempt = path.normalize(pluginPath + path.sep + name);
        try{
          var stats = fs.lstatSync(pluginPathAttempt);
          if(!pluginRoot && stats.isDirectory()){ pluginRoot = pluginPathAttempt; }
        }catch(e){ }<commit_msg>fix crash due to missing logger in linked config setup<commit_after>'use strict';

var fs = require('fs');
var path = require('path');

module.exports = {
  loadPriority:  0,
  initialize: function(api, next){

    if(!api.utils){ api.utils = {}; }

    ////////////////////////////////////////////////////////////////////////////
    // count the number of elements in a hash
    api.utils.hashLength = function(obj){
      var size = 0;
      var key;
      for(key in obj){
        if(obj.hasOwnProperty(key)){ size++; }
      }
      return size;
    };

    ////////////////////////////////////////////////////////////////////////////
    // merge two hashes recursively
    api.utils.hashMerge = function(a, b, arg){
      var c = {};
      var i;
      var response;

      for(i in a){
        if(api.utils.isPlainObject(a[i]) && Object.keys(a[i]).length > 0){
          c[i] = api.utils.hashMerge(c[i], a[i], arg);
        }else{
          if(typeof a[i] === 'function'){
            response = a[i](arg);
            if(api.utils.isPlainObject(response)){
              c[i] = api.utils.hashMerge(c[i], response, arg);
            }else{
              c[i] = response;
            }
          }else{
            c[i] = a[i];
          }
        }
      }
      for(i in b){
        if(api.utils.isPlainObject(b[i]) && Object.keys(b[i]).length > 0){
          c[i] = api.utils.hashMerge(c[i], b[i], arg);
        }else{
          if(typeof b[i] === 'function'){
            response = b[i](arg);
            if(api.utils.isPlainObject(response)){
              c[i] = api.utils.hashMerge(c[i], response, arg);
            }else{
              c[i] = response;
            }
          }else{
            c[i] = b[i];
          }
        }
      }
      return c;
    };

    api.utils.isPlainObject = function(o){
      var safeTypes     = [Boolean, Number, String, Function, Array, Date, RegExp, Buffer];
      var safeInstances = ['boolean', 'number', 'string', 'function'];
      var expandPreventMatchKey = '_toExpand'; // set `_toExpand = false` within an object if you don't want to expand it
      var i;

      if(!o){ return false; }
      if((o instanceof Object) === false){ return false; }
      for(i in safeTypes){
        if(o instanceof safeTypes[i]){ return false; }
      }
      for(i in safeInstances){
        if(typeof o === safeInstances[i]){ return false; }
      }
      if(o[expandPreventMatchKey] === false){ return false; }
      return (o.toString() === '[object Object]');
    };

    ////////////////////////////////////////////////////////////////////////////
    // string to hash
    // http://stackoverflow.com/questions/6393943/convert-javascript-string-in-dot-notation-into-an-object-reference
    api.utils.stringToHash = function(path, object){
      if(!object){ object = api; }
      function _index(obj, i){ return obj[i]; }
      return path.split('.').reduce(_index, object);
    };

    ////////////////////////////////////////////////////////////////////////////
    // unique-ify an array
    api.utils.arrayUniqueify = function(arr){
      var a = [];
      for(var i = 0; i < arr.length; i++){
        for(var j = i + 1; j < arr.length; j++){
          if(arr[i] === arr[j]){ j = ++i; }
        }
        a.push(arr[i]);
      }
      return a;
    };

    ////////////////////////////////////////////////////////////////////////////
    // get all .js files in a directory
    api.utils.recursiveDirectoryGlob = function(dir, extension, followLinkFiles){
      var results = [];

      if(!extension){ extension = '.js'; }
      if(!followLinkFiles){ followLinkFiles = true; }

      extension = extension.replace('.', '');
      if(dir[dir.length - 1] !== path.sep){ dir += path.sep; }

      if(fs.existsSync(dir)){
        fs.readdirSync(dir).forEach(function(file){
          var fullFilePath = path.normalize(dir + file);
          if(file[0] !== '.'){ // ignore 'system' files
            var stats = fs.statSync(fullFilePath);
            var child;
            if(stats.isDirectory()){
              child = api.utils.recursiveDirectoryGlob(fullFilePath, extension, followLinkFiles);
              child.forEach(function(c){ results.push(c); });
            }else if(stats.isSymbolicLink()){
              var realPath = fs.readlinkSync(fullFilePath);
              child = api.utils.recursiveDirectoryGlob(realPath, extension, followLinkFiles);
              child.forEach(function(c){ results.push(c); });
            }else if(stats.isFile()){
              var fileParts = file.split('.');
              var ext = fileParts[(fileParts.length - 1)];
              // real file match
              if(ext === extension){ results.push(fullFilePath); }
              // linkfile traversal
              if(ext === 'link' && followLinkFiles === true){
                var linkedPath = api.utils.sourceRelativeLinkPath(fullFilePath, api.config.general.paths.plugin);
                if(linkedPath){
                  child = api.utils.recursiveDirectoryGlob(linkedPath, extension, followLinkFiles);
                  child.forEach(function(c){ results.push(c); });
                }else{
                  try{
                    api.log(['cannot find linked refrence to `%s`', file], 'warning');
                  }catch(e){
                    throw('cannot find linked refrence to' + file);
                  }
                }
              }
            }
          }
        });
      }

      return results.sort();
    };

    api.utils.sourceRelativeLinkPath = function(linkfile, pluginPaths){
      var type = fs.readFileSync(linkfile).toString();
      var pathParts = linkfile.split(path.sep);
      var name = pathParts[(pathParts.length - 1)].split('.')[0];
      var pathsToTry = pluginPaths.slice(0);
      var pluginRoot;

      // TODO: always also try the local destination's `node_modules` to allow for nested plugins
      // This might be a security risk without requiring explicit sourcing

      pathsToTry.forEach(function(pluginPath){
        var pluginPathAttempt = path.normalize(pluginPath + path.sep + name);
        try{
          var stats = fs.lstatSync(pluginPathAttempt);
          if(!pluginRoot && stats.isDirectory()){ pluginRoot = pluginPathAttempt; }
        }catch(e){ }
{'fraction_non_alphanumeric': 0.2000161694558978, 'fraction_numerical': 0.0030721966205837174, 'mean_word_length': 1.6636520241171404, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733188', 'n_tokens_mistral': 3519, 'n_tokens_neox': 3419, 'n_words': 777}