<commit_before>'use strict'

const grpc = require('grpc')
const path = require('path')
const protobufjs = require('protobufjs')
const recursive = require('recursive-readdir')
const protosdirs = ['protos'].map(
  protopath => path.normalize(path.join(__dirname, '..', '..', '..', protopath))
)
let allprotos = {}

exports.load = filename => {
  let root = new protobufjs.Root()
  root.resolvePath = (origin, target) => {
    let foundValue = ''
    Object.keys(allprotos).forEach(base => {
      allprotos[base].forEach(filename => {
        if (filename.endsWith(target)) foundValue = filename
      })
    })
    return foundValue
  }

  return Promise.all(protosdirs.map(path => recursive(path)))
  .then(results => {
    protosdirs.map((key, index) => {
      allprotos[key] = results[index]
    })
    return protobufjs.load(filename, root)
  })
  .then(proto => {
    return grpc.loadObject(proto)
  })
}<commit_msg>Better proto loader.<commit_after>'use strict'

const grpc = require('grpc')
const path = require('path')
const protobufjs = require('protobufjs')
const recursive = require('recursive-readdir')

exports.load = (filenames, paths = ['protos']) => {
  let allprotos = {}
  let singleResult = false
  if (typeof filenames === 'string') {
    singleResult = true
    filenames = [filenames]
  }

  const protosdirs = paths.map(
    protopath => path.normalize(path.join(__dirname, '..', '..', '..', protopath))
  )
  let root = new protobufjs.Root()
  root.resolvePath = (origin, target) => {
    let foundValue = ''
    Object.keys(allprotos).forEach(base => {
      allprotos[base].forEach(filename => {
        if (filename === target) foundValue = path.join(base, filename)
      })
    })
    return foundValue
  }

  return Promise.all(protosdirs.map(path => recursive(path)))
  .then(results => {
    protosdirs.map((key, index) => {
      allprotos[key] = results[index].map(fullpath => path.relative(key, fullpath))
    })
    return Promise.all(filenames.map(filename => protobufjs.load(filename, root)))
  })
  .then(protos => {
    const results = protos.map(proto => grpc.loadObject(proto))
    return singleResult ? results[0] : results
  })
}
{'fraction_non_alphanumeric': 0.16474919466175794, 'fraction_numerical': 0.0004601932811780948, 'mean_word_length': 3.625531914893617, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6407928', 'n_tokens_mistral': 720, 'n_tokens_neox': 708, 'n_words': 136}