<commit_before>
getFiles = require './get-files'

module.exports = (options) ->

  {store} = options

  result = getFiles options

  # store the parsed objects
  objects = []

  # remember which files we've already read in case of overlap
  # Note: when working in your user directory it can find a file in there
  # because of both user scope and local scope.
  set = {}

  for file in result.files

    # let's see if we've already appended this file before...
    found = false       # mark whether we found it
    for each of set     # iterate over keys in `set`, which are files
      if each is file   # if they match then...
        found = true    # we found it
        break           # so we're all done looping

    # only append it if we didn't find it already in there
    if not found

      # remember we already appended this file
      set[file] = true

      # use the value store to read the file
      result = store.append file

      # let's pop it out of there
      object = store.array.pop()

      # add the object to our results
      objects.push object

  return objects:objects

# everything except 'cli' and 'env' are handled by files
module.exports.scopes = [ 'global', 'system', 'user', 'local', 'each', 'all' ]<commit_msg>use set as an object and get true/false value, only include object when it has some keys<commit_after>getFiles = require './get-files'

module.exports = (options) ->

  {store} = options

  result = getFiles options

  # store the parsed objects
  objects = []

  # remember which files we've already read in case of overlap
  # Note: when working in your user directory it can find a file in there
  # because of both user scope and local scope.
  set = {}

  for file in result.files

    # only append it if we didn't find it already in there
    if set[file] isnt true

      # remember we already appended this file
      set[file] = true

      # use the value store to read the file
      result = store.append file

      # let's pop it out of there
      object = store.array.pop()

      # add the object to our results if it has some keys
      if Object.keys(object).length > 0 then objects.push object

  return {objects}

# everything except 'cli' and 'env' are handled by files
module.exports.scopes = [ 'global', 'system', 'user', 'local', 'each', 'all' ]
{'fraction_non_alphanumeric': 0.08079071766222604, 'fraction_numerical': 0.0004297378599054577, 'mean_word_length': 2.6205287713841368, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6500167', 'n_tokens_mistral': 661, 'n_tokens_neox': 623, 'n_words': 312}