<commit_before>{
  "pipeline": {
    "name": "reduce"
  },
  "transform": {
    "cmd": ["sh"],
    "stdin": [
        "find /pfs/map/ -name '*' | while read count; do cat $count | awk '{ sum+=$1} END {print sum}' >/tmp/count; mv /tmp/count /pfs/out/`basename $count`; echo `basename $count`; done"
    ]
  },
  "inputs": [
    {
      "repo": {
        "name": "map"
      },
      "glob": "/*"
    }
  ]
}
<commit_msg>Use / for glob pattern to minimize the number of datums and thus improve performance<commit_after>{
  "pipeline": {
    "name": "reduce"
  },
  "transform": {
    "cmd": ["sh"],
    "stdin": [
        "find /pfs/map/ -name '*' | while read count; do cat $count | awk '{ sum+=$1} END {print sum}' >/tmp/count; mv /tmp/count /pfs/out/`basename $count`; echo `basename $count`; done"
    ]
  },
  "inputs": [
    {
      "repo": {
        "name": "map"
      },
      "glob": "/"
    }
  ]
}

{'fraction_non_alphanumeric': 0.21806167400881057, 'fraction_numerical': 0.0022026431718061676, 'mean_word_length': 2.15625, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6877635', 'n_tokens_mistral': 349, 'n_tokens_neox': 348, 'n_words': 85}