<commit_before>
__thousands() {
  awk '{$1=sprintf("%'\''19.lf", $1); print $0}' "$@"
}

__sum() {
  awk '{sum+=$1} END {print sum}' "$@"
}

__count() {
  awk '{counts[$0]++} END {for(c in counts) printf "%6d %s\n", counts[c], c}' "$@" | sort -n
}

__uniq() {
  awk '!seen[$0]++'
}

__count-ext() {
  print -l **/*(:t:e) | __count
}

__cut() {
  cut -c "1-${COLUMNS}" "$@"
}
<commit_msg>Improve duplicate flows with  __dupes and __count -d #zsh<commit_after>
__thousands() {
  awk '{$1=sprintf("%'\''19.lf", $1); print $0}' "$@"
}

__sum() {
  awk '{sum+=$1} END {print sum}' "$@"
}

__count() {
  local min_count=1
  if [ "$1" = "-d" ]; then
    min_count=2
    shift
  fi
  awk -v min_count="$min_count" '
    { counts[$0]++ }
    END {
      for(c in counts) {
        if (counts[c] >= min_count) {
          printf "%6d %s\n", counts[c], c
        }
      }
    }' "$@" | sort -n
}

__dupes() {
  awk '++seen[$0] == 2'
}

__uniq() {
  awk '!seen[$0]++'
}

__count-ext() {
  print -l **/*(:t:e) | __count
}

__cut() {
  cut -c "1-${COLUMNS}" "$@"
}

{'fraction_non_alphanumeric': 0.2718631178707224, 'fraction_numerical': 0.02376425855513308, 'mean_word_length': 2.7607142857142857, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1041478', 'n_tokens_mistral': 493, 'n_tokens_neox': 479, 'n_words': 104}