<commit_before># ========================================
# Autogen Options - manfile/GNUdoc scraper
# ========================================

function manopts() {
  emulate -LR zsh
  findopts() {
    if [ "$1" = -a ]; then
      shift; EXPR="$@"
    fi
    perl -n -e ' while ( /'"$EXPR"'/g ) { print("$1\n"); } '
  }
  'man' "$@" 2> /dev/null | col -bx | findopts -a "[    ]((-|--)[A-Za-z0-9-=]+)"
}

function automan() {
  emulate -LR zsh
  local cmds comp a
  cmds=$(comm -12 <(whatis -r . | cut -f1 -d " ") <(hash | cut -f1 -d "="))
  comp=$(typeset -f | \grep -P "^_" | cut -f1 -d " " | cut -c2-)
  for i in $(comm -13 <(echo $comp) <(echo $cmds)); do
    _autogen_$i() {
      local -a args
      if [[ $words[-1] = (*-*) ]]; then
        name=${0:9}
        manpage=$(man $name | col -bx)
        for arg in $(manopts $name | sort -u); do
          a=$arg
          if [[ $arg == (*=*) ]]; then
            a=${${(@s/=/)arg}[1]}
          fi
          arg=$(echo $manpage | grep -P -- "[^a-zA-Z]${a}[^a-zA-Z]" | tr -s " \n\t" " ")
          [[ -n $a ]] && args[$((${#args}+1))]="()${a}[(m)${(q)arg}]"
        done
        _arguments -s $args
      fi
      _files
    }
    compdef _autogen_$i $i
  done
}

function mangen() {
  emulate -LR zsh
  echo -e "$1" >> ~/.zsh.d/mangen
  sort ~/.zsh.d/mangen -o ~/.zsh.d/mangen
}<commit_msg>Refactored manpage scraper<commit_after># ========================================
# Autogen Options - manfile/GNUdoc scraper
# ========================================

function manopts() {
  emulate -LR zsh
  findopts() {
    if [ "$1" = -a ]; then
      shift; EXPR="$@"
    fi
    perl -n -e ' while ( /'"$EXPR"'/g ) { print("$1\n"); } '
  }
  'man' "$@" 2> /dev/null | col -bx | findopts -a "[    ]((-|--)[A-Za-z0-9-=]+)"
}

function automan() {
  zparseopts -D -E v=V
  emulate -LR zsh
  local -a cmds comp man_names command_names autogens

  # the names of man pages
  # the names of commands in path
  # the names of completion functions
  man_names=("${(fo)$(whatis -r . |cut -f1 -d' ')}")
  command_names=("${(kos/ /)commands}")
  comp=${${(M)${(koni)functions}:#_*}/_}

  
  cmds=${command_names:*man_names}
  cmds=("${(s/ /)cmds}")
  comp=("${(s/ /)comp}")
  autogens=${cmds:|comp}
  if [[ -n $V ]]; then
    print -l ${(s/ /)autogens}
  fi
  for i in ${(s/ /)autogens}; do
    _autogen_$i() {
      emulate -LR zsh
      setopt extended_glob
      local -a args a
      if [[ $words[-1] = (*-*) ]]; then
        name=${0:9}
        manpage=$(man $name | col -bx)
        for arg in $(manopts $name | sort -u); do
          a=$arg
          if [[ $arg == (*=*) ]]; then
            a=${${(@s/=/)arg}[1]}
          fi
          arg=$(echo $manpage | grep -P -- "[^a-zA-Z]${a}[^a-zA-Z]")
          arg=${arg//[[:space:]]##/ }
          if [[ -n $a ]]; then
            args+="()${a}[(m)${(q)arg}]"
          fi
        done
        _arguments -s $args
      fi
      _files
    }
    compdef _autogen_$i $i
  done
}

function mangen() {
  emulate -LR zsh
  echo -e "$1" >> ~/.zsh.d/mangen
  sort ~/.zsh.d/mangen -o ~/.zsh.d/mangen
}
{'fraction_non_alphanumeric': 0.2754374594944913, 'fraction_numerical': 0.009721322099805573, 'mean_word_length': 2.087, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3177583', 'n_tokens_mistral': 1270, 'n_tokens_neox': 1227, 'n_words': 293}