<commit_before>if command -v less &>/dev/null; then
  [ -e ${HOME}/bin/lessfilter.sh ] && export LESSOPEN='|~/bin/lessfilter.sh %s'
  alias less='less -RXcqeF'
  alias more='less -RXcqeNF'
fi

escape() { echo "${@}" | sed 's/[]\.|$(){}?+*^]/\\&/g'; }

domainSort() {
  # Take a list of URLS and sort it into a list of unique top-level domains
  local domain tmp opt helpstring list thirdLvlSubs
  local count=false
  local noSubs=false
  helpstring="Takes a list of URLs and sorts it into a list of unique top-level domains.\n \nOptions:\n\t-c:\t Add a count of the occurrences of each unique domain.\n \t-s:\t Remove subdomains"

  local OPTIND=1
  while getopts "hcs" opt; do
    case $opt in
      c) count=true ;;
      s) noSubs=true ;;
      h)
        echo -e "$helpstring"
        return
        ;;
      *) return 1 ;;
    esac
  done
  shift $((OPTIND - 1))

  list="$1"<commit_msg>no longer clear screen<commit_after>if command -v less &>/dev/null; then
  [ -e ${HOME}/bin/lessfilter.sh ] && export LESSOPEN='|~/bin/lessfilter.sh %s'
  alias less='less -RXqeF'
  alias more='less -RXqeNF'
fi

escape() { echo "${@}" | sed 's/[]\.|$(){}?+*^]/\\&/g'; }

domainSort() {
  # Take a list of URLS and sort it into a list of unique top-level domains
  local domain tmp opt helpstring list thirdLvlSubs
  local count=false
  local noSubs=false
  helpstring="Takes a list of URLs and sorts it into a list of unique top-level domains.\n \nOptions:\n\t-c:\t Add a count of the occurrences of each unique domain.\n \t-s:\t Remove subdomains"

  local OPTIND=1
  while getopts "hcs" opt; do
    case $opt in
      c) count=true ;;
      s) noSubs=true ;;
      h)
        echo -e "$helpstring"
        return
        ;;
      *) return 1 ;;
    esac
  done
  shift $((OPTIND - 1))

  list="$1"
{'fraction_non_alphanumeric': 0.1474036850921273, 'fraction_numerical': 0.00446677833612507, 'mean_word_length': 2.7024793388429753, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1507423', 'n_tokens_mistral': 658, 'n_tokens_neox': 627, 'n_words': 212}