<commit_before>alias hls='hadoop fs -ls '
alias hdu='hadoop fs -du -h '

# https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html
alias hquota='hadoop fs -count -q -h -v '
<commit_msg>Add hdir prefix for Hadoop folder loop and run a particular command line ex : hdircmd $HDFS_FOLDER "hdfs dfs -ls "<commit_after>alias hls='hadoop fs -ls '
alias hdu='hadoop fs -du -h '

# https://hadoop.apache.org/docs/r2.7.2/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html
alias hquota='hadoop fs -count -q -h -v '

hdirsize() {
  # MIN NUM OF ARG
  if [[ "$#" < "1" ]]; then
      echo "Usage : hdirsize HDFS_FOLDER [HDFS_MORE_ARG]. Run 'hdfs dfs -count -q' on each subfolders of HDFS_FOLDER" >&2
      echo " HDFS_MORE_ARG : -h = human readable, -v = verbose" >&2
      return -1
  fi

  local HDFS_FOLDER=$1
  local HDFS_MORE_ARG=${@:2}

  hdircmd $HDFS_FOLDER "hdfs dfs -count -q $HDFS_MORE_ARG"
}
hdirls() {
  # MIN NUM OF ARG
  if [[ "$#" < "1" ]]; then
      echo "Usage : hdirls HDFS_FOLDER [HDFS_MORE_ARG]. Run 'hdfs dfs -ls' on each subfolders of HDFS_FOLDER" >&2
      return -1
  fi

  local HDFS_FOLDER=$1
  local HDFS_MORE_ARG=${@:2}

  hdircmd $HDFS_FOLDER "hdfs dfs -ls $HDFS_MORE_ARG"
}

hdircmd() {
  # MIN NUM OF ARG
  if [[ "$#" < "2" ]]; then
      echo "Usage : hdircmd HDFS_FOLDER HDFS_CMD.. . Run 'HDFS_CMD..' on each subfolders of HDFS_FOLDER" >&2
      return -1
  fi

  local HDFS_FOLDER=$1
  local HDFS_CMD=${@:2}

  echo "hdfs dfs -ls $HDFS_FOLDER | awk -v COMMAND=\"$HDFS_CMD \" '{system(COMMAND \$8)}'"
  hdfs dfs -ls $HDFS_FOLDER | awk -v COMMAND="$HDFS_CMD " '{system(COMMAND $8)}'
}

{'fraction_non_alphanumeric': 0.14900060569351908, 'fraction_numerical': 0.014536644457904301, 'mean_word_length': 3.290909090909091, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '878291', 'n_tokens_mistral': 746, 'n_tokens_neox': 696, 'n_words': 182}