<commit_before># -------------------------------------------------------
# SCM Breeze - Streamline your SCM workflow.
# Copyright 2011 <NAME> (http://madebynathan.com). All Rights Reserved.
# Released under the LGPL (GNU Lesser General Public License)
# -------------------------------------------------------

# -------------------------------------------------------
# Repository Index scripts for Git projects
# -------------------------------------------------------


# * The `git_index` function makes it easy to list & switch between
#   git projects in $GIT_REPO_DIR (default = ~/src)
#
#   * Change directory to any of your git repos or submodules, with recursive tab completion.
#
#   * A repository index will be created at $GIT_REPO_DIR/.git_index
#     (Scanning for git projects and submodules can take a few seconds.)
#
#   * Cache can be rebuilt by running:
#       $ git_index --rebuild
#       ('--' commands have tab completion too.)
#
#   * Ignores projects within an 'archive' folder.
#
#   * Allows you to run batch commands across all your repositories:
#
#     - Update every repo from their remote: 'git_index --update-all'
#     - Produce a count of repos for each host: 'git_index --count-by-host'
#     - Run a custom command for each repo: 'git_index --batch-cmd <command>'
#
#   * You can also change to a top-level directory within $GIT_REPO_DIR by prefixing the argument
#     with '/'
#
# Examples:
#
#     $ git_index --list
#     # => Lists all git projects
#
#     $ git_index ub[TAB]
#     # => Provides tab completion for all project folders that begin with 'ub'
#
#     $ git_index ubuntu_config
#     # => Changes directory to ubuntu_config, and auto-updates code from git remote.
#
#     $ git_index buntu_conf
#     # => Same result as `git_index ubuntu_config`
#
#     $ git_index
#     # => cd $GIT_REPO_DIR


function git_index() {
  local IFS=$'\n'
  if [ -z "$1" ]; then
    # Just change to $GIT_REPO_DIR if no params given.
    "cd" $GIT_REPO_DIR
  else
    if [ "$1" = "--rebuild" ]; then
      _rebuild_git_index
    elif [ "$1" = "--update-all" ]; then
      _git_index_update_all
    elif [ "$1" = "--update-all-with-notifications" ]; then
      NOTIFY=true
      _git_index_update_all
    elif [ "$1" = "--batch-cmd" ]; then
      _git_index_batch_cmd "${@:2:$(($#-1))}" # Pass all args except $1
    elif [ "$1" = "--list" ] || [ "$1" = "-l" ]; then
      echo -e "$_bld_col$(_git_index_count)$_txt_col Git repositories in $_bld_col$GIT_REPO_DIR$_txt_col:\n"
      for repo in $(_git_index_dirs_without_home); do
        echo $(basename $repo | sed "s/ /_/g") : $repo
      done | sort -t ":" -k1,1 | column -t -s ':'
    elif [ "$1" = "--count-by-host" ]; then
      echo -e "=== Producing a report of the number of repos per host...\n"
      _git_index_batch_cmd git remote -v | \grep "origin.*(fetch)" |
      sed -e "s/origin\s*//" -e "s/(fetch)//" |
      sed -e "s/\(\([^/]*\/\/\)\?\([^@]*@\)\?\([^:/]*\)\).*/\1/" |
      sort | uniq -c
      echo

    # If $1 starts with '/', change to top-level directory within $GIT_REPO_DIR
    elif ([ $shell = "bash" ] && [ "${1:0:1}" = "/" ]) || \
         ([ $shell = "zsh" ]  && [ "${1[1]}"  = "/" ]); then
      if [ -d "$GIT_REPO_DIR$1" ]; then
        builtin cd "$GIT_REPO_DIR$1"
      fi
    else
      _check_git_index
      # Figure out which directory we need to change to.
      local project=$(echo $1 | cut -d "/" -f1)
      # Find base path of project
      local base_path="$(\grep -m1 "/$project$" "$GIT_REPO_DIR/.git_index")"
      if [ -n "$base_path" ]; then
        sub_path=$(echo $1 | sed "s:^$project::")
        # Append subdirectories to base path
        base_path="$base_path$sub_path"
      fi
      # Try partial matches
      # - string at beginning of project
      if [ -z "$base_path" ]; then base_path=$(_git_index_dirs_without_home | \grep -m1 -i "/$project"); fi
      # - string anywhere in project
      if [ -z "$base_path" ]; then base_path=$(_git_index_dirs_without_home | \grep -m1 -i "$project"); fi
      # --------------------
      # Go to our base path
      if [ -n "$base_path" ]; then
        local IFS=$' \t\n'
        # evaluate ~ if necessary
        if [[ "$base_path" == "~"* ]]; then
          base_path=$(eval echo ${base_path%%/*})/${base_path#*/}
        fi
        cd "$base_path"
        # Run git callback (either update or show changes), if we are in the root directory
        if [ -z "${sub_path%/}" ]; then _git_index_status_if_dirty; fi
      else
        echo -e "$_wrn_col'$1' did not match any git repos in $GIT_REPO_DIR$_txt_col"
      fi
    fi
  fi
}

_git_index_dirs_without_home() {
  sed -e "s/--.*//" -e "s%$HOME%~%" $GIT_REPO_DIR/.git_index
}

# Recursively searches for git repos in $GIT_REPO_DIR
function _find_git_repos() {
  # Find all unarchived projects
  local IFS=$'\n'
  for repo in $(find -L "$GIT_REPO_DIR" -maxdepth 3 -name ".git" -type d \! -wholename '*/archive/*'); do
    echo ${repo%/.git}          # Return project folder, with trailing ':'
    _find_git_submodules $repo  # Detect any submodules
  done
}

# List all submodules for a git repo, if any.
function _find_git_submodules() {
  if [ -e "$1/../.gitmodules" ]; then
    \grep "\[submodule" "$1/../.gitmodules" | sed "s%\[submodule \"%${1%/.git}/%g" | sed "s/\"]//g"
  fi
}


# Rebuilds index of git repos in $GIT_REPO_DIR.
function _rebuild_git_index() {
  if [ "$1" != "--silent" ]; then echo -e "== Scanning $GIT_REPO_DIR for git repos & submodules..."; fi
  # Get repos from src dir and custom dirs, then sort by basename
  local IFS=$'\n'
  for repo in $(echo -e "$(_find_git_repos)\n$(echo $GIT_REPOS | sed "s/:/\\\\n/g")"); do
    echo $(basename $repo | sed "s/ /_/g"):$repo
  done | sort -t ":" -k1,1 | cut -d ":" -f2- >| "$GIT_REPO_DIR/.git_index"

  if [ "$1" != "--silent" ]; then
    echo -e "===== Indexed $_bld_col$(_git_index_count)$_txt_col repos in $GIT_REPO_DIR/.git_index"
  fi
}

# Build index if empty
function _check_git_index() {
  if [ ! -f "$GIT_REPO_DIR/.git_index" ]; then
    _rebuild_git_index --silent
  fi
}

# Produces a count of repos in the tab completion index (excluding commands)
function _git_index_count() {
  echo $(sed -e "s/--.*//" "$GIT_REPO_DIR/.git_index" | \grep . | wc -l)
}

# Returns the current $GIT_BINARY branch (returns nothing if not a git repository)
function is_git_dirty {
    [[ $($GIT_BINARY status 2> /dev/null | tail -n1) != "nothing to commit (working directory clean)" ]] && echo "*"
 }
function parse_git_branch {
    $GIT_BINARY branch --no-color 2> /dev/null | sed -e '/^[^*]/d' -e "s/* \(.*\)/\1/"
}

# If the working directory is clean, update the git repository. Otherwise, show changes.
function _git_index_status_if_dirty() {
  if ! [ `git status --porcelain | wc -l` -eq 0 ]; then
    # Fall back to 'git status' if git status alias isn't configured
    if type $git_status_command 2>&1 | \grep -qv "not found"; then
      eval $git_status_command
    else
      git status
    fi
  fi
}

<commit_msg>Speed up and simplify parse_git_branch<commit_after># -------------------------------------------------------
# SCM Breeze - Streamline your SCM workflow.
# Copyright 2011 <NAME> (http://madebynathan.com). All Rights Reserved.
# Released under the LGPL (GNU Lesser General Public License)
# -------------------------------------------------------

# -------------------------------------------------------
# Repository Index scripts for Git projects
# -------------------------------------------------------


# * The `git_index` function makes it easy to list & switch between
#   git projects in $GIT_REPO_DIR (default = ~/src)
#
#   * Change directory to any of your git repos or submodules, with recursive tab completion.
#
#   * A repository index will be created at $GIT_REPO_DIR/.git_index
#     (Scanning for git projects and submodules can take a few seconds.)
#
#   * Cache can be rebuilt by running:
#       $ git_index --rebuild
#       ('--' commands have tab completion too.)
#
#   * Ignores projects within an 'archive' folder.
#
#   * Allows you to run batch commands across all your repositories:
#
#     - Update every repo from their remote: 'git_index --update-all'
#     - Produce a count of repos for each host: 'git_index --count-by-host'
#     - Run a custom command for each repo: 'git_index --batch-cmd <command>'
#
#   * You can also change to a top-level directory within $GIT_REPO_DIR by prefixing the argument
#     with '/'
#
# Examples:
#
#     $ git_index --list
#     # => Lists all git projects
#
#     $ git_index ub[TAB]
#     # => Provides tab completion for all project folders that begin with 'ub'
#
#     $ git_index ubuntu_config
#     # => Changes directory to ubuntu_config, and auto-updates code from git remote.
#
#     $ git_index buntu_conf
#     # => Same result as `git_index ubuntu_config`
#
#     $ git_index
#     # => cd $GIT_REPO_DIR


function git_index() {
  local IFS=$'\n'
  if [ -z "$1" ]; then
    # Just change to $GIT_REPO_DIR if no params given.
    "cd" $GIT_REPO_DIR
  else
    if [ "$1" = "--rebuild" ]; then
      _rebuild_git_index
    elif [ "$1" = "--update-all" ]; then
      _git_index_update_all
    elif [ "$1" = "--update-all-with-notifications" ]; then
      NOTIFY=true
      _git_index_update_all
    elif [ "$1" = "--batch-cmd" ]; then
      _git_index_batch_cmd "${@:2:$(($#-1))}" # Pass all args except $1
    elif [ "$1" = "--list" ] || [ "$1" = "-l" ]; then
      echo -e "$_bld_col$(_git_index_count)$_txt_col Git repositories in $_bld_col$GIT_REPO_DIR$_txt_col:\n"
      for repo in $(_git_index_dirs_without_home); do
        echo $(basename $repo | sed "s/ /_/g") : $repo
      done | sort -t ":" -k1,1 | column -t -s ':'
    elif [ "$1" = "--count-by-host" ]; then
      echo -e "=== Producing a report of the number of repos per host...\n"
      _git_index_batch_cmd git remote -v | \grep "origin.*(fetch)" |
      sed -e "s/origin\s*//" -e "s/(fetch)//" |
      sed -e "s/\(\([^/]*\/\/\)\?\([^@]*@\)\?\([^:/]*\)\).*/\1/" |
      sort | uniq -c
      echo

    # If $1 starts with '/', change to top-level directory within $GIT_REPO_DIR
    elif ([ $shell = "bash" ] && [ "${1:0:1}" = "/" ]) || \
         ([ $shell = "zsh" ]  && [ "${1[1]}"  = "/" ]); then
      if [ -d "$GIT_REPO_DIR$1" ]; then
        builtin cd "$GIT_REPO_DIR$1"
      fi
    else
      _check_git_index
      # Figure out which directory we need to change to.
      local project=$(echo $1 | cut -d "/" -f1)
      # Find base path of project
      local base_path="$(\grep -m1 "/$project$" "$GIT_REPO_DIR/.git_index")"
      if [ -n "$base_path" ]; then
        sub_path=$(echo $1 | sed "s:^$project::")
        # Append subdirectories to base path
        base_path="$base_path$sub_path"
      fi
      # Try partial matches
      # - string at beginning of project
      if [ -z "$base_path" ]; then base_path=$(_git_index_dirs_without_home | \grep -m1 -i "/$project"); fi
      # - string anywhere in project
      if [ -z "$base_path" ]; then base_path=$(_git_index_dirs_without_home | \grep -m1 -i "$project"); fi
      # --------------------
      # Go to our base path
      if [ -n "$base_path" ]; then
        local IFS=$' \t\n'
        # evaluate ~ if necessary
        if [[ "$base_path" == "~"* ]]; then
          base_path=$(eval echo ${base_path%%/*})/${base_path#*/}
        fi
        cd "$base_path"
        # Run git callback (either update or show changes), if we are in the root directory
        if [ -z "${sub_path%/}" ]; then _git_index_status_if_dirty; fi
      else
        echo -e "$_wrn_col'$1' did not match any git repos in $GIT_REPO_DIR$_txt_col"
      fi
    fi
  fi
}

_git_index_dirs_without_home() {
  sed -e "s/--.*//" -e "s%$HOME%~%" $GIT_REPO_DIR/.git_index
}

# Recursively searches for git repos in $GIT_REPO_DIR
function _find_git_repos() {
  # Find all unarchived projects
  local IFS=$'\n'
  for repo in $(find -L "$GIT_REPO_DIR" -maxdepth 3 -name ".git" -type d \! -wholename '*/archive/*'); do
    echo ${repo%/.git}          # Return project folder, with trailing ':'
    _find_git_submodules $repo  # Detect any submodules
  done
}

# List all submodules for a git repo, if any.
function _find_git_submodules() {
  if [ -e "$1/../.gitmodules" ]; then
    \grep "\[submodule" "$1/../.gitmodules" | sed "s%\[submodule \"%${1%/.git}/%g" | sed "s/\"]//g"
  fi
}


# Rebuilds index of git repos in $GIT_REPO_DIR.
function _rebuild_git_index() {
  if [ "$1" != "--silent" ]; then echo -e "== Scanning $GIT_REPO_DIR for git repos & submodules..."; fi
  # Get repos from src dir and custom dirs, then sort by basename
  local IFS=$'\n'
  for repo in $(echo -e "$(_find_git_repos)\n$(echo $GIT_REPOS | sed "s/:/\\\\n/g")"); do
    echo $(basename $repo | sed "s/ /_/g"):$repo
  done | sort -t ":" -k1,1 | cut -d ":" -f2- >| "$GIT_REPO_DIR/.git_index"

  if [ "$1" != "--silent" ]; then
    echo -e "===== Indexed $_bld_col$(_git_index_count)$_txt_col repos in $GIT_REPO_DIR/.git_index"
  fi
}

# Build index if empty
function _check_git_index() {
  if [ ! -f "$GIT_REPO_DIR/.git_index" ]; then
    _rebuild_git_index --silent
  fi
}

# Produces a count of repos in the tab completion index (excluding commands)
function _git_index_count() {
  echo $(sed -e "s/--.*//" "$GIT_REPO_DIR/.git_index" | \grep . | wc -l)
}

# Returns the current $GIT_BINARY branch (returns nothing if not a git repository)
function is_git_dirty {
    [[ $($GIT_BINARY status 2> /dev/null | tail -n1) != "nothing to commit (working directory clean)" ]] && echo "*"
 }
function parse_git_branch {
    git rev-parse --abbrev-ref HEAD 2> /dev/null
}

# If the working directory is clean, update the git repository. Otherwise, show changes.
function _git_index_status_if_dirty() {
  if ! [ `git status --porcelain | wc -l` -eq 0 ]; then
    # Fall back to 'git status' if git status alias isn't configured
    if type $git_status_command 2>&1 | \grep -qv "not found"; then
      eval $git_status_command
    else
      git status
    fi
  fi
}


{'fraction_non_alphanumeric': 0.18487933366555137, 'fraction_numerical': 0.006905389051042927, 'mean_word_length': 2.38587611472644, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6500168', 'n_tokens_mistral': 4944, 'n_tokens_neox': 4819, 'n_words': 1471}