<commit_before>whichbin() {
    local command_path

    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        command_path=$(\which $1 2> /dev/null)
    elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
        command_path=$(\which --skip-alias $1 2> /dev/null)
    else
        echo "${FUNCNAME[0]}: unknown distribution" 1>&2
        return 22
    fi

    result=$?
    if [ "$result" -ne 0 ]; then
        echo "no ${1} in (${PATH})" 1>&2
        return $result
    fi

    readlink -f $command_path
}

# Convert datetime to an epoch time.
#
# example: a datetime to an epoch time
#     $ date2epoch "2017-01-01 00:00:00+0900"
#     1483196400
#
# example: show current epoc time
#     $ date2epoch
#     1493254466
date2epoch() {
    if [ "$1" != "" ]; then
        \date +%s -d "$1"
    else
        \date +%s
    fi
}

# Convert an epoch time to a datetime
#
# example:
#     $ epoch2date 1483196400
#     2017-01-01 00:00:00+09:00
epoch2date() {
    if [ "$1" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} EPOCH_TIME" 1>&2
        return 22
    fi

    \date -d @"$1" --rfc-3339=seconds
}

# find files in a directory with exclude hidden files/directories
ff() {
    local path=$1
    local name_pattern=$2

    if [ "$path" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} DIR_PATH" 1>&2
        return 22
    fi

    if [ "$name_pattern" = "" ]; then
        \find "$path" -not -path '*/\.*' -type f
    else
        \find "$path" -not -path '*/\.*' -type f -name "$name_pattern"
    fi
}

# find directories in a directory with exclude hidden directories
fd() {
    local path=$1
    local name_pattern=$2

    if [ "$path" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} DIR_PATH" 1>&2
        return 22
    fi

    if [ "$name_pattern" = "" ]; then
        \find "$path" -not -path '*/\.*' -type d
    else
        \find "$path" -not -path '*/\.*' -type d -name "$name_pattern"
    fi
}

# find files and grep file name with exclude hidden files/directories
ffg() {
    if [ $# -ne 2 ]; then
        echo "Usage: ${FUNCNAME[0]} ROOT_DIR PATTERN" 1>&2
        return 22
    fi

    ff "$1" | \grep -E --ignore-case "$2"
}

psgrep() {
    local pattern=$1
    local psaux

    psaux=$(\ps aux)

    if [ "$pattern" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} PATTERN" 1>&2
        return 22
    fi

    echo -e "$psaux" | \head -1
    echo -e "$psaux" | \grep -E "$pattern"
}

pssort() {
    local sort_key=$1

    if \ps aux --sort "${sort_key}" > /dev/null 2>&1 ; then
        local stdout
        stdout=$(\ps aux --sort "${sort_key}")
        echo -e "${stdout}"
    else
        local header
        header=$(echo -e "$(\ps aux)" | \head -1)
        echo "invalid sort key. valid sort keys are:" ${header,,} 1>&2
        return 1
    fi
}

extract() {
    local archive_file=$1

    if [ ! -f "${archive_file}" ]; then
        echo "'${archive_file}' is not a valid file to extract" 1>&2
        return 1
    fi

    case "${archive_file}" in
        *.tar.bz2) \tar -xvf "${archive_file}"    ;;
        *.tar.gz)  \tar -xvf "${archive_file}"    ;;
        *.tar.xz)  \tar -xvf "${archive_file}"    ;;
        *.bz2)     \bunzip2 "${archive_file}"     ;;
        *.gz)      \gunzip "${archive_file}"      ;;
        *.tar)     \tar -xvf "${archive_file}"    ;;
        *.tbz2)    \tar -jxvf "${archive_file}"   ;;
        *.tgz)     \tar -zxvf "${archive_file}"   ;;
        *.zip)     \unzip "${archive_file}"       ;;
        *.ZIP)     \unzip "${archive_file}"       ;;
        *) echo "'${archive_file}' cannot be extracted via extract()" 1>&2; return 2 ;;
    esac
}

# list up installed packages
lspkg() {
    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        dpkg --list
    elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
        rpm -qa
    else
        echo "unknown distribution" 1>&2
        return 1
    fi
}

listpkg() {
    for command in $(compgen -c | sort); do
        bin_path=$(whichbin "$command" 2> /dev/null)
        if [ "$?" -ne 0 ]; then
            continue
        fi

        package=$(whichpkg "$bin_path" 2> /dev/null)
        if [ "$?" -ne 0 ]; then
            continue
        fi

        if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
            echo "$package"
        elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
            echo "$package: $bin_path"
        else
            echo "unknown distribution" 1>&2
            return 1
        fi
    done
}

# find a package which includes a specified command
whichpkg() {
    local command="$1"

    if [ "${command}" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} COMMAND" 1>&2
        return 22
    fi

    command_path=$(whichbin "${command}")
    result=$?
    if [ "$result" -ne 0 ]; then
        return $result
    fi

    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        package=$(dpkg -S "$command_path")<commit_msg>Add local to variable definitions<commit_after>whichbin() {
    local command_path

    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        command_path=$(\which $1 2> /dev/null)
    elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
        command_path=$(\which --skip-alias $1 2> /dev/null)
    else
        echo "${FUNCNAME[0]}: unknown distribution" 1>&2
        return 22
    fi

    result=$?
    if [ "$result" -ne 0 ]; then
        echo "no ${1} in (${PATH})" 1>&2
        return $result
    fi

    readlink -f $command_path
}

# Convert datetime to an epoch time.
#
# example: a datetime to an epoch time
#     $ date2epoch "2017-01-01 00:00:00+0900"
#     1483196400
#
# example: show current epoc time
#     $ date2epoch
#     1493254466
date2epoch() {
    if [ "$1" != "" ]; then
        \date +%s -d "$1"
    else
        \date +%s
    fi
}

# Convert an epoch time to a datetime
#
# example:
#     $ epoch2date 1483196400
#     2017-01-01 00:00:00+09:00
epoch2date() {
    if [ "$1" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} EPOCH_TIME" 1>&2
        return 22
    fi

    \date -d @"$1" --rfc-3339=seconds
}

# find files in a directory with exclude hidden files/directories
ff() {
    local path=$1
    local name_pattern=$2

    if [ "$path" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} DIR_PATH" 1>&2
        return 22
    fi

    if [ "$name_pattern" = "" ]; then
        \find "$path" -not -path '*/\.*' -type f
    else
        \find "$path" -not -path '*/\.*' -type f -name "$name_pattern"
    fi
}

# find directories in a directory with exclude hidden directories
fd() {
    local path=$1
    local name_pattern=$2

    if [ "$path" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} DIR_PATH" 1>&2
        return 22
    fi

    if [ "$name_pattern" = "" ]; then
        \find "$path" -not -path '*/\.*' -type d
    else
        \find "$path" -not -path '*/\.*' -type d -name "$name_pattern"
    fi
}

# find files and grep file name with exclude hidden files/directories
ffg() {
    if [ $# -ne 2 ]; then
        echo "Usage: ${FUNCNAME[0]} ROOT_DIR PATTERN" 1>&2
        return 22
    fi

    ff "$1" | \grep -E --ignore-case "$2"
}

psgrep() {
    local pattern=$1
    local psaux

    psaux=$(\ps aux)

    if [ "$pattern" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} PATTERN" 1>&2
        return 22
    fi

    echo -e "$psaux" | \head -1
    echo -e "$psaux" | \grep -E "$pattern"
}

pssort() {
    local sort_key=$1

    if \ps aux --sort "${sort_key}" > /dev/null 2>&1 ; then
        local stdout
        stdout=$(\ps aux --sort "${sort_key}")
        echo -e "${stdout}"
    else
        local header
        header=$(echo -e "$(\ps aux)" | \head -1)
        echo "invalid sort key. valid sort keys are:" ${header,,} 1>&2
        return 1
    fi
}

extract() {
    local archive_file=$1

    if [ ! -f "${archive_file}" ]; then
        echo "'${archive_file}' is not a valid file to extract" 1>&2
        return 1
    fi

    case "${archive_file}" in
        *.tar.bz2) \tar -xvf "${archive_file}"    ;;
        *.tar.gz)  \tar -xvf "${archive_file}"    ;;
        *.tar.xz)  \tar -xvf "${archive_file}"    ;;
        *.bz2)     \bunzip2 "${archive_file}"     ;;
        *.gz)      \gunzip "${archive_file}"      ;;
        *.tar)     \tar -xvf "${archive_file}"    ;;
        *.tbz2)    \tar -jxvf "${archive_file}"   ;;
        *.tgz)     \tar -zxvf "${archive_file}"   ;;
        *.zip)     \unzip "${archive_file}"       ;;
        *.ZIP)     \unzip "${archive_file}"       ;;
        *) echo "'${archive_file}' cannot be extracted via extract()" 1>&2; return 2 ;;
    esac
}

# list up installed packages
lspkg() {
    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        dpkg --list
    elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
        rpm -qa
    else
        echo "unknown distribution" 1>&2
        return 1
    fi
}

listpkg() {
    for command in $(compgen -c | sort); do
        bin_path=$(whichbin "$command" 2> /dev/null)
        if [ "$?" -ne 0 ]; then
            continue
        fi

        package=$(whichpkg "$bin_path" 2> /dev/null)
        if [ "$?" -ne 0 ]; then
            continue
        fi

        if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
            echo "$package"
        elif [ -e /etc/fedora-release ] || [ -e /etc/redhat-release ]; then
            echo "$package: $bin_path"
        else
            echo "unknown distribution" 1>&2
            return 1
        fi
    done
}

# find a package which includes a specified command
whichpkg() {
    local command=$1
    local command_path
    local result
    local package

    if [ "${command}" = "" ]; then
        echo "Usage: ${FUNCNAME[0]} COMMAND" 1>&2
        return 22
    fi

    command_path=$(whichbin "${command}")
    result=$?
    if [ "$result" -ne 0 ]; then
        return $result
    fi

    if [ -e /etc/debian_version ] || [ -e /etc/debian_release ]; then
        package=$(dpkg -S "$command_path")
{'fraction_non_alphanumeric': 0.16424447541309975, 'fraction_numerical': 0.03205255823213219, 'mean_word_length': 1.861577898034748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1399790', 'n_tokens_mistral': 3672, 'n_tokens_neox': 3377, 'n_words': 1009}