<commit_before>#!/usr/bin/env sh
# A GitHub API client library written in POSIX sh
#
# Requirements:
#
# * A POSIX environment (tested against Busybox v1.19.4)
# * curl (tested against 7.32.0)
#
# Optional requirements:
#
# * jq <http://stedolan.github.io/jq/> (tested against 1.3)
#   If jq is not installed commands will output raw JSON; if jq is installed
#   commands can be pretty-printed for use with other shell tools.
#
# Setup
#
# Authentication credentials are read from a ~/.netrc file with the following
# format. Generate the token on GitHub under Account Settings -> Applications.
# Restrict permissions on that file with `chmod 600 ~/.netrc`!
#
#   machine api.github.com
#       login <username>
#       password <<PASSWORD>>
#
# Configuration
#
# The following environment variables may be set to customize ${NAME}.
#
# OCTOKIT_SH_URL=${OCTOKIT_SH_URL}
#   Base URL for GitHub or GitHub Enterprise.
# OCTOKIT_SH_ACCEPT=${OCTOKIT_SH_ACCEPT}
#   The 'Accept' header to send with each request.
# OCTOKIT_SH_JQ_BIN=${OCTOKIT_SH_JQ_BIN}
#   The name of the jq binary, if installed.
# OCTOKIT_SH_VERBOSE=${OCTOKIT_SH_VERBOSE}
#   The debug logging verbosity level.
#   1 for info; 2 for debug; 3 for trace (full curl request/reponse output).
# OCTOKIT_SH_RATE_LIMIT=${OCTOKIT_SH_RATE_LIMIT}
#   Output current GitHub rate limit information to stderr.

export NAME=$(basename $0)
export VERSION='0.1.0'

ALL_FUNCS=$(awk 'BEGIN {ORS=" "} !/^_/ && /^[a-zA-Z0-9_]+\s*\(\)/ {
    sub(/\(\)$/, "", $1); print $1 }' $0 | sort)
export ALL_FUNCS

export OCTOKIT_SH_URL=${OCTOKIT_SH_URL:-'https://api.github.com'}
export OCTOKIT_SH_ACCEPT=${OCTOKIT_SH_ACCEPT:-'application/vnd.github.v3+json'}
export OCTOKIT_SH_JQ_BIN="${OCTOKIT_SH_JQ_BIN:-jq}"
export OCTOKIT_SH_VERBOSE="${OCTOKIT_SH_VERBOSE:-0}"
export OCTOKIT_SH_RATE_LIMIT="${OCTOKIT_SH_RATE_LIMIT:-0}"

# Detect if jq is installed.
type "$OCTOKIT_SH_JQ_BIN" 1>/dev/null 2>/dev/null
NO_JQ=$?

# Customizable logging output.
exec 4>/dev/null
exec 5>/dev/null
exec 6>/dev/null
export LINFO=4      # Info-level log messages.
export LDEBUG=5     # Debug-level log messages.
export LSUMMARY=6    # Summary output.

_log() {
    # A lightweight logging system based on file descriptors
    #
    local level=${1:?Level is required.}
    #   The level for a given log message. (info or debug)
    local message=${2:?Message is required.}
    #   The log message.
    #
    # Usage:
    #   _log debug 'Starting the combobulator!'

    shift 2

    local lname

    case "$level" in
        info) lname='INFO'; level=$LINFO ;;
        debug) lname='DEBUG'; level=$LDEBUG ;;
        *) printf 'Invalid logging level: %s\n' "$level" ;;
    esac

    printf '%s %s: %s\n' "$NAME" "$lname" "$message" 1>&$level
}

_helptext() {
    # Extract contiguous lines of comments and function params as help text
    #
    # Indentation will be ignored. The first line of the match will be ignored
    # (this is to ignore the she-bang of a file or the function name).  Local
    # variable declarations and their default values can also be pulled in as
    # documentation.  Exits upon encountering the first blank line.
    #
    # Exported environment variables can be used for string interpolation in
    # the extracted commented text.
    #
    # - (stdin)
    #   The text of a function body to parse.
    # $1
    #   A file name to parse.

    awk 'NR != 1 && /^\s*#/ {
        line=$0
        while(match(line, "[$]{[^}]*}")) {
            var=substr(line, RSTART+2, RLENGTH -3)
            gsub("[$]{"var"}", ENVIRON[var], line)
        }
        gsub(/^\s*#\s?/, "", line)
        print line
    }
    /^\s*local/ {
        sub(/^\s*local /, "")
        sub(/:.*$/, "")
        sub(/=/, " : ")
        sub(/\${/, "$")
        print
    }
    !NF { exit }' "$@"
}

help() {
    # Output the help text for a command
    #
    # Usage:
    #   help commandname
    #
    # $1
    #   Function name to search for; if omitted searches whole file.

    if [ $# -gt 0 ]; then
        awk -v fname="^$1" '$0 ~ fname, /^}/ { print }' $0 | _helptext
    else
        _helptext $0
        printf '\n'
        help _main
    fi
}

_main() {
    # Available commands: ${ALL_FUNCS}
    #
    # Usage: ${NAME} [<options>] command [<name=value>]
    # Command-specific help: ${NAME} help command
    #
    # Options:
    #   -h      Show this screen.
    #   -V      Show version.
    #   -v      Enable verbose output; same as `$OCTOKIT_SH_VERBOSE`.
    #   -x      Enable xtrace debug logging.
    #   -r      Print your current GitHub API rate limit to stderr.
    #   -q      Quiet; don't print to stdout.
    #   -j      Output raw JSON; don't process with jq.

    local cmd ret opt OPTARG OPTIND
    local quiet=0
    local temp_dir="/tmp/oksh-${RANDOM}-${$}"
    local summary_fifo="${temp_dir}/oksh_summary.fifo"

    trap '
        excode=$?; trap - EXIT;
        exec 4>&-
        exec 5>&-
        exec 6>&-
        rm -rf '"$temp_dir"'
        exit
        echo $excode<commit_msg>Move verbose info from env var to flag<commit_after>#!/usr/bin/env sh
# A GitHub API client library written in POSIX sh
#
# Requirements:
#
# * A POSIX environment (tested against Busybox v1.19.4)
# * curl (tested against 7.32.0)
#
# Optional requirements:
#
# * jq <http://stedolan.github.io/jq/> (tested against 1.3)
#   If jq is not installed commands will output raw JSON; if jq is installed
#   commands can be pretty-printed for use with other shell tools.
#
# Setup
#
# Authentication credentials are read from a ~/.netrc file with the following
# format. Generate the token on GitHub under Account Settings -> Applications.
# Restrict permissions on that file with `chmod 600 ~/.netrc`!
#
#   machine api.github.com
#       login <username>
#       password <<PASSWORD>>
#
# Configuration
#
# The following environment variables may be set to customize ${NAME}.
#
# OCTOKIT_SH_URL=${OCTOKIT_SH_URL}
#   Base URL for GitHub or GitHub Enterprise.
# OCTOKIT_SH_ACCEPT=${OCTOKIT_SH_ACCEPT}
#   The 'Accept' header to send with each request.
# OCTOKIT_SH_JQ_BIN=${OCTOKIT_SH_JQ_BIN}
#   The name of the jq binary, if installed.
# OCTOKIT_SH_VERBOSE=${OCTOKIT_SH_VERBOSE}
#   The debug logging verbosity level. Same as the verbose flag.
# OCTOKIT_SH_RATE_LIMIT=${OCTOKIT_SH_RATE_LIMIT}
#   Output current GitHub rate limit information to stderr.

export NAME=$(basename $0)
export VERSION='0.1.0'

ALL_FUNCS=$(awk 'BEGIN {ORS=" "} !/^_/ && /^[a-zA-Z0-9_]+\s*\(\)/ {
    sub(/\(\)$/, "", $1); print $1 }' $0 | sort)
export ALL_FUNCS

export OCTOKIT_SH_URL=${OCTOKIT_SH_URL:-'https://api.github.com'}
export OCTOKIT_SH_ACCEPT=${OCTOKIT_SH_ACCEPT:-'application/vnd.github.v3+json'}
export OCTOKIT_SH_JQ_BIN="${OCTOKIT_SH_JQ_BIN:-jq}"
export OCTOKIT_SH_VERBOSE="${OCTOKIT_SH_VERBOSE:-0}"
export OCTOKIT_SH_RATE_LIMIT="${OCTOKIT_SH_RATE_LIMIT:-0}"

# Detect if jq is installed.
type "$OCTOKIT_SH_JQ_BIN" 1>/dev/null 2>/dev/null
NO_JQ=$?

# Customizable logging output.
exec 4>/dev/null
exec 5>/dev/null
exec 6>/dev/null
export LINFO=4      # Info-level log messages.
export LDEBUG=5     # Debug-level log messages.
export LSUMMARY=6    # Summary output.

_log() {
    # A lightweight logging system based on file descriptors
    #
    local level=${1:?Level is required.}
    #   The level for a given log message. (info or debug)
    local message=${2:?Message is required.}
    #   The log message.
    #
    # Usage:
    #   _log debug 'Starting the combobulator!'

    shift 2

    local lname

    case "$level" in
        info) lname='INFO'; level=$LINFO ;;
        debug) lname='DEBUG'; level=$LDEBUG ;;
        *) printf 'Invalid logging level: %s\n' "$level" ;;
    esac

    printf '%s %s: %s\n' "$NAME" "$lname" "$message" 1>&$level
}

_helptext() {
    # Extract contiguous lines of comments and function params as help text
    #
    # Indentation will be ignored. The first line of the match will be ignored
    # (this is to ignore the she-bang of a file or the function name).  Local
    # variable declarations and their default values can also be pulled in as
    # documentation.  Exits upon encountering the first blank line.
    #
    # Exported environment variables can be used for string interpolation in
    # the extracted commented text.
    #
    # - (stdin)
    #   The text of a function body to parse.
    # $1
    #   A file name to parse.

    awk 'NR != 1 && /^\s*#/ {
        line=$0
        while(match(line, "[$]{[^}]*}")) {
            var=substr(line, RSTART+2, RLENGTH -3)
            gsub("[$]{"var"}", ENVIRON[var], line)
        }
        gsub(/^\s*#\s?/, "", line)
        print line
    }
    /^\s*local/ {
        sub(/^\s*local /, "")
        sub(/:.*$/, "")
        sub(/=/, " : ")
        sub(/\${/, "$")
        print
    }
    !NF { exit }' "$@"
}

help() {
    # Output the help text for a command
    #
    # Usage:
    #   help commandname
    #
    # $1
    #   Function name to search for; if omitted searches whole file.

    if [ $# -gt 0 ]; then
        awk -v fname="^$1" '$0 ~ fname, /^}/ { print }' $0 | _helptext
    else
        _helptext $0
        printf '\n'
        help _main
    fi
}

_main() {
    # Available commands: ${ALL_FUNCS}
    #
    # Usage: ${NAME} [<options>] command [<name=value>]
    # Command-specific help: ${NAME} help command
    #
    # Options:
    #   -h      Show this screen.
    #   -V      Show version.
    #   -v      Enable verbose output to stderr. 1 for info; 2 for debug;
    #           3 for trace (full curl request/reponse output).
    #   -x      Enable xtrace debug logging.
    #   -r      Print current GitHub API rate limit to stderr.
    #   -q      Quiet; don't print to stdout.
    #   -j      Output raw JSON; don't process with jq.

    local cmd ret opt OPTARG OPTIND
    local quiet=0
    local temp_dir="/tmp/oksh-${RANDOM}-${$}"
    local summary_fifo="${temp_dir}/oksh_summary.fifo"

    trap '
        excode=$?; trap - EXIT;
        exec 4>&-
        exec 5>&-
        exec 6>&-
        rm -rf '"$temp_dir"'
        exit
        echo $excode
{'fraction_non_alphanumeric': 0.13333994250024783, 'fraction_numerical': 0.010905125408942203, 'mean_word_length': 2.5925925925925926, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1811925', 'n_tokens_mistral': 3574, 'n_tokens_neox': 3392, 'n_words': 1051}