<commit_before>set -e

test -z $NIX_GCC && NIX_GCC=@gcc@


# Set up the initial path.
PATH=
for i in $NIX_GCC @initialPath@; do
    PATH=$PATH${PATH:+:}$i/bin
done

if test "$NIX_DEBUG" = "1"; then
    echo "Initial path: $PATH"
fi


# Execute the pre-hook.
export SHELL=@shell@
export shell=@shell@
param1=@param1@
param2=@param2@
param3=@param3@
param4=@param4@
param5=@param5@
if test -n "@preHook@"; then
    source @preHook@
fi


# Check that the pre-hook initialised SHELL.
if test -z "$SHELL"; then echo "SHELL not set"; exit 1; fi


# Hack: run gcc's setup hook.
envHooks=()
if test -f $NIX_GCC/nix-support/setup-hook; then
    source $NIX_GCC/nix-support/setup-hook
fi

    
# Ensure that the given directory exists.
ensureDir() {
    local dir=$1
    if ! test -x "$dir"; then mkdir -p "$dir"; fi
}


# Called when some build action fails.  If $succeedOnFailure is set,
# create the file `$out/nix-support/failed' to signal failure, and
# exit normally.  Otherwise, exit with failure.
fail() {
    exitCode=$?
    if test "$succeedOnFailure" = 1; then
        ensureDir "$out/nix-support"
        touch "$out/nix-support/failed"
        exit 0
    else
        exit $?
    fi
}


# Allow the caller to augment buildInputs (it's not always possible to
# do this before the call to setup.sh, since the PATH is empty at that
# point; here we have a basic Unix environment).
eval "$addInputsHook"


# Recursively find all build inputs.
findInputs()
{
    local pkg=$1

    case $pkgs in
        *\ $pkg\ *)
            return 0
            ;;
    esac
    
    pkgs="$pkgs $pkg "

    if test -f $pkg/nix-support/setup-hook; then
        source $pkg/nix-support/setup-hook
    fi
    
    if test -f $pkg/nix-support/propagated-build-inputs; then
        for i in $(cat $pkg/nix-support/propagated-build-inputs); do
            findInputs $i
        done
    fi
}

pkgs=""
if test -n "$buildinputs"; then
    buildInputs="$buildinputs" # compatibility
fi
for i in $buildInputs $propagatedBuildInputs; do
    findInputs $i
done


# Set the relevant environment variables to point to the build inputs
# found above.
addToEnv()
{
    local pkg=$1

    if test "$ignoreFailedInputs" != "1" -a -e $1/nix-support/failed; then
        echo "failed input $1" >&2
        fail
    fi

    if test -d $1/bin; then
        export _PATH=$_PATH${_PATH:+:}$1/bin
    fi

    for i in "${envHooks[@]}"; do
        $i $pkg
    done
}

for i in $pkgs; do
    addToEnv $i
done


# Add the output as an rpath.
if test "$NIX_NO_SELF_RPATH" != "1"; then
    export NIX_LDFLAGS="-rpath $out/lib $NIX_LDFLAGS"
fi


# Strip debug information by default.
if test -z "$NIX_STRIP_DEBUG"; then
    export NIX_STRIP_DEBUG=1
    export NIX_CFLAGS_STRIP="-g0 -Wl,--strip-debug"
fi


# Do we know where the store is?  This is required for purity checking.
if test -z "$NIX_STORE"; then
    echo "Error: you have an old version of Nix that does not set the" \
        "NIX_STORE variable.  Please upgrade." >&2
    exit 1
fi


# We also need to know the root of the build directory for purity checking.
if test -z "$NIX_BUILD_TOP"; then
    echo "Error: you have an old version of Nix that does not set the" \
        "NIX_BUILD_TOP variable.  Please upgrade." >&2
    exit 1
fi


# Set the TZ (timezone) environment variable, otherwise commands like
# `date' will complain (e.g., `Tue Mar 9 10:01:47 Local time zone must
# be set--see zic manual page 2004').
export TZ=UTC


# Execute the post-hook.
if test -n "@postHook@"; then
    source @postHook@
fi

PATH=$_PATH${_PATH:+:}$PATH
if test "$NIX_DEBUG" = "1"; then
    echo "Final path: $PATH"
fi


######################################################################
# Textual substitution functions.


substitute() {
    local input="$1"
    local output="$2"

    local params=("$@")

    local sedScript=$NIX_BUILD_TOP/.sedargs
    rm -f $sedScript
    touch $sedScript

    local n p pattern replacement varName
    
    for ((n = 2; n < ${#params[*]}; n += 1)); do
        p=${params[$n]}

        if test "$p" = "--replace"; then
            pattern=${params[$((n + 1))]}
            replacement=${params[$((n + 2))]}
            n=$((n + 2))
            echo "s^$pattern^$replacement^g" >> $sedScript
            sedArgs=("${sedArgs[@]}" "-e" )<commit_msg>* Workaround a bug in older versions of bash.<commit_after>set -e

test -z $NIX_GCC && NIX_GCC=@gcc@


# Set up the initial path.
PATH=
for i in $NIX_GCC @initialPath@; do
    PATH=$PATH${PATH:+:}$i/bin
done

if test "$NIX_DEBUG" = "1"; then
    echo "Initial path: $PATH"
fi


# Execute the pre-hook.
export SHELL=@shell@
export shell=@shell@
param1=@param1@
param2=@param2@
param3=@param3@
param4=@param4@
param5=@param5@
if test -n "@preHook@"; then
    source @preHook@
fi


# Check that the pre-hook initialised SHELL.
if test -z "$SHELL"; then echo "SHELL not set"; exit 1; fi


# Hack: run gcc's setup hook.
envHooks=()
if test -f $NIX_GCC/nix-support/setup-hook; then
    source $NIX_GCC/nix-support/setup-hook
fi

    
# Ensure that the given directory exists.
ensureDir() {
    local dir=$1
    if ! test -x "$dir"; then mkdir -p "$dir"; fi
}


# Called when some build action fails.  If $succeedOnFailure is set,
# create the file `$out/nix-support/failed' to signal failure, and
# exit normally.  Otherwise, exit with failure.
fail() {
    exitCode=$?
    if test "$succeedOnFailure" = 1; then
        ensureDir "$out/nix-support"
        touch "$out/nix-support/failed"
        exit 0
    else
        exit $?
    fi
}


# Allow the caller to augment buildInputs (it's not always possible to
# do this before the call to setup.sh, since the PATH is empty at that
# point; here we have a basic Unix environment).
eval "$addInputsHook"


# Recursively find all build inputs.
findInputs()
{
    local pkg=$1

    case $pkgs in
        *\ $pkg\ *)
            return 0
            ;;
    esac
    
    pkgs="$pkgs $pkg "

    if test -f $pkg/nix-support/setup-hook; then
        source $pkg/nix-support/setup-hook
    fi
    
    if test -f $pkg/nix-support/propagated-build-inputs; then
        for i in $(cat $pkg/nix-support/propagated-build-inputs); do
            findInputs $i
        done
    fi
}

pkgs=""
if test -n "$buildinputs"; then
    buildInputs="$buildinputs" # compatibility
fi
for i in $buildInputs $propagatedBuildInputs; do
    findInputs $i
done


# Set the relevant environment variables to point to the build inputs
# found above.
addToEnv()
{
    local pkg=$1

    if test "$ignoreFailedInputs" != "1" -a -e $1/nix-support/failed; then
        echo "failed input $1" >&2
        fail
    fi

    if test -d $1/bin; then
        export _PATH=$_PATH${_PATH:+:}$1/bin
    fi

    for i in "${envHooks[@]}"; do
        $i $pkg
    done
}

for i in $pkgs; do
    addToEnv $i
done


# Add the output as an rpath.
if test "$NIX_NO_SELF_RPATH" != "1"; then
    export NIX_LDFLAGS="-rpath $out/lib $NIX_LDFLAGS"
fi


# Strip debug information by default.
if test -z "$NIX_STRIP_DEBUG"; then
    export NIX_STRIP_DEBUG=1
    export NIX_CFLAGS_STRIP="-g0 -Wl,--strip-debug"
fi


# Do we know where the store is?  This is required for purity checking.
if test -z "$NIX_STORE"; then
    echo "Error: you have an old version of Nix that does not set the" \
        "NIX_STORE variable.  Please upgrade." >&2
    exit 1
fi


# We also need to know the root of the build directory for purity checking.
if test -z "$NIX_BUILD_TOP"; then
    echo "Error: you have an old version of Nix that does not set the" \
        "NIX_BUILD_TOP variable.  Please upgrade." >&2
    exit 1
fi


# Set the TZ (timezone) environment variable, otherwise commands like
# `date' will complain (e.g., `Tue Mar 9 10:01:47 Local time zone must
# be set--see zic manual page 2004').
export TZ=UTC


# Execute the post-hook.
if test -n "@postHook@"; then
    source @postHook@
fi

PATH=$_PATH${_PATH:+:}$PATH
if test "$NIX_DEBUG" = "1"; then
    echo "Final path: $PATH"
fi


######################################################################
# Textual substitution functions.


substitute() {
    local input="$1"
    local output="$2"

    local -a params=("$@")

    local sedScript=$NIX_BUILD_TOP/.sedargs
    rm -f $sedScript
    touch $sedScript

    local n p pattern replacement varName
    
    for ((n = 2; n < ${#params[*]}; n += 1)); do
        p=${params[$n]}

        if test "$p" = "--replace"; then
            pattern=${params[$((n + 1))]}
            replacement=${params[$((n + 2))]}
            n=$((n + 2))
            echo "s^$pattern^$replacement^g" >> $sedScript
            sedArgs=("${sedArgs[@]}" "-e" )
{'fraction_non_alphanumeric': 0.141431720617241, 'fraction_numerical': 0.01160227404571296, 'mean_word_length': 2.774080560420315, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5910530', 'n_tokens_mistral': 3067, 'n_tokens_neox': 2901, 'n_words': 1009}