<commit_before>#!/bin/bash
   
export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
   
set +u
set -e
#set -x

#echo -e "$0 Parameter: $*"

declare RED_BEG="\\033[31m"
declare PUR_BEG="\\033[35m"
declare GREEN_BEG="\\033[32m"
declare BLUE_BEG="\\033[34m"
declare TUERK_BEG="\\033[34m"
declare COL_END="\\033[0m"
declare Underline="\\033[4m"

declare ARG_OPTION=$1

set -u

needed_tools() {        #Validate if the needed tool are on the shell
set +e
    Head_trace "Validate if the needed tool are on the shell"
    
    wget 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "Wget isn't install on your shell"
        error_trace "Please install wget"
        exit
    else
        Correct_trace "Wget is on your shell"
    fi
    
    gunzip 2>/dev/null

    if [ $? -gt 2 ]; then
        error_trace "gunzip isn't install on your shell"
        error_trace "Please install gunzip"
        exit
    else
         Correct_trace "gunzip is on your shell"
    fi
    
    dd d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "dd isn't install on your shell"
        error_trace "Please install dd"
        exit
    else
    Correct_trace "dd is on your shell"
    fi
    
    md5sum d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "md5sum isn't install on your shell"
        error_trace "Please install m5sum"
        exit
    else
        Correct_trace "md5sum is on your shell"
    fi
        Correct_trace "All tools are on the shell"
    set -e
}

needed_truncate() {
    set +e
    truncate d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "truncate isn't install on your shell"
        error_trace "Please install truncate"
        exit
    else
        Correct_trace "truncate is on your shell"
    fi
    set -e
}

needed_truncate_OS() {
    set +e
    if [ -d /usr/local/bin/truncate ]; then
        Correct_trace "truncate is on your shell"
    else
        error_trace "truncate isn't install on your shell"
        error_trace "Please install truncate"
        exit
    fi
    set -e
}
    
download() {             #Download the Software and unpack them, if required 
    Head_trace "download process"
    Info_trace "Download the Software"
    if ! wget $LINK; then
        error_trace "Maybe the URL is not available or the URL ist passed off "
        help
        exit
    fi
    Info_trace "Try to unpack the downloaded Software"
    if ! gunzip  $FILENAME >/dev/null 2>/dev/null; then
        unpack_text
    fi
}

unpack() {               #Unpack the Software
    Head_trace "Unpack process"
    Info_trace "Unpack the Software"<commit_msg>fixed a truncate error by mac OS<commit_after>#!/bin/bash
   
export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
   
set +u
set -e
#set -x

#echo -e "$0 Parameter: $*"

declare RED_BEG="\\033[31m"
declare PUR_BEG="\\033[35m"
declare GREEN_BEG="\\033[32m"
declare BLUE_BEG="\\033[34m"
declare TUERK_BEG="\\033[34m"
declare COL_END="\\033[0m"
declare Underline="\\033[4m"

declare ARG_OPTION=$1

set -u

needed_tools() {        #Validate if the needed tool are on the shell
set +e
    Head_trace "Validate if the needed tool are on the shell"
    
    wget 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "Wget isn't install on your shell"
        error_trace "Please install wget"
        exit
    else
        Correct_trace "Wget is on your shell"
    fi
    
    gunzip 2>/dev/null

    if [ $? -gt 2 ]; then
        error_trace "gunzip isn't install on your shell"
        error_trace "Please install gunzip"
        exit
    else
         Correct_trace "gunzip is on your shell"
    fi
    
    dd d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "dd isn't install on your shell"
        error_trace "Please install dd"
        exit
    else
    Correct_trace "dd is on your shell"
    fi
    
    md5sum d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "md5sum isn't install on your shell"
        error_trace "Please install m5sum"
        exit
    else
        Correct_trace "md5sum is on your shell"
    fi
        Correct_trace "All tools are on the shell"
    set -e
}

needed_truncate() {
    set +e
    truncate d 2>/dev/null
    if [ $? -gt 2 ]; then
        error_trace "truncate isn't install on your shell"
        error_trace "Please install truncate"
        exit
    else
        Correct_trace "truncate is on your shell"
    fi
    set -e
}

needed_truncate_OS() {
    set +e
    if [ -e /usr/local/bin/truncate ]; then
        Correct_trace "truncate is on your shell"
    else
        error_trace "truncate isn't install on your shell"
        error_trace "Please install truncate"
        exit
    fi
    set -e
}
    
download() {             #Download the Software and unpack them, if required 
    Head_trace "download process"
    Info_trace "Download the Software"
    if ! wget $LINK; then
        error_trace "Maybe the URL is not available or the URL ist passed off "
        help
        exit
    fi
    Info_trace "Try to unpack the downloaded Software"
    if ! gunzip  $FILENAME >/dev/null 2>/dev/null; then
        unpack_text
    fi
}

unpack() {               #Unpack the Software
    Head_trace "Unpack process"
    Info_trace "Unpack the Software"
{'fraction_non_alphanumeric': 0.09299677358132473, 'fraction_numerical': 0.020117669386980452, 'mean_word_length': 2.172787477423239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '558524', 'n_tokens_mistral': 1858, 'n_tokens_neox': 1717, 'n_words': 589}