<commit_before>#!/bin/bash
###########################################################################
# a2o Essential Puppet Modules                                            #
#-------------------------------------------------------------------------#
# Copyright (c) 2012 <NAME>                                       #
#-------------------------------------------------------------------------#
# This source file is subject to version 2.0 of the Apache License,       #
# that is bundled with this package in the file LICENSE, and is           #
# available through the world-wide-web at the following url:              #
# http://www.apache.org/licenses/LICENSE-2.0                              #
#-------------------------------------------------------------------------#
# Authors: <NAME> <my_name [at] a2o {dot} si>                     #
###########################################################################



export SUCCESS=0
export FAILURE=255



### Get configuration if exists
if [ -f /var/src/build_functions.conf-site ]; then
    . /var/src/build_functions.conf-site
fi
if [ -f /var/src/build_functions.conf ]; then
    . /var/src/build_functions.conf
fi



CheckSrcRoot ()
{
    if [ ! -n "$SRCROOT" ]; then
	return $FAILURE
    else
	return $SUCCESS
    fi
}



GetNoTestArchive ()
{
    echo "----------[ Retrieving archive ]------------------------------"
    CheckSrcRoot &&
    cd $SRCROOT &&

    if [ "x$CTOOL_PROXY_URI" != "x" ]; then
	# Proxy-ed download
        RES=`echo "$PURI" | grep -c "$CTOOL_PROXY_EXCEPTION_REGEX" | cat` &&
	if [ "$RES" -gt "0" ]; then
	    wget -c --no-check-certificate --inet4-only --timeout=10 --tries=4 $PURI

	elif [ "`wget --help | grep -c content-disposition | cat`" -gt "0" ]; then
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
	    --content-disposition "${CTOOL_PROXY_URI}${PURI}"

	else
	    # Older wget, does not support --content-disposition switch (1.10 for example)
	    FILENAME=`echo $PURI | awk -F/ '{print $NF}'`
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
	    --output-document=$FILENAME "${CTOOL_PROXY_URI}${PURI}"
	fi

    else

	# Direct download
#	if [ "`wget --help | grep -c content-disposition | cat`" -gt "0" ]; then
#    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
#	    --content-disposition "${PURI}"
#
#	else
	    # Older wget, does not support --content-disposition switch (1.10 for example)
	    FILENAME=`echo $PURI | awk -F/ '{print $NF}'`
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
		--output-document=$FILENAME "${PURI}"
#	fi

    fi # CTOOL_PROXY_URI

    RES=$?
    if [ "$RES" != "0" ]; then
	exit 1
    fi
}



GetArchive ()
{
    GetNoTestArchive
    if [ "$?" != "0" ]; then
	return $FAILURE
    fi
}



UnpackArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT &&
    if [ ! -e $PDIR ]; then
	UnpackNoTestArchive
    fi
}



UnpackNoTestArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT &&
    UnpackHere
}



UnpackHere ()
{
    echo; echo "----------[ Unpacking archive ]------------------------------"
    if [[ ! -z `echo $PFILE | grep '\.\(tar\.gz\|tgz\)$'` ]]; then
	tar -xzf $PFILE
    elif [[ ! -z `echo $PFILE | grep '\.tar\.bz2$'` ]]; then
	tar -xjf $PFILE
    elif [[ ! -z `echo $PFILE | grep '\.tar\.xz$'` ]]; then
	xz -dc $PFILE | tar -x
    elif [[ ! -z `echo $PFILE | grep '\.tar$'` ]]; then
	tar -xf $PFILE
    else
	echo "ERROR - unknown archive $PFILE" &&
	return $FAILURE
    fi
}



CleanArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT/$PDIR &&

    echo; echo "----------[ make clean ]------------------------------"
    make clean
    echo; echo "----------[ make realclean ]------------------------------"
    make realclean
    echo; echo "----------[ make distclean ]------------------------------"
    make distclean
    return $SUCCESS
}



GetUnpack ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



GetUnpackCd ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    cd $SRCROOT/$PDIR &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



GetUnpackClean ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    CleanArchive &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



unset PNAME &&
unset PVERSION &&
unset PDIR &&
unset PFILE &&
unset PURI
<commit_msg>Added support for .zip files to compiletool<commit_after>#!/bin/bash
###########################################################################
# a2o Essential Puppet Modules                                            #
#-------------------------------------------------------------------------#
# Copyright (c) 2012 <NAME>                                       #
#-------------------------------------------------------------------------#
# This source file is subject to version 2.0 of the Apache License,       #
# that is bundled with this package in the file LICENSE, and is           #
# available through the world-wide-web at the following url:              #
# http://www.apache.org/licenses/LICENSE-2.0                              #
#-------------------------------------------------------------------------#
# Authors: <NAME> <my_name [at] a2o {dot} si>                     #
###########################################################################



export SUCCESS=0
export FAILURE=255



### Get configuration if exists
if [ -f /var/src/build_functions.conf-site ]; then
    . /var/src/build_functions.conf-site
fi
if [ -f /var/src/build_functions.conf ]; then
    . /var/src/build_functions.conf
fi



CheckSrcRoot ()
{
    if [ ! -n "$SRCROOT" ]; then
	return $FAILURE
    else
	return $SUCCESS
    fi
}



GetNoTestArchive ()
{
    echo "----------[ Retrieving archive ]------------------------------"
    CheckSrcRoot &&
    cd $SRCROOT &&

    if [ "x$CTOOL_PROXY_URI" != "x" ]; then
	# Proxy-ed download
        RES=`echo "$PURI" | grep -c "$CTOOL_PROXY_EXCEPTION_REGEX" | cat` &&
	if [ "$RES" -gt "0" ]; then
	    wget -c --no-check-certificate --inet4-only --timeout=10 --tries=4 $PURI

	elif [ "`wget --help | grep -c content-disposition | cat`" -gt "0" ]; then
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
	    --content-disposition "${CTOOL_PROXY_URI}${PURI}"

	else
	    # Older wget, does not support --content-disposition switch (1.10 for example)
	    FILENAME=`echo $PURI | awk -F/ '{print $NF}'`
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
	    --output-document=$FILENAME "${CTOOL_PROXY_URI}${PURI}"
	fi

    else

	# Direct download
#	if [ "`wget --help | grep -c content-disposition | cat`" -gt "0" ]; then
#    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
#	    --content-disposition "${PURI}"
#
#	else
	    # Older wget, does not support --content-disposition switch (1.10 for example)
	    FILENAME=`echo $PURI | awk -F/ '{print $NF}'`
    	    wget -c --no-check-certificate --inet4-only --timeout=120 --tries=2 \
		--output-document=$FILENAME "${PURI}"
#	fi

    fi # CTOOL_PROXY_URI

    RES=$?
    if [ "$RES" != "0" ]; then
	exit 1
    fi
}



GetArchive ()
{
    GetNoTestArchive
    if [ "$?" != "0" ]; then
	return $FAILURE
    fi
}



UnpackArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT &&
    if [ ! -e $PDIR ]; then
	UnpackNoTestArchive
    fi
}



UnpackNoTestArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT &&
    UnpackHere
}



UnpackHere ()
{
    echo; echo "----------[ Unpacking archive ]------------------------------"
    if [[ ! -z `echo $PFILE | grep '\.\(tar\.gz\|tgz\)$'` ]]; then
	tar -xzf $PFILE
    elif [[ ! -z `echo $PFILE | grep '\.tar\.bz2$'` ]]; then
	tar -xjf $PFILE
    elif [[ ! -z `echo $PFILE | grep '\.tar\.xz$'` ]]; then
	xz -dc $PFILE | tar -x
    elif [[ ! -z `echo $PFILE | grep '\.tar$'` ]]; then
	tar -xf $PFILE
    elif [[ ! -z `echo $PFILE | grep '\.zip$'` ]]; then
	unzip $PFILE
    else
	echo "ERROR - unknown archive $PFILE" &&
	return $FAILURE
    fi
}



CleanArchive ()
{
    CheckSrcRoot &&
    cd $SRCROOT/$PDIR &&

    echo; echo "----------[ make clean ]------------------------------"
    make clean
    echo; echo "----------[ make realclean ]------------------------------"
    make realclean
    echo; echo "----------[ make distclean ]------------------------------"
    make distclean
    return $SUCCESS
}



GetUnpack ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



GetUnpackCd ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    cd $SRCROOT/$PDIR &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



GetUnpackClean ()
{
    CheckSrcRoot &&
    GetArchive &&
    UnpackArchive &&
    CleanArchive &&
    echo &&
    echo "----------[ Configuring, building and installing ]------------------------------"
}



unset PNAME &&
unset PVERSION &&
unset PDIR &&
unset PFILE &&
unset PURI

{'fraction_non_alphanumeric': 0.2786037491919845, 'fraction_numerical': 0.01098901098901099, 'mean_word_length': 1.612721643681396, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3019068', 'n_tokens_mistral': 3048, 'n_tokens_neox': 2824, 'n_words': 731}