<commit_before>#!/bin/sh
#*************************************************************************
#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# Copyright 2008 by Sun Microsystems, Inc.
#
# OpenOffice.org - a multi-platform office productivity suite
#
# $RCSfile: makefile.mk,v $
#
# $Revision: 1.8 $
#
# This file is part of OpenOffice.org.
#
# OpenOffice.org is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenOffice.org is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenOffice.org.  If not, see
# <http://www.openoffice.org/license.html>
# for a copy of the LGPLv3 License.
#
#*************************************************************************

if [ -z "$TARFILE_LOCATION" ]; then
    echo "ERROR: no destination defined! please set TARFILE_LOCATION!"
    exit
fi

if [ ! -d "$TARFILE_LOCATION" ]; then
    mkdir $TARFILE_LOCATION
fi
if [ ! -d "$TARFILE_LOCATION" ]; then
    echo "ERROR: can't create"
    exit
fi

if [ -z "$1" ]; then
    echo "ERROR: parameter missing!"
    echo "usage: $0 <fetch list>"
    echo "first line must define the base url."
    exit
fi

# check for wget and md5sum
wget=
md5sum=
curl=

for i in wget /usr/bin/wget /usr/local/bin/wget /usr/sfw/bin/wget /opt/sfw/bin/wget; do
    eval "$i --version" > /dev/null 2>&1
    ret=$?
    if [ $ret -eq 0 ]; then
        wget=$i
        echo found wget: $wget
        break 2
    fi
done

for i in curl /usr/bin/curl /usr/local/bin/curl /usr/sfw/bin/curl /opt/sfw/bin/curl; do
# mac curl returns "2" on --version
#    eval "$i --version" > /dev/null 2>&1
#    ret=$?
#    if [ $ret -eq 0 ]; then
    if [ -x $i ]; then
        curl=$i
        echo found curl: $curl
        break 2
    fi
done

if [ -z "$wget" -a -z "$curl" ]; then
    echo "ERROR: neither  wget nor curl found!"
    exit
fi

for i in md5sum /usr/local/bin/md5sum gmd5sum /usr/sfw/bin/md5sum /opt/sfw/bin/gmd5sum; do
    eval "$i --version" > /dev/null 2>&1
    ret=$?
    if [ $ret -eq 0 ]; then
        md5sum=$i
        echo found md5sum: $md5sum
        break 2
    fi
done

if [ -z "$md5sum" ]; then
    echo "Warning: no md5sum: found!"
fi

start_dir=`pwd`
for i in `cat $1` ; do
#   echo $i
    if [ "$i" != `echo $i | sed "s/^http:\///"` ]; then
        tarurl=$i
    # TODO: check for comment
    else
        if [ "$tarurl" != "" ]; then
            cd $TARFILE_LOCATION<commit_msg>ause110: #i106731# new dir to check<commit_after>#!/bin/sh
#*************************************************************************
#
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# Copyright 2008 by Sun Microsystems, Inc.
#
# OpenOffice.org - a multi-platform office productivity suite
#
# $RCSfile: makefile.mk,v $
#
# $Revision: 1.8 $
#
# This file is part of OpenOffice.org.
#
# OpenOffice.org is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenOffice.org is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenOffice.org.  If not, see
# <http://www.openoffice.org/license.html>
# for a copy of the LGPLv3 License.
#
#*************************************************************************

if [ -z "$TARFILE_LOCATION" ]; then
    echo "ERROR: no destination defined! please set TARFILE_LOCATION!"
    exit
fi

if [ ! -d "$TARFILE_LOCATION" ]; then
    mkdir $TARFILE_LOCATION
fi
if [ ! -d "$TARFILE_LOCATION" ]; then
    echo "ERROR: can't create"
    exit
fi

if [ -z "$1" ]; then
    echo "ERROR: parameter missing!"
    echo "usage: $0 <fetch list>"
    echo "first line must define the base url."
    exit
fi

# check for wget and md5sum
wget=
md5sum=
curl=

for i in wget /usr/bin/wget /usr/local/bin/wget /usr/sfw/bin/wget /opt/sfw/bin/wget /opt/local/bin/wget; do
    eval "$i --version" > /dev/null 2>&1
    ret=$?
    if [ $ret -eq 0 ]; then
        wget=$i
        echo found wget: $wget
        break 2
    fi
done

for i in curl /usr/bin/curl /usr/local/bin/curl /usr/sfw/bin/curl /opt/sfw/bin/curl /opt/local/bin/curl; do
# mac curl returns "2" on --version
#    eval "$i --version" > /dev/null 2>&1
#    ret=$?
#    if [ $ret -eq 0 ]; then
    if [ -x $i ]; then
        curl=$i
        echo found curl: $curl
        break 2
    fi
done

if [ -z "$wget" -a -z "$curl" ]; then
    echo "ERROR: neither  wget nor curl found!"
    exit
fi

for i in md5sum /usr/local/bin/md5sum gmd5sum /usr/sfw/bin/md5sum /opt/sfw/bin/gmd5sum /opt/local/bin/md5sum; do
    eval "$i --version" > /dev/null 2>&1
    ret=$?
    if [ $ret -eq 0 ]; then
        md5sum=$i
        echo found md5sum: $md5sum
        break 2
    fi
done

if [ -z "$md5sum" ]; then
    echo "Warning: no md5sum: found!"
fi

start_dir=`pwd`
for i in `cat $1` ; do
#   echo $i
    if [ "$i" != `echo $i | sed "s/^http:\///"` ]; then
        tarurl=$i
    # TODO: check for comment
    else
        if [ "$tarurl" != "" ]; then
            cd $TARFILE_LOCATION
{'fraction_non_alphanumeric': 0.16678093932122043, 'fraction_numerical': 0.014741172437435722, 'mean_word_length': 2.8112344872632264, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '302698', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1937, 'n_words': 723}