<commit_before>#!/bin/bash

. ../tools/error_handler

trap 'error ${LINENO}' ERR

target=${TARGET-/kb/runtime}

if [[ $# -ne 0 ]] ; then
        target=$1
        shift
fi

IDIR=$target
CURL="http://sourceforge.net/projects/cdbfasta/files/latest/download?source=files"
CBASE="cdbfasta"

echo "###### downloading $CBASE ######"
curl -L $CURL > $CBASE".tar.gz"
tar zxf $CBASE".tar.gz"

echo "###### installing $CBASE ######"
pushd $CBASE
make
cp cdbfasta $IDIR/bin/.
cp cdbyank $IDIR/bin/.
popd
rm $CBASE".tar.gz"
rm -rf $CBASE

<commit_msg>add curl retry<commit_after>#!/bin/bash

. ../tools/error_handler

trap 'error ${LINENO}' ERR

target=${TARGET-/kb/runtime}

if [[ $# -ne 0 ]] ; then
        target=$1
        shift
fi

IDIR=$target
CURL="http://sourceforge.net/projects/cdbfasta/files/latest/download?source=files"
CBASE="cdbfasta"

echo "###### downloading $CBASE ######"
curl --retry 30 -L $CURL > $CBASE".tar.gz"
tar zxf $CBASE".tar.gz"

echo "###### installing $CBASE ######"
pushd $CBASE
make
cp cdbfasta $IDIR/bin/.
cp cdbyank $IDIR/bin/.
popd
rm $CBASE".tar.gz"
rm -rf $CBASE


{'fraction_non_alphanumeric': 0.20733944954128442, 'fraction_numerical': 0.005504587155963303, 'mean_word_length': 3.722943722943723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1051466', 'n_tokens_mistral': 448, 'n_tokens_neox': 429, 'n_words': 94}