<commit_before>#!/usr/bin/env bash
if [ $# -ne 2 ]
then 
    echo "usage: update-resource.sh <username> <branchname>"
    exit 0
fi
if [ "$2" != "hawk" ]
then
    echo 'now we just support hawk branch.'
    exit 0
fi
home=`dirname "$0"`
home=`cd "$home"; pwd`

datestr=`date +%Y%m%d%H%M%S`
cp -r "$home"/resource "$home/resource-$datestr"

rsync -avP "$1"<EMAIL>:/data/sf1r-resource/"$2"/resource/nec/ "$home"/resource/nec/
rsync -avP "$1"@<EMAIL>:/data/sf1r-resource/"$2"/resource/kpe/ "$home"/resource/kpe/
<commit_msg>a little<commit_after>#!/usr/bin/env bash
if [ $# -ne 2 ]
then 
    echo "usage: update-resource.sh <username> <branchname>"
    exit 0
fi
if [ "$2" != "hawk" ]
then
    echo 'now we just support hawk branch.'
    exit 0
fi
home=`dirname "$0"`
home=`cd "$home"; pwd`

datestr=`date +%Y%m%d%H%M%S`

if [ -d "$home/resource" ]
then
    a=1
else
    mkdir "$home/resource"
fi

cp -r "$home"/resource "$home/resource-$datestr"

rsync -avP "$1"<EMAIL>:/data/sf1r-resource/"$2"/resource/nec/ "$home"/resource/nec/
rsync -avP "$1"@<EMAIL>:/data/sf1r-resource/"$2"/resource/kpe/ "$home"/resource/kpe/

{'fraction_non_alphanumeric': 0.21723518850987433, 'fraction_numerical': 0.02064631956912029, 'mean_word_length': 3.7245762711864407, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1956646', 'n_tokens_mistral': 493, 'n_tokens_neox': 467, 'n_words': 104}