<commit_before>#!/bin/bash
set -ex
set -o pipefail

cooccurrenceDir=$1
occurrenceDir=$2
sentenceCountDir=$3

splitYear=$4
outDir=$5

mkdir -p $outDir

cooccurrenceDir=`readlink -f $cooccurrenceDir`
occurrenceDir=`readlink -f $occurrenceDir`
sentenceCountDir=`readlink -f $sentenceCountDir`
outDir=`readlink -f $outDir`

tmpDir=tmp

trainingAndValidationSplit=$splitYear
trainingSplit=$(($splitYear-1))

testSize=1000000
validationSize=$testSize

# Get directory of this script
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

###########################
# Training and Validation #
###########################

mkdir -p $tmpDir/trainingAndValidation/cooccurrences
mkdir -p $tmpDir/trainingAndValidation/occurrences
mkdir -p $tmpDir/trainingAndValidation/sentenceCounts

find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingAndValidationSplit ' { if ( $1<=splitYear) print; } ' > $tmpDir/trainingAndValidation.years
find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingAndValidationSplit ' { if ( $1>splitYear) print; } ' > $tmpDir/test.years

cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $cooccurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.cooccurrences
cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $occurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.occurrences
cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $sentenceCountDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.sentenceCounts

cat $tmpDir/trainingAndValidationFiles.cooccurrences | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/cooccurrences/
cat $tmpDir/trainingAndValidationFiles.occurrences | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/occurrences
cat $tmpDir/trainingAndValidationFiles.sentenceCounts | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/sentenceCounts

bash $HERE/mergeMatrix_2keys.sh tmp/trainingAndValidation/cooccurrences/ $outDir/trainingAndValidation.cooccurrences
bash $HERE/mergeMatrix_1key.sh tmp/trainingAndValidation/occurrences $outDir/trainingAndValidation.occurrences.unfiltered
bash $HERE/mergeMatrix_0keys.sh tmp/trainingAndValidation/sentenceCounts $outDir/trainingAndValidation.sentenceCounts

# Get the list of term IDs that actually occur in cooccurrences
cat $outDir/trainingAndValidation.cooccurrences | cut -f 1,2 -d $'\t' | tr '\t' '\n' | sort -un > $outDir/trainingAndValidation.ids

bash $HERE/filterOccurrences.sh $outDir/trainingAndValidation.occurrences.unfiltered $outDir/trainingAndValidation.ids $outDir/trainingAndValidation.occurrences
rm $outDir/trainingAndValidation.occurrences.unfiltered

python $HERE/checkFilteredOccurrences.py --occurrenceFile $outDir/trainingAndValidation.occurrences --acceptedIDs $outDir/trainingAndValidation.ids

#############
# Training  #
#############

mkdir -p $tmpDir/training/cooccurrences
mkdir -p $tmpDir/training/occurrences
mkdir -p $tmpDir/training/sentenceCounts

find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingSplit ' { if ( $1<=splitYear) print; } ' > $tmpDir/training.years

cat $tmpDir/training.years | xargs -I YEAR echo "find $cooccurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.cooccurrences
cat $tmpDir/training.years | xargs -I YEAR echo "find $occurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.occurrences
cat $tmpDir/training.years | xargs -I YEAR echo "find $sentenceCountDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.sentenceCounts

cat $tmpDir/trainingFiles.cooccurrences | xargs -I FILE ln -s FILE $tmpDir/training/cooccurrences/
cat $tmpDir/trainingFiles.occurrences | xargs -I FILE ln -s FILE $tmpDir/training/occurrences
cat $tmpDir/trainingFiles.sentenceCounts | xargs -I FILE ln -s FILE $tmpDir/training/sentenceCounts

bash $HERE/mergeMatrix_2keys.sh tmp/training/cooccurrences/ $outDir/training.cooccurrences
bash $HERE/mergeMatrix_1key.sh tmp/training/occurrences $outDir/training.occurrences.unfiltered
bash $HERE/mergeMatrix_0keys.sh tmp/training/sentenceCounts $outDir/training.sentenceCounts

# Get the list of term IDs that actually occur in cooccurrences
cat $outDir/training.cooccurrences | cut -f 1,2 -d $'\t' | tr '\t' '\n' | sort -un > $outDir/training.ids

bash $HERE/filterOccurrences.sh $outDir/training.occurrences.unfiltered $outDir/training.ids $outDir/training.occurrences
rm $outDir/training.occurrences.unfiltered

python $HERE/checkFilteredOccurrences.py --occurrenceFile $outDir/training.occurrences --acceptedIDs $outDir/training.ids

##############<commit_msg>Fix wrong working directory used<commit_after>#!/bin/bash
set -ex
set -o pipefail

cooccurrenceDir=$1
occurrenceDir=$2
sentenceCountDir=$3

splitYear=$4
outDir=$5

mkdir -p $outDir

cooccurrenceDir=`readlink -f $cooccurrenceDir`
occurrenceDir=`readlink -f $occurrenceDir`
sentenceCountDir=`readlink -f $sentenceCountDir`
outDir=`readlink -f $outDir`

tmpDir=tmp.$HOSTNAME.$$.$RANDOM
rm -fr $tmpDir

trainingAndValidationSplit=$splitYear
trainingSplit=$(($splitYear-1))

testSize=1000000
validationSize=$testSize

# Get directory of this script
HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

###########################
# Training and Validation #
###########################

mkdir -p $tmpDir/trainingAndValidation/cooccurrences
mkdir -p $tmpDir/trainingAndValidation/occurrences
mkdir -p $tmpDir/trainingAndValidation/sentenceCounts

find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingAndValidationSplit ' { if ( $1<=splitYear) print; } ' > $tmpDir/trainingAndValidation.years
find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingAndValidationSplit ' { if ( $1>splitYear) print; } ' > $tmpDir/test.years

cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $cooccurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.cooccurrences
cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $occurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.occurrences
cat $tmpDir/trainingAndValidation.years | xargs -I YEAR echo "find $sentenceCountDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingAndValidationFiles.sentenceCounts

cat $tmpDir/trainingAndValidationFiles.cooccurrences | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/cooccurrences/
cat $tmpDir/trainingAndValidationFiles.occurrences | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/occurrences
cat $tmpDir/trainingAndValidationFiles.sentenceCounts | xargs -I FILE ln -s FILE $tmpDir/trainingAndValidation/sentenceCounts

bash $HERE/mergeMatrix_2keys.sh $tmpDir/trainingAndValidation/cooccurrences/ $outDir/trainingAndValidation.cooccurrences
bash $HERE/mergeMatrix_1key.sh $tmpDir/trainingAndValidation/occurrences $outDir/trainingAndValidation.occurrences.unfiltered
bash $HERE/mergeMatrix_0keys.sh $tmpDir/trainingAndValidation/sentenceCounts $outDir/trainingAndValidation.sentenceCounts

# Get the list of term IDs that actually occur in cooccurrences
cat $outDir/trainingAndValidation.cooccurrences | cut -f 1,2 -d $'\t' | tr '\t' '\n' | sort -un > $outDir/trainingAndValidation.ids

bash $HERE/filterOccurrences.sh $outDir/trainingAndValidation.occurrences.unfiltered $outDir/trainingAndValidation.ids $outDir/trainingAndValidation.occurrences
rm $outDir/trainingAndValidation.occurrences.unfiltered

python $HERE/checkFilteredOccurrences.py --occurrenceFile $outDir/trainingAndValidation.occurrences --acceptedIDs $outDir/trainingAndValidation.ids

#############
# Training  #
#############

mkdir -p $tmpDir/training/cooccurrences
mkdir -p $tmpDir/training/occurrences
mkdir -p $tmpDir/training/sentenceCounts

find $cooccurrenceDir -type f | xargs -I FILE basename FILE | cut -f 1 -d '.' | sort -n -u | awk -v splitYear=$trainingSplit ' { if ( $1<=splitYear) print; } ' > $tmpDir/training.years

cat $tmpDir/training.years | xargs -I YEAR echo "find $cooccurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.cooccurrences
cat $tmpDir/training.years | xargs -I YEAR echo "find $occurrenceDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.occurrences
cat $tmpDir/training.years | xargs -I YEAR echo "find $sentenceCountDir -type f -name 'YEAR*'" | sh > $tmpDir/trainingFiles.sentenceCounts

cat $tmpDir/trainingFiles.cooccurrences | xargs -I FILE ln -s FILE $tmpDir/training/cooccurrences/
cat $tmpDir/trainingFiles.occurrences | xargs -I FILE ln -s FILE $tmpDir/training/occurrences
cat $tmpDir/trainingFiles.sentenceCounts | xargs -I FILE ln -s FILE $tmpDir/training/sentenceCounts

bash $HERE/mergeMatrix_2keys.sh $tmpDir/training/cooccurrences/ $outDir/training.cooccurrences
bash $HERE/mergeMatrix_1key.sh $tmpDir/training/occurrences $outDir/training.occurrences.unfiltered
bash $HERE/mergeMatrix_0keys.sh $tmpDir/training/sentenceCounts $outDir/training.sentenceCounts

# Get the list of term IDs that actually occur in cooccurrences
cat $outDir/training.cooccurrences | cut -f 1,2 -d $'\t' | tr '\t' '\n' | sort -un > $outDir/training.ids

bash $HERE/filterOccurrences.sh $outDir/training.occurrences.unfiltered $outDir/training.ids $outDir/training.occurrences
rm $outDir/training.occurrences.unfiltered

python $HERE/checkFilteredOccurrences.py --occurrenceFile $outDir/training.occurrences --acceptedIDs $outDir/training.ids

##############
{'fraction_non_alphanumeric': 0.12923619027878536, 'fraction_numerical': 0.0062182609596849415, 'mean_word_length': 5.664364640883978, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2337354', 'n_tokens_mistral': 3352, 'n_tokens_neox': 3034, 'n_words': 743}