<commit_before>#!/bin/bash

################################################################################################################################################################
# FLIMerge.sh - this script does something awful.
# By <NAME>
# 10/2013 v3
#
#  this script parses files and batches a dodgy Perl script
# 
################################################################################################################################################################



# fix file name's first!
#convert data files spaces to _/underscore if you have weird Windows file names etc.
cd ./data ;  
ls *.asc | while read -r FILE  
do
    mv -v "$FILE" `echo $FILE | tr ' ' '_' `
done


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# convert all *.asc (data files) to files where 0 is replaced with a dash/-)
for i in *.asc ; do
perl -pi -e 's/\s0\s/\ \-\n/g' $i ;
done

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# dummy run. Use the x-values from the first file in the directory as a guide
FILE=$(ls | sort -n | head -1)
cp $FILE ../input.txt ; 
# \rm `ls | sort -n | head -1` 
cd ../ ;
# add header
perl -pi -le 'print "PixelID dummy" if $. == 1' input.txt ;   #  ; # Add this line to the top\n";
# Dummy run
perl mergefiles.pl input.txt input.txt ;
cp largerfile.txt ./data ;
rm ./temp.txt ; # get rid of your temp files
rm ./largerfile.txt ; # get rid of your temp files
cd ./data ;

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# run the script on all your datafiles 
for j in *.asc ; do
perl ../mergefiles.pl largerfile.txt $j ;
rm ./temp.txt ; # get rid of your temp files
done
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# get rid of the dummy columns
cut -f1,4- ./largerfile.txt >../output.csv ;  # column 1 and 2 contain "dummy" 
rm ./largerfile.txt ; # get rid of your temp files
rm ../input.txt ;
# use the unix command cut for this
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# convert dashes back to zero
perl -pi -e 's/-/0/g' ../output.csv ;
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# unnecessary echo here 
clear ;
echo -en '\E[47;31m'"\033[1mall done! Your output will be in the tabulated file output.csv \033[0m"   # Red
echo -e "\e[00;37m" ;

echo '
   ,---.
   CO-|| Y  
   O  || |
   |   D J
  // /"""'
<commit_msg>Update mergeorama.sh<commit_after>#!/bin/bash

################################################################################################################################################################
# FLIMerge.sh - this script does something awful.
# By <NAME>
# 10/2013 v3
#
#  this script parses files and batches a dodgy Perl script
# 
################################################################################################################################################################



# fix file name's first!
#convert data files spaces to _/underscore if you have weird Windows file names etc.
cd ./data ;  
ls *.asc | while read -r FILE  
do
    mv -v "$FILE" `echo $FILE | tr ' ' '_' `
done


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# convert all *.asc (data files) to files where 0 is replaced with a dash/-)
for i in *.asc ; do
perl -pi -e 's/\s0\s/\ \-\n/g' $i ;
done

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# dummy run. Use the x-values from the first file in the directory as a guide
FILE=$(ls | sort -n | head -1)
cp $FILE ../input.txt ; 
# \rm `ls | sort -n | head -1` 
cd ../ ;
# add header
perl -pi -le 'print "PixelID dummy" if $. == 1' input.txt ;   #  ; # Add this line to the top\n";
# Dummy run
perl FLIMerge.pl input.txt input.txt ;
cp largerfile.txt ./data ;
rm ./temp.txt ; # get rid of your temp files
rm ./largerfile.txt ; # get rid of your temp files
cd ./data ;

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# run the script on all your datafiles 
for j in *.asc ; do
perl ../FLIMerge.pl largerfile.txt $j ;
rm ./temp.txt ; # get rid of your temp files
done
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# get rid of the dummy columns
cut -f1,4- ./largerfile.txt >../output.csv ;  # column 1 and 2 contain "dummy" 
rm ./largerfile.txt ; # get rid of your temp files
rm ../input.txt ;
# use the unix command cut for this
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# convert dashes back to zero
perl -pi -e 's/-/0/g' ../output.csv ;
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~`
# unnecessary echo here 
clear ;
echo -en '\E[47;31m'"\033[1mall done! Your output will be in the tabulated file output.csv \033[0m"   # Red
echo -e "\e[00;37m" ;

echo '
   ,---.
   CO-|| Y  
   O  || |
   |   D J
  // /"""'

{'fraction_non_alphanumeric': 0.4805799176660104, 'fraction_numerical': 0.011813137640952211, 'mean_word_length': 4.582417582417582, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1312101', 'n_tokens_mistral': 1606, 'n_tokens_neox': 1425, 'n_words': 503}