<commit_before>echo "ORIGIN,LABO,BENEF_PS_QUALITE_NOM_PRENOM,BENEF_PS_ADR,BENEF_PS_QUALIFICATION,BENEF_PS_RPPS,DECL_CONV_DATE,DECL_CONV_OBJET,DECL_CONV_PROGRAMME,DECL_AVANT_MONTANT,DECL_AVANT_DATE,DECL_AVANT_NATURE,BENEF_ETUD_ETA,BENEF_PS_CODEPOSTAL," > data/formatted/transparencesante_avantages.formatted.csv
#entreprise,type_beneficiaire,beneficiaire,date,nature,montant,code_postal_beneficaire
cat data/raw/transparencesante_avantages.csv | sed 's/\r//' | sed 1d | iconv -f utf8 -t utf8//IGNORE | perl -e 'while(<STDIN>){chomp; while(s/"([^"]*),([^"]*)"/"\1 -\2"/g){}; @a = split /,/; @s = split /\//, $a[3] ; $a[3] = $s[2]."-".$s[1]."-".$s[0]; print join ",", @a; print "\n"; }' | awk -F ',' '{print $2","$1","$3","$7","$2",,,,,"$6","$4","$5","$7}' | sed 's/  */ /g' | sed 's/, /,/g' | sed 's/ ,/,/g' >> data/formatted/transparencesante_avantages.formatted.csv


#sed "s/Forfait *journée d[^é]*étude/Forfait journée d'étude/" | sed "s/FRAIS D'HOSPITALIT[^,]*,/FRAIS D'HOSPITALITE,/" | sed "s/’/'/g" | sed 's/ÃƒÂ©/e/g' | sed 's/MAZET Anne-Aur[^,]lie/MAZET Anne-Aurelie/'
<commit_msg>useless comments<commit_after>echo "ORIGIN,LABO,BENEF_PS_QUALITE_NOM_PRENOM,BENEF_PS_ADR,BENEF_PS_QUALIFICATION,BENEF_PS_RPPS,DECL_CONV_DATE,DECL_CONV_OBJET,DECL_CONV_PROGRAMME,DECL_AVANT_MONTANT,DECL_AVANT_DATE,DECL_AVANT_NATURE,BENEF_ETUD_ETA,BENEF_PS_CODEPOSTAL," > data/formatted/transparencesante_avantages.formatted.csv
#entreprise,type_beneficiaire,beneficiaire,date,nature,montant,code_postal_beneficaire
cat data/raw/transparencesante_avantages.csv | sed 's/\r//' | sed 1d | iconv -f utf8 -t utf8//IGNORE | perl -e 'while(<STDIN>){chomp; while(s/"([^"]*),([^"]*)"/"\1 -\2"/g){}; @a = split /,/; @s = split /\//, $a[3] ; $a[3] = $s[2]."-".$s[1]."-".$s[0]; print join ",", @a; print "\n"; }' | awk -F ',' '{print $2","$1","$3","$7","$2",,,,,"$6","$4","$5","$7}' | sed 's/  */ /g' | sed 's/, /,/g' | sed 's/ ,/,/g' >> data/formatted/transparencesante_avantages.formatted.csv

{'fraction_non_alphanumeric': 0.24491869918699186, 'fraction_numerical': 0.019308943089430895, 'mean_word_length': 6.16, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '405112', 'n_tokens_mistral': 957, 'n_tokens_neox': 923, 'n_words': 111}