<commit_before>#!/bin/bash

egrep -i 'How do you say.+in Italian\]\]' newpages.txt >> EnglishtoItalian.txt
egrep -i 'How do you spell.+in Italian\]\]' newpages.txt >> EnglishtoItalian.txt
egrep -i '[abcdefghijklmopqrstuvwxyz]{1,} in Italian' newpages.txt >> EnglishtoItalian.txt
egrep -i 'What is the Italian word for' newpages.txt >> EnglishtoItalian.txt
egrep -i 'How to say.+in Italian' newpages.txt >> EnglishtoItalian.txt

ENGLISHTOITALIAN=`stat --print=%s EnglishtoItalian.txt`

if [ $ENGLISHTOITALIAN -ne 0 ];
then
  export CATFILE="EnglishtoItalian.txt"
  export CATNAME="English to Italian"
  $CATEGORIZE
fi

rm EnglishtoItalian.txt
<commit_msg>Make English to Italian slightly more efficient<commit_after>#!/bin/bash

egrep -i 'How (to|do you) (say|spell).+in Italian' newpages.txt >> EnglishtoItalian.txt
egrep -i '[abcdefghijklmopqrstuvwxyz]{1,} in Italian' newpages.txt >> EnglishtoItalian.txt
egrep -i "What('s| is) the Italian word for" newpages.txt >> EnglishtoItalian.txt
egrep -i 'How to say.+in Italian' newpages.txt >> EnglishtoItalian.txt

ENGLISHTOITALIAN=`stat --print=%s EnglishtoItalian.txt`

if [ $ENGLISHTOITALIAN -ne 0 ];
then
  export CATFILE="EnglishtoItalian.txt"
  export CATNAME="English to Italian"
  $CATEGORIZE
fi

rm EnglishtoItalian.txt

{'fraction_non_alphanumeric': 0.12470588235294118, 'fraction_numerical': 0.003137254901960784, 'mean_word_length': 5.164251207729468, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6990689', 'n_tokens_mistral': 552, 'n_tokens_neox': 473, 'n_words': 121}