<commit_before>import auto_correct as auto
import sys
import StringIO

# queries to train the model
queries = ['how to handle a 1.5 year old when hitting',
 'how can i avoid getting sick in china',
 'how do male penguins survive without eating for four months',
 'how do i remove candle wax from a polar fleece',
 'how do i find an out of print book',
 'yeh query hai']

# train the model

# supressing the output of training model
stdout = sys.stdout
sys.stdout = StringIO.StringIO()

model = auto.auto_correct(re_train=True,data=queries)

# enabling the output standard output
sys.stdout = stdout

# running the model
model.run()
<commit_msg>add data<commit_after>import auto_correct as auto
import sys
import StringIO

# queries to train the model
queries = ['how to handle a 1.5 year old when hitting',
 'how can i avoid getting sick in china',
 'how do male penguins survive without eating for four months',
 'how do i remove candle wax from a polar fleece',
 'how do i find an out of print book',
 'yeh query hai',
 'Aap Kaise Hain',
'I am Fine Main theek Hoon',
'We’ll meet again',
 'See you latter <NAME>',
'What’s your name? Aap ka naam kya hai',
'My Name is <name> <NAME> <name> Hai',
'I am Going Main Ja Raha Hoon, Main Ja Rahi Hoon',
'Where are you going? Aap kahan ja rahe ho?',
'Where are we going? Hum kahan ja rahe hain?',
'How is your family? Aapka parivaar kaisa hai?''
'Nice to meet you Aapse mil kar khushi hu]''

# train the model

# supressing the output of training model
stdout = sys.stdout
sys.stdout = StringIO.StringIO()

model = auto.auto_correct(re_train=True,data=queries)

# enabling the output standard output
sys.stdout = stdout

# running the model
model.run()

{'fraction_non_alphanumeric': 0.08731563421828908, 'fraction_numerical': 0.002359882005899705, 'mean_word_length': 4.3841269841269845, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5458463', 'n_tokens_mistral': 546, 'n_tokens_neox': 535, 'n_words': 256}