<jupyter_start><jupyter_text>Used for ceating a cleanset of Monolingual words to used for sanitization of predictions.Data can be of text corpus or word list, and script cleans words with diffenrent language unicodes.Objective is distillas much as possible to purest words yest maintaining a larger word set<jupyter_code>scripts = {'ऄ','अ', 'आ', 'इ', 'ई', 'उ', 'ऊ','ऍ', 'ऎ', 'ए', 'ऐ',
    'ऑ', 'ऒ', 'ओ', 'औ','ऋ','ॠ','ऌ','ॡ','ॲ', 'ॐ',
    'क', 'ख', 'ग', 'घ', 'ङ', 'च', 'छ', 'ज', 'झ', 'ञ', 'ट', 'ठ', 'ड', 'ढ', 'ण',
    'त', 'थ', 'द', 'ध', 'न', 'ऩ', 'प', 'फ', 'ब', 'भ', 'म', 'य', 'र', 'ऱ', 'ल',
    'ळ', 'ऴ', 'व', 'श', 'ष', 'स', 'ह', 'क़', 'ख़', 'ग़', 'ज़', 'ड़', 'ढ़', 'फ़', 'य़',
    '्', 'ा', 'ि', 'ी', 'ु', 'ू', 'ॅ', 'ॆ', 'े', 'ै', 'ॉ', 'ॊ', 'ो', 'ौ',
    'ृ', 'ॄ', 'ॢ', 'ॣ', 'ँ', 'ं', 'ः', '़', '॑',  'ऽ',

    chr(0x200c), # ZeroWidth-NonJoiner U+200c
    chr(0x200d), # ZeroWidthJoiner U+200d
}
## Removal Set

Spl_puncs = [chr(i) for i in range(0x2000,0x208F) ]
Latin_chars = [chr(i) for i in range(0x0020,0x007F)]
Latin_supli_exten = [chr(i) for i in range(0x00A1,0x017F)]
dgri_num = [chr(i) for i in range(0x0966, 0x0970)]
Other_chars = ["\n",'×', "॥", '।', '॰']

remove_chars_set = Latin_chars + Spl_puncs + Latin_supli_exten + dgri_num + Other_chars 
print(remove_chars_set)<jupyter_output><empty_output><jupyter_text>From **Mono Corpus**  Reading Files<jupyter_code>##Directly from TXT
with open("/content/monolingual/monolingual.hi") as f:
    raw_data = f.read()<jupyter_output><empty_output><jupyter_text>Cleaning the text<jupyter_code>clean_data = raw_data
for c in remove_chars_set:
    clean_data = clean_data.replace(c, " ")

word_tks = clean_data.split()
#print(word_tks) 
print(len(word_tks))
word_dict = dict()
for w in word_tks:
    v = word_dict.get(w)
    if v:
        word_dict[w] = v+1
    else:
        word_dict[w] = 1
# print(word_dict)
print(len(word_dict))
## Check if the characters are devanagari
new_set = set(word_dict)
removal_word_set = set()
for n in new_set:
    for c in set(n):
        if c in scripts:
            continue
        else:
            print(n)
            removal_word_set.add(n)
            break
print("Removal set", len(removal_word_set))
for k in removal_word_set:
    word_dict.pop(k)<jupyter_output><empty_output><jupyter_text>Save data<jupyter_code>import pandas as pd
import csv
data_items = word_dict.items()
# data_list = list(data_items)
df = pd.DataFrame(data_items, columns =["WORD", "FREQ"] )
df = df.sort_values(["FREQ"], ascending= False, ignore_index =True)

df.to_csv('out_data.csv', index=False, quoting=csv.QUOTE_NONE)<jupyter_output><empty_output><jupyter_text>From **Vocab Frequency**AI4Bharat based  Read the data<jupyter_code>with open("/content/mr.vocabfreq.tsv") as f:
    data = f.readlines()

## Take only Frequency greater than 1
import re
word_list = []
for d in data:
    y = re.split("\t|\n", d)
    if int(y[1]) > 1:
        word_list.append(y[0])<jupyter_output><empty_output><jupyter_text>Clean data<jupyter_code>new_set = set(word_list)
removal_word_set = set()
for n in new_set:
    for c in set(n):
        if c in scripts:
            continue
        else:
            print(n)
            removal_word_set.add(n)
            break
print("Removal set", len(removal_word_set))
word_set = set(word_list)
for k in removal_word_set:
    word_set.remove(k)

word_list = list(word_set)<jupyter_output><empty_output><jupyter_text>Save data<jupyter_code>print(len(word_list))

word_list = sorted(word_list)
import json
with open("out_words.json" ,"w", encoding = "utf-8") as f:
    json.dump(word_list, f, ensure_ascii=False, indent=4, sort_keys=True,)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1851447296559257, 'fraction_numerical': 0.016111414527580557, 'mean_word_length': 4.347445255474453, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '308333', 'license': 'Apache-2.0', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1571, 'n_words': 354, 'path': 'tools/vocab_n_embeds/Monolingual_Vocab_creator.ipynb', 'repo_name': 'harisankarh/IndianNLP-Transliteration', 'size': '3662'}