<jupyter_start><jupyter_text>NoteThis file is compilation all possible POS tags<jupyter_code># # Rename the files to gold and silver
# import argparse
# import os
# path = './data/train_20k/'
# count = 0
# for file in os.listdir(path):
#         count= count + 1
#         dst ="Silver_" + str(count) + ".csv"
#         src =file
#         os.rename(path+src,path+dst) 
ma_set = set()
word = None
with open('../data/multilingual_word_embeddings/pos_embedding_1_hot.475', 'r', encoding='utf-8') as f:
    # skip first line
    for i, line in enumerate(f):
        if i == 0:
            continue
        word, vec = line.split('@', 1)
        ma_set.add(word)
# ma_set
import re
def clean_ma(ma):
    ma = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []').replace(' ac','').replace(' ps','').replace('sgpl','sg').replace('sgdu','sg')
    ma = ma.replace('i.','inst.').replace('.','').replace(' ','')
    return ma
def IND(ma):
    indeclinable = ['ind','prep','interj','prep','conj','part','abs','ca abs']
    for i in indeclinable:
        if i in ma:
            ma = 'indecl'
    return ma
s = set()
for ma in ma_set:
    s.add(clean_ma(ma))
len(s)
df = pd.read_csv('../../../Documents/DCST/data/4.csv',sep=',')
# df.iloc[:,1]
# temp =[]
# for w in df.iloc[:,1]:
#     temp.append(transliterate(w, sanscript.WX, sanscript.SLP1))
# temp
list(df.iloc[:,1])
def allow(words,temp):
    ct = 0 
    for w,t in zip(words,temp):
        if w==t:
            ct+=1
    if ct == len(words) -1:
        return True
    else:
        return False
import os
import pandas as pd
from indic_transliteration import sanscript
from indic_transliteration.sanscript import SchemeMap, SCHEMES, transliterate
data = pd.read_csv('../data/sanskrit_treebank/train/Gold_633.csv',sep=',')
def get_gold_POS_Tag(words):
    kautilya_id ='NA'
    path = '../../../Documents/DCST/data/4k_data_my_prepocessing/'
    files = os.listdir(path)
    gold_pos= ['NA']*len(words)
    for file in files:
        df = pd.read_csv(path+file,sep='\t',header = None)
        temp =[]
        if len(words) != len(df):
            continue
        for w in df.iloc[:,1]:
            temp.append(transliterate(w, sanscript.WX, sanscript.SLP1))
        words = [x.lower() for x in words]
        temp = [x.lower() for x in temp]
        if words == temp:
            kautilya_id = file
            for u,g in enumerate(df.iloc[:,2]):
                if g == g:
                    gold_pos[u] = g
            break
        elif allow(words,temp):
            kautilya_id = file
            for u,g in enumerate(df.iloc[:,2]):
                if g == g:
                    gold_pos[u] = g
        else:
            continue
            
    return gold_pos,kautilya_id
get_gold_POS_Tag(list(df.iloc[:,1]))
        
    
transliterate('nandigrAmam', sanscript.WX, sanscript.SLP1)
import argparse
import os
path = '../data/sanskrit_treebank/'
languages_for_low_resource = ['san']
languages = sorted(list(set(languages_for_low_resource)))
#
splits = ['train','dev','test']
lng_to_files = dict((language, {}) for language in languages)
for language, d in lng_to_files.items():
    for split in splits:
        d[split] = []
    lng_to_files[language] = d
sub_folders = os.listdir(path)
for sub_folder in sub_folders:
    folder = os.path.join(path, sub_folder)
    files = sorted(os.listdir(folder))
    for file in files:
        full_path = os.path.join(folder, file)
        lng_to_files[language][sub_folder].append(full_path)
###################################
# # # # Add silver in train:
# counter = 0
# amount = 2000
# for f in os.listdir('../data/train_20k/'):
#     full_path = os.path.join('../data/train_20k/', f)
#     lng_to_files[language]['train'].append(full_path)
#     counter += 1
# #     if counter >= amount:
# #         break
    
word_set = set()
for language, split_dict in lng_to_files.items():
    posi =set()
    for split, files in split_dict.items():
        sentences = []
        num_sentences = 0
        for file in sorted(files):
#             if split =='train':
#                 print(file)
            
            flag = 0
            data = pd.read_csv(file,sep=',')
            gold_pos,kautilya_id = get_gold_POS_Tag(list(data.iloc[:,1]))
            if kautilya_id == 'NA':
                continue
            num_sentences += 1
            k=0
            with open(file, 'r') as file:
                for line in file:
                    new_line = []
                    line = line.strip()
#                     print(line)
                    if flag == 0:
                        flag = 1
                        continue
                    tokens = line.split(',')
                    id = str(int(tokens[0]) + 1)
                    word = tokens[1]
                    word_set.add(word)
                    ############################
                    # To add rule features id of rule is given
                    my_data_id = file.name.replace('../data/sanskrit_treebank/','').replace('.csv','')\
                    .replace('../data/train_20k/','').replace('train/','').replace('dev/','').replace('test/','')
                    lemma = tokens[2]
                    pos = tokens[3]
#                     tag.add(pos)
                    if tokens[6]:
                        arc_tag = tokens[5]
                        head = str(int(tokens[6]) + 1)
                    else:
                        arc_tag = 'root'
                        head = '0'
                    
                    new_line = [id, word ,pos,get_modified_coarse(pos),gold_pos[k],my_data_id,head, arc_tag,kautilya_id]
                    k+=1
                    posi.add(pos)
#                     print(new_line)
                    sentences.append(new_line)
            sentences.append([])
            print(num_sentences)
#         ud_pos_ner_dp_
        print('Language: %s Split: %s Num. Sentences: %s ' % (language, split, num_sentences))
        if not os.path.exists('data'):
            os.makedirs('data')
#         ud_pos_ner_dp_
        write_data_path = '../../../DCST_scratch/data/data_with_gold_pos_' + split + '_' + language
        print('creating %s' % write_data_path)
        with open(write_data_path, 'w') as f:
            for line in sentences:
                f.write('\t'.join(line) + '\n')
def get_case_from_244(ma):
    indeclinable = ['ind','prep','interj','prep','conj','part','indecl']
    case_list = ['nom','voc','acc','inst','dat','abl','loc','i','g']
    gender_list = ['n','f','m','*']
    person_list = ['1','2','3']
    no_list = ['du','sg','pl']
    ma=ma.replace('sgpl','sg').replace('sgdu','sg')
    # Remove active passive
    case=''
    if ma == 'comp':
        case ='comp'
    for tag in indeclinable:
        if tag in ma:
            case= "IND"

    # Get case
    if case =='':
        for tag in case_list:
            if tag in ma:
                if tag == 'i':
                    if tag+'sg' in ma or tag+'du' in ma or tag+'pl' in ma:
                        case = 'i'
                elif tag == 'g':
                    if tag+'sg' in ma or tag+'du' in ma or tag+'pl' in ma:
                        case = 'g'
                else:
                    case = tag
    if case == '':
        if 'adv' in ma:
            case = 'adv'
    
    if case =='':
        for tag in no_list:
            if tag in ma:
                case = 'FV'
    if case=='':
        for tag in no_list:
            if tag in ma:
                case = 'FV'
    if case=='':
         for tag in person_list:
            if tag in ma:
                case = 'FV'
    if case=='':
        case='IV'
        
    return case
s = []
for m in ma_set:
    s.append([m,get_case_from_244(m)])
import json
indeclinable = ['ind','prep','interj','prep','conj','part','abs','ca abs']
case_list = ['nom','voc','acc','i','inst','dat','abl','g','loc']
gender_list = ['n','f','m','*']
person_list = ['1','2','3']
no_list = ['du','sg','pl']
pops = [' ac',' ps']
def get_modified_coarse(ma):
    ma = ma.replace('sgpl','sg').replace('sgdu','sg')
    with open('../../L2S/files/coarse_to_ma_dict.json', 'r') as fh:
        coarse_dict = json.load(fh)
    for key in coarse_dict.keys():
        if ma in coarse_dict[key]:
            return key
s=[]
for m in ma_set:
    s.append([m,get_modified_coarse(m)])
sorted(s)
print(get_modified_coarse('acc. pl. m'))
import re
def get_case(ma):
        indeclinable = ['ind','prep','interj','prep','conj','part']
        case_list = ['nom','voc','acc','i','inst','dat','abl','g','loc']
        gender_list = ['n','f','m','*']
        person_list = ['1','2','3']
        no_list = ['du','sg','pl']
        pops = [' ac',' ps']
        ma=ma.replace('sgpl','sg').replace('sgdu','sg')
        temp = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []')
        temp = temp.split('.')
        if temp[-1] == '':
            temp.pop(-1)
        # Remove active passive
        case=''
        no=''
        person=''
        gender=''
        tense=''
        coarse=''
        for a,b in enumerate(temp):
            if b in pops:
                temp.pop(a)
        # Get gender
        for a,b in enumerate(temp):
            if b.strip() in gender_list:
                gender = b.strip()
                temp.pop(a)
        # Get case
        for a,b in enumerate(temp):
            if b.strip() in case_list:
                case = b.strip()
                temp.pop(a)
        if case!= '':
            coarse ='Noun'
        # Get person
        for a,b in enumerate(temp):
            if b.strip() in person_list:
                person = b.strip()
                temp.pop(a)
        # Get no
        for a,b in enumerate(temp):
            if b.strip() in no_list:
                no = b.strip()
                temp.pop(a)
        # Get Tense
        for b in temp:
            tense=tense+ ' '+b.strip()
        tense=tense.strip()
        
#         print(tense)
        if tense == 'adv':
            coarse = 'adv'
        for ind in indeclinable:
            if tense == ind:
                coarse = 'Ind'
        if tense == 'abs' or tense == 'ca abs':
            coarse = 'IV'
        if tense!='' and coarse=='':
            if person !='' or no!='':
                coarse= 'FV'
            else:
                coarse = 'IV'
        if case == 'i':
            return 'inst'
        
        if case !='':
            return case
        else:
            return coarse
def variation_III(ma):
    ##############################################
    # Description:
    # (1) Active passive is removed
    # (2) Tense [*] is removed
    ma=ma.replace('sgpl','sg').replace('sgdu','sg')
    temp = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []')
    temp = temp.split('.')
    if temp[-1] == '':
        temp.pop(-1)
    # Remove active passive
    case=''
    no=''
    person=''
    gender=''
    tense=''

    for a,b in enumerate(temp):
        if b in pops:
            temp.pop(a)
    # Get gender
    for a,b in enumerate(temp):
        if b.strip() in gender_list:
            gender = b.strip()
            temp.pop(a)
    # Get case
    for a,b in enumerate(temp):
        if b.strip() in case_list:
            case = b.strip()
            temp.pop(a)
#     if case == 'i':
#         case = 'inst'
    # Get person
    for a,b in enumerate(temp):
        if b.strip() in person_list:
            person = b.strip()
            temp.pop(a)
    # Get no
    for a,b in enumerate(temp):
        if b.strip() in no_list:
            no = b.strip()
            temp.pop(a)
    # Get Tense
    for b in temp:
        tense=tense+ ' '+b.strip()
    tense=tense.strip()
    var=''

    if case:
        t = 'case=%s '%case
        var=var+t
    if gender:
        t = 'gender=%s '%gender
        var=var+t
    if tense:
        t = 'tense=%s '%tense
        var=var+t
    if no:
        t = 'no=%s '%no
        var=var+t
    if person:
        t = 'person=%s '%person
        var=var+t
    return var.strip()
variation_III('pp. . sg. ')

import re
def variation_IV(ma):
    ma=ma.replace('sgpl','sg').replace('sgdu','sg')
    temp = re.sub("([\(\[]).*?([\)\]])", "\g<1>\g<2>", ma).replace('[] ','').strip(' []')
    temp = temp.split('.')
    if temp[-1] == '':
        temp.pop(-1)
    # Remove active passive
    case=''
    no=''
    person=''
    gender=''
    tense=''
    coarse=''
    for a,b in enumerate(temp):
        if b in pops:
            temp.pop(a)
    # Get gender
    for a,b in enumerate(temp):
        if b.strip() in gender_list:
            gender = b.strip()
            temp.pop(a)
    # Get case
    for a,b in enumerate(temp):
        if b.strip() in case_list:
            case = b.strip()
            temp.pop(a)
    if case!= '':
        coarse ='Noun'
#     if case == 'i':
#         case = 'inst'
    # Get person
    for a,b in enumerate(temp):
        if b.strip() in person_list:
            person = b.strip()
            temp.pop(a)
    # Get no
    for a,b in enumerate(temp):
        if b.strip() in no_list:
            no = b.strip()
            temp.pop(a)
    # Get Tense

    for b in temp:
        tense=tense+ ' '+b.strip()
    tense=tense.strip()
    if tense == 'adv':
        coarse = 'adv'
        
    for ind in indeclinable:
        if tense == ind:
            coarse = 'Ind'

    if tense!='' and coarse=='':
        if person !='' and no!='':
            coarse= 'FV'
        else:
            coarse = 'IV'
    var=''
    if case:
        t = 'case=%s '%case
        var=var+t
    if gender:
        t = 'gender=%s '%gender
        var=var+t
    if tense:
        t = 'tense=%s '%tense
        var=var+t
    if no:
        t = 'no=%s '%no
        var=var+t
    if person:
        t = 'person=%s '%person
        var=var+t
    coarse = get_modified_coarse(ma)
    if coarse:
        t = 'coarse=%s '%coarse
        var=var+t
    return var.strip();
l = []
# for ma in ma_set:
#     l.append(ma+' = '+variation_IV(ma))
variation_IV('pp. . sg. ')
# onehot_encoded[0]
count =0 
with open("pos_embedding_1_hot.475", "w") as f:   
    f.write(str(len(tag))+' '+str(475)+'\n')
    for word in tag:
        word=word+'@'
        f.write(word) 
        for v in onehot_encoded[count]:
            f.write(' '+str(v))
        f.write('\n')
        count = count + 1
print(count)<jupyter_output>475
{'fraction_non_alphanumeric': 0.17335462371563454, 'fraction_numerical': 0.007220216606498195, 'mean_word_length': 1.7082158300432413, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '63884', 'license': 'MIT', 'n_tokens_mistral': 4934, 'n_tokens_neox': 4786, 'n_words': 1135, 'path': 'notebooks/Prepare_gold_data_POS_tags.ipynb', 'repo_name': 'krishnamrith12/DCST', 'size': '14404'}