<jupyter_start><jupyter_text>Preparing data for training<jupyter_code>pp = Preprocess([])

print('Loading Data\n\n')
x_train = pd.read_csv('../Data/training_data_marathi.csv')['text'].apply(lambda x : pp.clean_text(x))
y_train = pd.read_csv('../Data/training_data_marathi.csv')['label']
x_val = pd.read_csv('../Data/test_data_marathi.csv')['text'].apply(lambda x : pp.clean_text(x))
y_val = pd.read_csv('../Data/test_data_marathi.csv')['label']


print('Encoding Labels\n\n')
y_train, y_val, label_mappings = label_encoder(y_train, y_val)
y_train = y_train.reshape(y_train.shape[0], -1)
y_val = y_val.reshape(y_val.shape[0], -1)
print(label_mappings)
 

print('Building Vocab\n\n')
pad_len=100
padding_type='post'
truncating_type='post'

corpus = []

for text in x_train:
    corpus.append(text)

for text in x_val:
    corpus.append(text)
    

print('Tokenizing Train and Test\n\n')
tokenizer, x_train_tokenzied, x_val_tokenzied = tokenize_text(corpus,x_train, x_val)

print('Padding Train and Test\n\n')
x_train_padded, x_val_padded = pad_text(x_train_tokenzied, 
                                        x_val_tokenzied, 
                                        pad_len, 
                                        padding_type, 
                                        truncating_type)<jupyter_output>Loading Data


Encoding Labels<jupyter_text>Predicting results<jupyter_code>model = model_load('../models/FINALS/LSTM_combined_embeddings.h5.h5')
results = np.argmax(model.predict(x_val_padded),axis = -1)
del(model)
wc, yact, yhat = perf_measure(y_val,results)
mistakes = pd.DataFrame(columns=['text','actual label','predicted label'])
mistakes['text'] = x_val.loc[wc]
mistakes['actual label'] = yact
mistakes['predicted label'] = yhat
mistakes['actual label'] = mistakes['actual label'].apply(lambda x : x[0])<jupyter_output><empty_output><jupyter_text>Classification Analysis<jupyter_code>print('Total classifaction mistakes: ',mistakes.shape[0])
heatmap = generate_pairwise_heatmap(mistakes, 'actual label', 'predicted label',
                                    label_names= label_mappings.keys())
mistakes['word count'] = mistakes['text'].apply(lambda x : len(x.split()))
print('Word count stats:\n\n',mistakes['word count'].describe(),'\n\n')

mistakes['word count'].iplot(kind = 'hist', xTitle = "word count", yTitle = 'num records', 
                            title = "Histogram depicting distibution of word count across misclassified data" ,bins=(100))
mistakes['actual label'] = mistakes['actual label'].apply(lambda x: label_convert(x,label_mappings))
mistakes['predicted label'] = mistakes['predicted label'].apply(lambda x: label_convert(x,label_mappings))
print('Misclassified class frequency')
mistakes['actual label'].value_counts()
print('Misclassification class frequency')

mistakes['predicted label'].value_counts()
below_20 = [x  for x in mistakes[mistakes['word count'] <= 20 ].values.tolist()]
for rec in below_20:
    print(rec, '\n\n', translator.translate(rec[0]).text,'\n\n')
above_50 = [x  for x in mistakes[mistakes['word count'] > 50].values.tolist()]
for rec in above_50:
    print(rec, '\n\n', translator.translate(rec[0]).text,'\n\n')<jupyter_output>['म्हणून आपल्याला येथे पहाण्याची कोणतीही विशिष्ट ऑर्डर नाही जी आपल्यास जे आवडेल ते आपल्या अनुप्रयोगावर खरोखरच अवलंबून असते आणि म्हणूनच मी म्हणेन की आपल्याला माहित आहे की या दिवसात जेव्हा ते आपल्याला पाहतात तेव्हा आपल्याला छंद क्लब क्रियाकलाप माहित असतात आपल्या विशिष्ट गरजा पूर्ण करणाऱ्या काही गोष्टी निवडण्याची लवचिकता असलेल्या विद्यापीठांमध्ये आपल्याकडे आहे', 'phy', 'com_tech', 53] 

 So there’s no specific order to watch here that really depends on your application of what you like and that’s why I would say you know these days when they see you you know the hobby club activities with the flexibility to choose something that meets your specific needs We have it in universities 


['कारण आज आपल्या आयुष्यात सामान्यतः प्रकाश शीर्षस्थानावरून येतो आणि जर प्रकाश शीर्षस्थानापासून आला आणि जर एखादे वस्तु उत्परिवर्तनीय असेल तर छाया सावलीच्या पृष्ठभागाच्या खालच्या भागावर असेल आणि जर तो अव्यवस्थित असेल तर प्रकाश प्रकाश खालच्या भागात असेल आणि सावली वरच्या भागावर आणि उत्तल पृष्ठभागावर असेल तर प्रका[...]<jupyter_text>Document Similarity between misclassified classes<jupyter_code>bioche = mistakes.loc[mistakes['actual label'] == 'bioche']
phy =  mistakes.loc[mistakes['actual label'] == 'phy']
cse =  mistakes.loc[mistakes['actual label'] == 'cse']
com_tech =  mistakes.loc[mistakes['actual label'] == 'com_tech']
v_bioche = get_vocab(bioche.text.values.tolist())
v_phy = get_vocab(phy.text.values.tolist())
v_cse = get_vocab(cse.text.values.tolist())
v_com_tech = get_vocab(com_tech.text.values.tolist())
print('Bioche:\n\n Physics: {} \n CSE: {} \n Com_tech: {}'.format(len(v_bioche.intersection(v_phy)),len(v_bioche.intersection(v_cse)),
      len(v_bioche.intersection(v_com_tech))))
print('Physics:\n\n Bioche: {} \n CSE: {} \n Com_tech: {}'.format(len(v_phy.intersection(v_bioche)),len(v_phy.intersection(v_cse)),
      len(v_phy.intersection(v_com_tech))))
print('CSE :\n\n Physics: {} \n Bioche: {} \n Com_tech: {}'.format(len(v_cse.intersection(v_phy)),len(v_cse.intersection(v_bioche)),
      len(v_cse.intersection(v_com_tech))))
print('Com_tech:\n\n Physics: {} \n CSE: {} \n Biology: {}'.format(len(v_com_tech.intersection(v_phy)),len(v_com_tech.intersection(v_cse)),
      len(v_com_tech.intersection(v_bioche))))<jupyter_output>Com_tech:

 Physics: 250 
 CSE: 216 
 Biology: 218<jupyter_text>Classification analysis of different models for Marathi<jupyter_code>model1 = pd.read_csv('../Data/Test/set2/marathi-embedding-LDA-attention-predictions.tsv', sep = '\t')
model2 = pd.read_csv('../Data/Test/set2/marathi-multi-channel-CNN-BiLSTM-attention-BiLSTM-then-CNN-predictions.tsv', sep = '\t')
model3 = pd.read_csv('../Data/Test/set2/marathi-multi-channel-CNN-BiLSTM-attention-parallel-predictions.tsv', sep = '\t')
difference1 = pd.concat([model2,model3]).drop_duplicates(keep=False)
len(difference1)
differences = []
cnt = 0
for text1, label1, text2, label2,label3, text3 in zip(model1.text.values.tolist(),model1.predicted_label.values.tolist(),
                                        model2.text.values.tolist(),model2.predicted_label.values.tolist(),
                                                      model3.text.values.tolist(),model3.predicted_label.values.tolist()):
    
    if label1 != label2 or label2 != label3 or label1 != label3:
        differences.append([text1,label1, label2, label3])<jupyter_output><empty_output><jupyter_text>*Divergence of Vocabs*<jupyter_code>data = pd.read_csv('../Data/training_data_marathi.csv')
data['text'] = data['text'].apply(lambda x : pp.clean_text(x))
data.label.value_counts()
vocab = set()
for text in data['text']:
    for token in text.split():
        vocab.add(token)
print(len(vocab))
vocab = list(vocab)

words = [y for x in data['text'] for y in x.split()]
freq = defaultdict(int)
for word in words:
    freq[word] += 1
phy = data[data.label == 'phy'].text.values.tolist()
bioche = data[data.label == 'bioche'].text.values.tolist()
cse = data[data.label == 'cse'].text.values.tolist()
com_tech = data[data.label == 'com_tech'].text.values.tolist()

freq1 = defaultdict(int)
freq2 = defaultdict(int)
freq3 = defaultdict(int)
freq4 = defaultdict(int)
words1 = [y for x in phy for y in x.split()]
for word in words1:
    freq1[word] += 1
words2 = [y for x in bioche for y in x.split()]
for word in words2:
    freq2[word] += 1
words3 = [y for x in cse for y in x.split()]
for word in words3:
    freq3[word] += 1
words4 = [y for x in com_tech for y in x.split()]
for word in words4:
    freq4[word] += 1
phy_wd = np.zeros((len(vocab),))
bioche_wd = np.zeros((len(vocab),))
comtech_wd = np.zeros((len(vocab),))
cse_wd = np.zeros((len(vocab),))
for i in range(len(vocab)):
    word = vocab[i]
    
    if word in freq1.keys():
        phy_wd[i] = ( freq1[word] + 1 ) / ( freq[word] + len(vocab) )
    else:
        phy_wd[i] = 1 / ( freq[word] + len(vocab) )
        
    if word in freq2.keys():
        bioche_wd[i] = ( freq2[word] + 1 ) / ( freq[word] + len(vocab) )
    else:
        bioche_wd[i] = 1 / ( freq[word] + len(vocab) )
        
    if word in freq3.keys():
        cse_wd[i] = ( freq3[word] + 1 ) / ( freq[word] + len(vocab) )
    else:
        cse_wd[i] = 1 / ( freq[word] + len(vocab) )
        
    if word in freq4.keys():
        comtech_wd[i] = ( freq4[word] + 1 ) / ( freq[word] + len(vocab) )
    else:
        comtech_wd[i]  = 1 / ( freq[word] + len(vocab) )
print('Relative Entropy comparisons\n')

print('Bioche - Physics\t',entropy(bioche_wd.tolist(),phy_wd.tolist()))
print('Bioche - CSE\t\t',entropy(bioche_wd.tolist(),cse_wd.tolist()))
print('Bioche - Comtech\t',entropy(bioche_wd.tolist(),comtech_wd.tolist()))

print('\nCSE - Physics\t',entropy(cse_wd.tolist(),phy_wd.tolist()))
print('CSE - Bioche\t',entropy(cse_wd.tolist(),bioche_wd.tolist()))
print('CSE - Comtech\t',entropy(cse_wd.tolist(),comtech_wd.tolist()))

print('\nComtech - Physics\t',entropy(comtech_wd.tolist(),phy_wd.tolist()))
print('Comtech - Bioche\t',entropy(comtech_wd.tolist(),bioche_wd.tolist()))
print('Comtech - CSE\t\t',entropy(comtech_wd.tolist(),cse_wd.tolist()))

print('\nPhy - Bio\t',entropy(phy_wd.tolist(),bioche_wd.tolist()))
print('Phy - CSE\t',entropy(phy_wd.tolist(),cse_wd.tolist()))
print('Phy - Comtech\t',entropy(phy_wd.tolist(),comtech_wd.tolist()))
from scipy.special import kl_div
print('KL - Divergence Tests\n')

print('Bioche - Physics\t',np.sum(kl_div(bioche_wd.tolist(),phy_wd.tolist())))
print('Bioche - CSE\t\t',np.sum(kl_div(bioche_wd.tolist(),cse_wd.tolist())))
print('Bioche - Comtech\t',np.sum(kl_div(bioche_wd.tolist(),comtech_wd.tolist())))

print('\nCSE - Physics\t',np.sum(kl_div(cse_wd.tolist(),phy_wd.tolist())))
print('CSE - Bioche\t',np.sum(kl_div(cse_wd.tolist(),bioche_wd.tolist())))
print('CSE - Comtech\t',np.sum(kl_div(cse_wd.tolist(),comtech_wd.tolist())))

print('\nComtech - Physics\t',np.sum(kl_div(comtech_wd.tolist(),phy_wd.tolist())))
print('Comtech - Bioche\t',np.sum(kl_div(comtech_wd.tolist(),bioche_wd.tolist())))
print('Comtech - CSE\t\t',np.sum(kl_div(comtech_wd.tolist(),cse_wd.tolist())))

print('\nPhy - Bio\t',np.sum(kl_div(phy_wd.tolist(),bioche_wd.tolist())))
print('Phy - CSE\t',np.sum(kl_div(phy_wd.tolist(),cse_wd.tolist())))
print('Phy - Comtech\t',np.sum(kl_div(phy_wd.tolist(),comtech_wd.tolist())))
from scipy.stats import chisquare
print('Chi-Square Tests\n')

print('Bioche - Physics\t',chisquare(bioche_wd.tolist(),phy_wd.tolist())[0])
print('Bioche - CSE\t\t',chisquare(bioche_wd.tolist(),cse_wd.tolist())[0])
print('Bioche - Comtech\t',chisquare(bioche_wd.tolist(),comtech_wd.tolist())[0])

print('\nCSE - Physics\t',chisquare(cse_wd.tolist(),phy_wd.tolist())[0])
print('CSE - Bioche\t',chisquare(cse_wd.tolist(),bioche_wd.tolist())[0])
print('CSE - Comtech\t',chisquare(cse_wd.tolist(),comtech_wd.tolist())[0])

print('\nComtech - Physics\t',chisquare(comtech_wd.tolist(),phy_wd.tolist())[0])
print('Comtech - Bioche\t',chisquare(comtech_wd.tolist(),bioche_wd.tolist())[0])
print('Comtech - CSE\t\t',chisquare(comtech_wd.tolist(),cse_wd.tolist())[0])

print('\nPhy - Bio\t',chisquare(phy_wd.tolist(),bioche_wd.tolist())[0])
print('Phy - CSE\t',chisquare(phy_wd.tolist(),cse_wd.tolist())[0])
print('Phy - Comtech\t',chisquare(phy_wd.tolist(),comtech_wd.tolist())[0])<jupyter_output>Chi-Square Tests

Bioche - Physics	 16.17580634150167
Bioche - CSE		 21.03258973770377
Bioche - Comtech	 17.708920398411276

CSE - Physics	 20.049280012188163
CSE - Bioche	 35.416670596347295
CSE - Comtech	 20.935164295088438

Comtech - Physics	 47.14298713395994
Comtech - Bioche	 76.90365036949514
Comtech - CSE		 61.878695572456515

Phy - Bio	 120.99147580812752
Phy - CSE	 126.17391380916007
Phy - Comtech	 53.79397294776871
{'fraction_non_alphanumeric': 0.18186419232391396, 'fraction_numerical': 0.02707718262336567, 'mean_word_length': 4.760932944606414, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '142757', 'license': 'MIT', 'n_tokens_mistral': 5249, 'n_tokens_neox': 5027, 'n_words': 780, 'path': 'classifiers/classification-analysis.ipynb', 'repo_name': 'mr-atharva-kulkarni/SPPU_AKAH-ICON-2020-TechDOfication-Shared-Task-Submission', 'size': '11855'}