<jupyter_start><jupyter_text>Import Libraries<jupyter_code>%matplotlib inline 
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import numpy as np
from sklearn.pipeline import Pipeline
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import LeaveOneOut
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.metrics import classification_report
from sklearn.dummy import DummyClassifier<jupyter_output><empty_output><jupyter_text>Classification Fold 1 * This code runs SVM with TF-IDF model for classification of keyphrases    * The training and testing for each of the phases is seperated.  * This can be modified using line ```traindata=pd.read_csv("../data/test1.tsv",delimiter="\t")``` and ```traindata=pd.read_csv("../data/train1.tsv",delimiter="\t")```* Also to run for SVM with Bag of words use ```clf = Pipeline([('vect', CountVectorizer(ngram_range=(1,2))), ('clf', SGDClassifier(loss='log', penalty='l2',alpha=0.001, max_iter=100, random_state=42,class_weight="balanced",warm_start=True))])```<jupyter_code>traindata=pd.read_csv("../data/tsv/test1.tsv",delimiter="\t")
X_test=traindata.text.values
Y_test=np.array(traindata.label.values).astype(np.int32)

traindata=pd.read_csv("../data/tsv/train1.tsv",delimiter="\t")
X_train=traindata.text.values
Y_train=np.array(traindata.label.values).astype(np.int32)
clf = Pipeline([('vect', CountVectorizer(ngram_range=(1,2))), ('tfidf', TfidfTransformer(use_idf=True)),
                         ('clf', SGDClassifier(loss='log', penalty='l2',alpha=0.001, max_iter=100, random_state=42,class_weight="balanced",warm_start=True))])



clf.fit(X_train,Y_train)
ypred=clf.predict(X_test)

print(classification_report(Y_test,ypred,digits=5))<jupyter_output><empty_output><jupyter_text>Classification Fold 2 * This code runs SVM with TF-IDF model for classification of keyphrases    * The training and testing for each of the phases is seperated.  * This can be modified using line ```traindata=pd.read_csv("../data/test1.tsv",delimiter="\t")``` and ```traindata=pd.read_csv("../data/train1.tsv",delimiter="\t")```* Also to run for SVM with Bag of words use ```clf = Pipeline([('vect', CountVectorizer(ngram_range=(1,2))), ('clf', SGDClassifier(loss='log', penalty='l2',alpha=0.001, max_iter=100, random_state=42,class_weight="balanced",warm_start=True))])```<jupyter_code>traindata=pd.read_csv("../data/tsv/train2.tsv",delimiter="\t")
X_train=traindata.text.values
Y_train=np.array(traindata.label.values).astype(np.int32)

traindata=pd.read_csv("../data/tsv/test2.tsv",delimiter="\t")
X_test=traindata.text.values
Y_test=np.array(traindata.label.values).astype(np.int32)

clf2 = Pipeline([('vect', CountVectorizer(ngram_range=(1,2))), ('tfidf', TfidfTransformer(use_idf=True)),
                         ('clf', SGDClassifier(loss='log', penalty='l2',alpha=0.001, max_iter=100, random_state=42,class_weight="balanced",warm_start=True))])

clf2.fit(X_train,Y_train)
ypred=clf2.predict(X_test)

print(classification_report(Y_test,ypred,digits=5))<jupyter_output><empty_output><jupyter_text>Intepretability * This code evaluates previously trained model for interpretability using *eli5* library    * The training and testing for each of the phases can be modified using line ```xtrain=pickle.load(open('../data/summary_pkl/xtrain.pkl', 'rb'))``````xtest=pickle.load(open('../data/summary_pkl/xtest.pkl', 'rb'))``````ytrain=pickle.load(open('../data/summary_pkl/ytrain.pkl', 'rb'))``````ytest=pickle.load(open('../data/summary_pkl/ytest.pkl', 'rb'))```* Following function returns new features for LIME generated texts```def returnfeatures(text):    global xtraindict    res=[]    for i in text:        res.append(xtraindict[i])    return np.array(res)```<jupyter_code>import eli5
from eli5.lime import TextExplainer
import pickle
import eli5
from eli5.lime import TextExplainer
from xgboost import XGBClassifier

xtrain=pickle.load(open('../data/summary_pkl/xtrain.pkl', 'rb'))
xtest=pickle.load(open('../data/summary_pkl/xtest.pkl', 'rb'))

ytrain=pickle.load(open('../data/summary_pkl/ytrain.pkl', 'rb'))
ytest=pickle.load(open('../data/summary_pkl/ytest.pkl', 'rb'))


traindata=pd.read_csv("../data/tsv/test2.tsv",delimiter="\t")
X_test=traindata.text.values
Y_test=np.array(traindata.label.values).astype(np.int32)

traindata=pd.read_csv("../data/tsv/train2.tsv",delimiter="\t")
X_train=traindata.text.values
Y_train=np.array(traindata.label.values).astype(np.int32)


xtraindict={}


for i,j in zip(X_train,xtrain):
    xtraindict[i]=j

for i,j in zip(X_test,xtest):
    xtraindict[i]=j

def returnfeatures(text):
    global xtraindict
    res=[]
    for i in text:
        res.append(xtraindict[i])
    return np.array(res)


clf11 = SGDClassifier(loss='log', penalty='l2',alpha=1e-3, max_iter=200, random_state=42,class_weight="balanced",warm_start=True)  #Pipeline([('custom', FunctionTransformer(returnfeatures)),('clf', SGDClassifier(loss='log', penalty='l2',alpha=1e-3, max_iter=200, random_state=42,class_weight="balanced",warm_start=True))])
clf11.fit(np.array(xtrain),np.array(ytrain))
ypred=clf11.predict(xtest)
print(classification_report(ytest,ypred,digits=5))


clf12=XGBClassifier(n_estimators=200,random_state=10,max_depth=3,learning_rate =0.1)
clf12.fit(np.array(xtrain),np.array(ytrain))
ypred=clf12.predict(xtest)
print(classification_report(ytest,ypred,digits=5))<jupyter_output><empty_output><jupyter_text>Display Interpretablity Results * Change testfile in ``` traindata=pd.read_csv("../data/tsv/test2.tsv",delimiter="\t")```<jupyter_code>from IPython.display import display
te = TextExplainer(random_state=42)

#Change file names here
traindata=pd.read_csv("../data/tsv/test2.tsv",delimiter="\t")
X_test=traindata.text.values
Y_test=np.array(traindata.label.values).astype(np.int32)

X_test=np.array(X_test)
for i in range(len(X_test)):
    print(X_test[i])
    te.fit(X_test[i], clf11.predict_proba)
    a=te.show_prediction(target_names=[1,2,3,4])
    display(a)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16281341582546402, 'fraction_numerical': 0.02149137088896125, 'mean_word_length': 7.190666666666667, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '277132', 'license': 'MIT', 'n_tokens_mistral': 2363, 'n_tokens_neox': 2293, 'n_words': 336, 'path': 'RQ2.3/code/traditional_interpretability.ipynb', 'repo_name': 'manikandan-ravikiran/complex-assignments', 'size': '6142'}