<jupyter_start><jupyter_text>Coreference Resolution  Data preparation Read training data set and split it into train and evaluate part.<jupyter_code>from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
import numpy as np
import pandas as pd


#label,distance,POS1,gramFct1,Case1,Gender1,Number1,POS2,gramFct2,Case2,Gender2,Number2,sameHead,samePOS,sameGramFct,matchOrNot,sameCase,sameGender,sameNumber
df = (pd.read_csv('df_gold_std_all_18_12_with_case_and_gender_number.csv', 
                 header=0)
        .replace({'?': 'unknown'}))  # NaN are represented by '?'
#df=df.sample(frac=0.003)
#print(df)
X = df.drop(['no','label'], axis=1)#drop the 'label' and 'no' list.
y = df['label'].copy() # use the column 'label' as target y.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)#split the dataset to train and evaluate set with a ratio of 7:3<jupyter_output><empty_output><jupyter_text>Read the extracted markable as test set<jupyter_code>from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder
import numpy as np
import pandas as pd

#label,distance,POS1,gramFct1,Case1,Gender1,POS2,gramFct2,Case2,Gender2,sameHead,samePOS,sameGramFct,matchOrNot,sameCase,sameGender
df1 = (pd.read_csv('test_data_with_case_and_gender_number.csv', 
                 header=0)
        .replace({'?': 'unknown'}))  # NaN are represented by '?'
#df1=df1.sample(frac=0.1) #shuffle the data and get a 1/10 subset
X_te = df1.drop(['no','label'], axis=1)
y_te = df1['label'].copy()
X_te,y_te<jupyter_output><empty_output><jupyter_text>data details and distribution<jupyter_code>df.info()
#df1.info()
df['label'].value_counts()#see the distrubution of y
import matplotlib.pyplot as plt
df=df.drop(columns=['no'])
df1=df1.drop(columns=['no'])
df.hist(bins=50,figsize=(20,15))# plot the distrabution of all the matching features in training dataset.
df1.hist(bins=50,figsize=(20,15))# plot the distrabution of all the matching features in testing dataset.
df.describe()
corr_matrix=df.corr()

corr_matrix["label"].sort_values(ascending=False) #find the correlate metrics of  label in training set
corr_matrix1=df1.corr()
corr_matrix1["label"].sort_values(ascending=False)#find the correlate metrics of labels in testing set<jupyter_output><empty_output><jupyter_text>Data preprocessing  Label encoding transfer the columns(features) into numeric metrics<jupyter_code>class MultiColumnLabelEncoder:
    
    def __init__(self, columns = None):
        self.columns = columns # list of column to encode
	
    def fit(self, X, y=None):
        return self
	
    def transform(self, X):
        '''
        Transforms columns of X specified in self.columns using
        LabelEncoder(). If no columns specified, transforms all
        columns in X.
        '''
        
        output = X.copy()
        
        if self.columns is not None:
            for col in self.columns:
                output[col] = LabelEncoder().fit_transform(output[col])
        else:
            for colname, col in output.iteritems():
                output[colname] = LabelEncoder().fit_transform(col)
        
        return output
	
    def fit_transform(self, X, y=None):
        return self.fit(X, y).transform(X)
le = MultiColumnLabelEncoder(columns=['POS1','gramFct1','Case1','Gender1','Number1','POS2','gramFct2','gramFct2','Case2','Gender2','Number2'])
X_train_le = le.fit_transform(X_train)
X_train_le.head()<jupyter_output><empty_output><jupyter_text>one hot encoding of each features encode transfer the training set with one hot format<jupyter_code>ohe = OneHotEncoder(sparse=True , handle_unknown='ignore')
X_train_ohe = ohe.fit_transform(X_train_le)
X_train_ohe<jupyter_output><empty_output><jupyter_text>Model training SGD<jupyter_code>from sklearn.linear_model import SGDClassifier
sgd=SGDClassifier(random_state=42,max_iter=100)
sgd.fit(X_train_ohe,y_train)<jupyter_output><empty_output><jupyter_text>Evaluation with SGD<jupyter_code>from sklearn.model_selection import cross_val_score

#transform the input(X) in evaluating and testing data to an approprate way for SGD
X_test_le = le.transform(X_test)
X_test_ohe = ohe.transform(X_test_le)
X_te_le = le.transform(X_te)
X_te_ohe = ohe.transform(X_te_le)

#calculate the accuracy of sgd classifier of both data set
scores1 = cross_val_score(sgd,X_test_ohe,y_test,scoring='accuracy',cv=5) 
scores2=cross_val_score(sgd,X_te_ohe,y_te,scoring='accuracy',cv=5) 
scores1,scores2<jupyter_output><empty_output><jupyter_text>Test on SGD  Test with precision, recall and f1 score<jupyter_code>from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score 


print("Evaluate results on training set")
y_eva=sgd.predict(X_test_ohe)
print(precision_score(y_test,y_eva), recall_score(y_test,y_eva), f1_score(y_test,y_eva))
print("Test results on generated markable")
y_te_predict=sgd.predict(X_te_ohe)
print(precision_score(y_te,y_te_predict), recall_score(y_te,y_te_predict), f1_score(y_te,y_te_predict))<jupyter_output>Evaluate results on training set
0.7958779884583677 0.4052556460414743 0.5370493991989319
Test results on generated markable
0.29365079365079366 0.03737373737373737 0.06630824372759857<jupyter_text>Decision Tree<jupyter_code>from sklearn import tree

clf = tree.DecisionTreeClassifier(criterion="entropy")
clf = clf.fit(X_train_ohe,y_train)

y_test_predict=clf.predict(X_te_ohe)
print("Evaluation score:")
precision_score(y_te,y_test_predict), recall_score(y_te,y_test_predict), f1_score(y_te,y_test_predict)<jupyter_output>Evaluation score:<jupyter_text>Naive Bayes<jupyter_code>from sklearn.naive_bayes import BernoulliNB

clf = BernoulliNB()
clf = clf.fit(X_train_ohe,y_train)

y_test_predict=clf.predict(X_te_ohe)

precision_score(y_te,y_test_predict), recall_score(y_te,y_test_predict), f1_score(y_te,y_test_predict)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression

clf = LogisticRegression(C=1000.0, solver= 'liblinear', random_state=0)
clf.fit(X_train_ohe, y_train)

y_test_predict=clf.predict(X_te_ohe)

precision_score(y_te,y_test_predict), recall_score(y_te,y_test_predict), f1_score(y_te,y_test_predict)<jupyter_output><empty_output><jupyter_text>Perceptron<jupyter_code>from sklearn.linear_model import Perceptron

clf = Perceptron(tol=1e-5, random_state=0)

clf.fit(X_train_ohe, y_train)

y_test_predict=clf.predict(X_te_ohe)

precision_score(y_te,y_test_predict), recall_score(y_te,y_test_predict), f1_score(y_te,y_test_predict)<jupyter_output><empty_output><jupyter_text>Multi-Layer-Perceptron  parameter tuning: gridsearch<jupyter_code>from sklearn import svm
from sklearn.model_selection import GridSearchCV
from sklearn.neural_network import MLPClassifier
from sklearn.pipeline import Pipeline


def svc_param_selection(X, y, nfolds):
    Cs = [0.01, 0.1, 1, 10]
    gammas = [0.01, 0.1, 1]
    param_grid = {'C': Cs, 'gamma' : gammas}
    grid_search = GridSearchCV(svm.SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=-1)
    grid_search.fit(X, y)
    grid_search.best_params_
    return grid_search.best_params_
def mlp_param_selection(X,y,nfolds):
    mlp =MLPClassifier()
    param_grid={
       'hidden_layer_sizes':[(100, 100), (50,50), (100,)],
       'activations':['logistic', 'tanh', 'relu'],
        'solvers': ['adam'],
       'alphas': [0.0001, 0.001],
        'learning_rates':['constant', 'invscaling', 'adaptive'],
        'max_iters':[100, 200, 300],
    }
    mlp_g = GridSearchCV(mlp, param_grid, cv=nfolds, n_jobs=-1, verbose=1)
    mlp_g.fit(X, y)
    grid_search.best_params_
    return grid_search.best_params_
#p=svc_param_selection(X_train_ohe,y_train,2)
m=mlp_param_selection(X_train_ohe,y_train,5)<jupyter_output>Fitting 5 folds for each of 1 candidates, totalling 5 fits<jupyter_text>MLP training and testing<jupyter_code>from sklearn.metrics import f1_score
from sklearn.neural_network import MLPClassifier

mlp = MLPClassifier(solver='adam', alpha=1e-5, activation='relu',
                   hidden_layer_sizes=(200,200), random_state=1,max_iter=200, batch_size = 512, verbose=1)                  
mlp.fit(X_train_ohe,y_train)
y_test_predict=mlp.predict(X_te_ohe)
precision_score(y_te,y_test_predict,labels=np.unique(y_te_predict)), recall_score(y_te,y_test_predict,labels=np.unique(y_te_predict)), f1_score(y_te,y_test_predict,labels=np.unique(y_te_predict))<jupyter_output>Iteration 1, loss = 0.27578892
Iteration 2, loss = 0.22494391
Iteration 3, loss = 0.21233261
Iteration 4, loss = 0.20284774
Iteration 5, loss = 0.19511146
Iteration 6, loss = 0.18804923
Iteration 7, loss = 0.18187548
Iteration 8, loss = 0.17618617
Iteration 9, loss = 0.17106811
Iteration 10, loss = 0.16610981
Iteration 11, loss = 0.16128495
Iteration 12, loss = 0.15642207
Iteration 13, loss = 0.15197854
Iteration 14, loss = 0.14790045
Iteration 15, loss = 0.14390398
Iteration 16, loss = 0.13935871
Iteration 17, loss = 0.13626575
Iteration 18, loss = 0.13232528
Iteration 19, loss = 0.12945024
Iteration 20, loss = 0.12615267
Iteration 21, loss = 0.12284962
Iteration 22, loss = 0.12001398
Iteration 23, loss = 0.11714512
Iteration 24, loss = 0.11420103
Iteration 25, loss = 0.11188780
Iteration 26, loss = 0.10970814
Iteration 27, loss = 0.10679606
Iteration 28, loss = 0.10464124
Iteration 29, loss = 0.10222781
Iteration 30, loss = 0.10037747
Iteration 31, loss = 0.09829679
Iteration 32, los[...]<jupyter_text>KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(X_train_ohe,y_train)

y_test_predict=neigh.predict(X_te_ohe)

precision_score(y_te,y_test_predict), recall_score(y_te,y_test_predict), f1_score(y_te,y_test_predict)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10922598132717598, 'fraction_numerical': 0.06184118060435699, 'mean_word_length': 6.298168498168498, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '146390', 'license': 'Apache-2.0', 'n_tokens_mistral': 4116, 'n_tokens_neox': 3670, 'n_words': 704, 'path': 'Method1/model_training.ipynb', 'repo_name': 'wangqiaowen/Coreference-resolution', 'size': '9961'}