<jupyter_start><jupyter_text>**Imports**<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
!wget http://nlp.stanford.edu/data/glove.6B.zip
!unzip glove.6B.zip -d embeddings
import os
import pandas as pd
import numpy as np
import xgboost as xgb

from sklearn.svm import SVR
from sklearn import preprocessing
from sklearn.neural_network import MLPRegressor
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor
FOLDER_PATH = "/content/drive/MyDrive/CS60075-Team28-Task-1"
DATA_FOLDER = os.path.join(FOLDER_PATH,"data/preprocessed")
# import evaluate function
import sys
sys.path.append(FOLDER_PATH)
from eval import evaluate
data = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_single_train_preprocessed.csv"), index_col=0)
data['token'] = data['token'].astype(str)
data['sentence'] = data['sentence'].astype(str)

data.head()
data_multi = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_multi_train_preprocessed.csv"), index_col=0)
data_multi['token'] = data_multi['token'].astype(str)
data_multi['sentence'] = data_multi['sentence'].astype(str)

data_multi.head()
# # use if model is to be trained on both data together

# data = pd.concat([data, data_multi])
def read_glove_vector(glove_vec):
  with open(glove_vec, 'r', encoding='UTF-8') as f:
    words = set()
    word_to_vec_map = {}
    for line in f:
      w_line = line.split()
      curr_word = w_line[0]
      word_to_vec_map[curr_word] = np.array(w_line[1:], dtype=np.float64)

  return word_to_vec_map

word_to_vec_map = read_glove_vector('embeddings/glove.6B.300d.txt')
print(len(word_to_vec_map)," words loaded!")
def get_embeddings(sentences, tokens):
    token_emb = []
    for s,t in zip(sentences, tokens):
        
        # fill unk by nan
        # calculate mean over non nan embeddings
        # fill unk by the mean embedding of sentence
        # pad 0 vectors till max_len
        
        temp_emb = [ word_to_vec_map[x] if x in word_to_vec_map else np.full((300,), np.nan) for x in t.split() ]
        
        # calculate mean for filling null values <unk>
        temp_sent_emb = [ word_to_vec_map[x] if x in word_to_vec_map else np.full((300,), np.nan) for x in s.split() ]
        mean_emb = np.nanmean(np.array(temp_sent_emb), axis=0)
        
        # single or multi - will be converted to (1,300) 
        temp_emb = np.mean(np.array([ mean_emb if np.isnan(x[0]) else x for x in temp_emb ]), axis=0)

        token_emb.append(temp_emb)

    return np.array(token_emb)<jupyter_output><empty_output><jupyter_text>CNN Regression Testing<jupyter_code>from sklearn.model_selection import train_test_split             
from keras.preprocessing.text import Tokenizer                    
from keras.preprocessing.sequence import pad_sequences

sentences_train_list = list(data['sentence'])
complexity_train_list = list(data['complexity'])

sentences_train,sentences_test,y_train,y_test = train_test_split(
                                                sentences_train_list, complexity_train_list,  
                                                test_size=0.25,  
                                                random_state=1000)
tokenizer = Tokenizer(num_words=5000)
tokenizer.fit_on_texts(sentences_train)

X_train = tokenizer.texts_to_sequences(sentences_train)
X_test = tokenizer.texts_to_sequences(sentences_test)
# Adding 1 because of  reserved 0 index
vocab_size = len(tokenizer.word_index) + 1                          

maxlen = 128

X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)
X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)
import numpy as np

def create_embedding_matrix(filepath, word_index, embedding_dim):
  vocab_size = len(word_index) + 1  
  embedding_matrix = np.zeros((vocab_size, embedding_dim))

  with open(filepath) as f:
    for line in f:
      word, *vector = line.split()
      if word in word_index:
        idx = word_index[word] 
        embedding_matrix[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]

  return embedding_matrix
embedding_dim = 300
embedding_matrix = create_embedding_matrix('embeddings/glove.6B.300d.txt ,tokenizer.word_index,  
                                            embedding_dim' )<jupyter_output><empty_output><jupyter_text>**Test Single Word**<jupyter_code>SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/baselines_ankit/single")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)
sentences_train_list = list(data['sentence'])
complexity_train_list = list(data['complexity'])
tokens_train_list = list(data['token'])

vectors = get_embeddings(sentences_train_list, tokens_train_list)
print(vectors.shape)
test_df = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_single_test_preprocessed.csv"), index_col=0)
test_df['token'] = test_df['token'].astype(str)
test_df['sentence'] = test_df['sentence'].astype(str)
sentences_test_list = list(test_df['sentence'])
test_tokens_list = list(test_df['token'])

testdf_vectors = get_embeddings(sentences_test_list, test_tokens_list)
testdf_vectors.shape
# Linear Regression
reg = LinearRegression().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/linear_regression_baseline.csv", index=False, header=False)

# Gradient Boosting
reg = GradientBoostingRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/gradient_boosting_baseline.csv", index=False, header=False)

# AdaBoost
reg = AdaBoostRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/ada_boost_baseline.csv", index=False, header=False)


# SVM regressor
reg = SVR().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/SVM_baseline.csv", index=False, header=False)

# MLP Regressor
reg = MLPRegressor(hidden_layer_sizes=(150)).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/MLP_baseline.csv", index=False, header=False)
evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_single_test_labelled_preprocessed.csv")
For file gradient_boosting_baseline.csv
pearson  :  0.7214444729661804
spearman :  0.6954546622318855
mae      :  0.06718193713274725
mse      :  0.00777120977316972
r2       :  0.5198533483837506<jupyter_output><empty_output><jupyter_text>**Multi Word**<jupyter_code>SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/baselines_ankit/multi")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)
#  DO NOT run if both is being trained together
sentences_train_list = list(data_multi['sentence'])
complexity_train_list = list(data_multi['complexity'])
tokens_train_list = list(data_multi['token'])
#  DO NOT run if both is being trained together
vectors = get_embeddings(sentences_train_list, tokens_train_list)
print(vectors.shape)
test_df = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_multi_test_preprocessed.csv"), index_col=0)
test_df['token'] = test_df['token'].astype(str)
test_df['sentence'] = test_df['sentence'].astype(str)
sentences_test_list = list(test_df['sentence'])
test_tokens_list = list(test_df['token'])
testdf_vectors = get_embeddings(sentences_test_list, test_tokens_list)
testdf_vectors.shape
# Linear Regression
reg = LinearRegression().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/linear_regression_baseline.csv", index=False, header=False)

# Gradient Boosting
reg = GradientBoostingRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/gradient_boosting_baseline.csv", index=False, header=False)

# AdaBoost
reg = AdaBoostRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/ada_boost_baseline.csv", index=False, header=False)


# SVM regressor
reg = SVR().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/SVM_baseline.csv", index=False, header=False)

# MLP Regressor
reg = MLPRegressor(hidden_layer_sizes=(150)).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(testdf_vectors)

pred = pd.DataFrame({"ID":test_df.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/MLP_baseline.csv", index=False, header=False)
evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_multi_test_labelled_preprocessed.csv")
For file linear_regression_baseline.csv
pearson  :  0.7384735689252496
spearman :  0.7096751511559939
mae      :  0.08420414291697727
mse      :  0.011177467719053704
r2       :  0.5369181534723191

For file gradient_boosting_baseline.csv
pearson  :  0.7870767227623038
spearman :  0.7664566772179571
mae      :  0.07831874126395687
mse      :  0.009391251928283358
r2       :  0.6109209712372773

For file SVM_baseline.csv
pearson  :  0.7747686594582374
spearman :  0.7563503882622532
mae      :  0.07906457106883466
mse      :  0.009776174954316456
r2       :  0.5949736323456092<jupyter_output><empty_output><jupyter_text>Features - load data<jupyter_code>f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_single_train_features.csv"), index_col=0)
f1['token'] = f1['token'].astype(str)
f1['sentence'] = f1['sentence'].astype(str)
f1.set_index("id", inplace=True)

# drop unwanted features
f1.drop(['parse', 'lemma'], axis=1, inplace=True)

print(f1.columns)

f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_single_train_preprocessed.csv"), index_col=0)
f2['token'] = f2['token'].astype(str)
f2['sentence'] = f2['sentence'].astype(str)
print(f2.columns)

features = f1.merge(f2, on=['id','sentence', 'corpus', 'token', 'complexity'])
# fill pos nan by NN, as they are in majority
features['pos'] = features['pos'].fillna('NN')
features['token_length'] = features['token_length'].fillna(0)

# categorical encoding
labels = dict(features['pos'].value_counts())
labels = { k:i for i,k in enumerate(labels)}
labels['POS'] = len(labels)
print(labels)


def get_vowels(word):
    val = 0
    for w in word:
        if(w in ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U','u']):
            val+=1
    return val

features['token_vowels'] = features['token'].apply(lambda x: get_vowels(x) )


features['pos'] = features['pos'].apply((lambda x: labels[x]))

# scaler = preprocessing.StandardScaler()
# features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']] =  \
#     scaler.fit_transform(features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']])


features.head()
multi_f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_multi_train_split_features.csv"), index_col=0)
multi_f1['token'] = multi_f1['token'].astype(str)
multi_f1['sentence'] = multi_f1['sentence'].astype(str)
multi_f1.set_index("id", inplace=True)

# drop unwanted features
multi_f1.drop(['parse', 'token1', 'token2', 'lemma1', 'lemma2', 'Unnamed: 0.1'], axis=1, inplace=True)

multi_f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_multi_train_preprocessed.csv"), index_col=0)
multi_f2['token'] = multi_f2['token'].astype(str)
multi_f2['sentence'] = multi_f2['sentence'].astype(str)

multi_features = multi_f1.merge(multi_f2, on=['id','sentence', 'corpus', 'token', 'complexity'])
multi_features.head(2)
# fill pos nan by NN, as they are in majority
multi_features['pos2'] = multi_features['pos2'].fillna('NN')

multi_features['pos1'] = multi_features['pos1'].apply((lambda x: labels[x]))
multi_features['pos2'] = multi_features['pos2'].apply((lambda x: labels[x]))

# scaler = preprocessing.StandardScaler()
# multi_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']] =  \
#     scaler.fit_transform(multi_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']])


multi_features.head()
# merge both single and multi features

features['pos1'] = features['pos'].copy()
features['pos2'] = features['pos'] 

features['dep num1'] = features['dep num'] 
features['dep num2'] = features['dep num'] 

features['synonyms1'] = features['synonyms'] 
features['synonyms2'] = features['synonyms'] 

features['hypernyms1'] = features['hypernyms'] 
features['hypernyms2'] = features['hypernyms'] 

features['hyponyms1'] = features['hyponyms'] 
features['hyponyms2'] = features['hyponyms'] 

features['google frequency1'] = features['google frequency'] 
features['google frequency2'] = features['google frequency1'] 

features.drop(['pos','dep num', 'synonyms', 'hyponyms', 'hypernyms', 'google frequency'], axis=1, inplace=True)

features = features.append( multi_features)
print(len(features))

scaler = preprocessing.StandardScaler()
features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']] =  \
    scaler.fit_transform(features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']])

features.head()<jupyter_output>9179<jupyter_text>Test with features - single<jupyter_code>sentences_train_list = list(features['sentence'])
complexity_train_list = list(features['complexity'])
tokens_train_list = list(features['token'])

vectors = get_embeddings(sentences_train_list, tokens_train_list)
print(vectors.shape)

# f_vectors = features[['token_length', 'token_vowels', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values
# or 

f_vectors = features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

print(f_vectors.shape)
vectors = np.concatenate((vectors, f_vectors), axis=1)
SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/baselines_ankit/features/single")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)
test_f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_single_test_features.csv"), index_col=0)
test_f1['token'] = test_f1['token'].astype(str)
test_f1['sentence'] = test_f1['sentence'].astype(str)
test_f1.set_index("id", inplace=True)

# drop unwanted features
test_f1.drop(['parse', 'lemma'], axis=1, inplace=True)

test_f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_single_test_preprocessed.csv"), index_col=0)
test_f2['token'] = test_f2['token'].astype(str)
test_f2['sentence'] = test_f2['sentence'].astype(str)

test_features = test_f1.merge(test_f2, on=['id','sentence', 'corpus', 'token'])

test_features.head()
# # fill pos nan by NN, as they are in majority
# test_features['pos'] = test_features['pos'].fillna('NN')

# # categorical encoding
# test_features['pos'] = test_features['pos'].apply((lambda x: labels[x]))

# def get_vowels(word):
#     val = 0
#     for w in word:
#         if(w in ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U','u']):
#             val+=1
#     return val

# test_features['token_vowels'] = test_features['token'].apply(lambda x: get_vowels(x) )

# test_features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']] =  \
#     scaler.transform(test_features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']])

# ##########  for all features

# fill pos nan by NN, as they are in majority
test_features['pos'] = test_features['pos'].fillna('NN')
test_features['token_length'] = test_features['token_length'].fillna(0)

test_features['pos'] = test_features['pos'].apply((lambda x: labels[x]))

def get_vowels(word):
    val = 0
    for w in word:
        if(w in ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U','u']):
            val+=1
    return val

test_features['token_vowels'] = test_features['token'].apply(lambda x: get_vowels(x) )

test_features['pos1'] = test_features['pos'].copy()
test_features['pos2'] = test_features['pos'] 

test_features['dep num1'] = test_features['dep num'] 
test_features['dep num2'] = test_features['dep num'] 

test_features['synonyms1'] = test_features['synonyms'] 
test_features['synonyms2'] = test_features['synonyms'] 

test_features['hypernyms1'] = test_features['hypernyms'] 
test_features['hypernyms2'] = test_features['hypernyms'] 

test_features['hyponyms1'] = test_features['hyponyms'] 
test_features['hyponyms2'] = test_features['hyponyms'] 

test_features['google frequency1'] = test_features['google frequency'] 
test_features['google frequency2'] = test_features['google frequency'] 

test_features.drop(['pos','dep num', 'synonyms', 'hyponyms', 'hypernyms', 'google frequency'], axis=1, inplace=True)


test_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']] =  \
    scaler.transform(test_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']])

test_features.head()
sentences_test_list = list(test_features['sentence'])
test_tokens_list = list(test_features['token'])

test_vectors = get_embeddings(sentences_test_list, test_tokens_list)
print(test_vectors.shape)

# test_f_vectors = test_features[['token_length', 'token_vowels', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

test_f_vectors = test_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

test_vectors = np.concatenate((test_vectors, test_f_vectors), axis=1)
print(test_vectors.shape)
# # # Gradient Boosting
reg = GradientBoostingRegressor(n_estimators=250).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/gradient_boosting_baseline.csv", index=False, header=False)

# Linear Regression
reg = LinearRegression().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/linear_regression_baseline.csv", index=False, header=False)

# # xgb Regression
# from xgboost import XGBRegressor
# reg = XGBRegressor(objective ='reg:squarederror', n_estimators=250).fit(vectors, np.array(complexity_train_list))
# y_pred = reg.predict(test_vectors)

# pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
# pred.to_csv(SUBMISSION_FOLDER+"/xgb_regression_baseline.csv", index=False, header=False)


# # AdaBoost
reg = AdaBoostRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/ada_boost_baseline.csv", index=False, header=False)

# SVM regressor
reg = SVR().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/SVM_baseline.csv", index=False, header=False)

# MLP Regressor
reg = MLPRegressor(hidden_layer_sizes=(150)).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/MLP_baseline.csv", index=False, header=False)
evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_single_test_labelled_preprocessed.csv")
pearson  :  0.7420614648980268
spearman :  0.7104726955580672
mae      :  0.06555679719442302
mse      :  0.007306746999771031
r2       :  0.5485503276131269

pearson  :  0.7401818712131418
spearman :  0.7073212533102632
mae      :  0.06548487250652647
mse      :  0.007338576693601775
r2       :  0.5465837199212888<jupyter_output><empty_output><jupyter_text>Test with features - multi<jupyter_code>sentences_train_list = list(multi_features['sentence'])
complexity_train_list = list(multi_features['complexity'])
tokens_train_list = list(multi_features['token'])

vectors = get_embeddings(sentences_train_list, tokens_train_list)
print(vectors.shape)

f_vectors = multi_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values
print(f_vectors.shape)
vectors = np.concatenate((vectors, f_vectors), axis=1)
SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/baselines_ankit/features/multi")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)
test_multi_f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_multi_test_split_features.csv"), index_col=0)
test_multi_f1['token'] = test_multi_f1['token'].astype(str)
test_multi_f1['sentence'] = test_multi_f1['sentence'].astype(str)
test_multi_f1.set_index("id", inplace=True)

# drop unwanted features
test_multi_f1.drop(['parse', 'token1', 'token2', 'lemma1', 'lemma2', 'Unnamed: 0.1'], axis=1, inplace=True)

test_multi_f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_multi_test_preprocessed.csv"), index_col=0)
test_multi_f2['token'] = test_multi_f2['token'].astype(str)
test_multi_f2['sentence'] = test_multi_f2['sentence'].astype(str)

test_multi_features = test_multi_f1.merge(test_multi_f2, on=['id','sentence', 'corpus', 'token'])
test_multi_features['token'] = test_multi_f2['token'].astype(str)
test_multi_features.head(2)
# fill pos nan by NN, as they are in majority
test_multi_features['pos2'] = test_multi_features['pos2'].fillna('NN')

test_multi_features['pos1'] = test_multi_features['pos1'].apply((lambda x: labels[x]))
test_multi_features['pos2'] = test_multi_features['pos2'].apply((lambda x: labels[x]))

test_multi_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']] =  \
    scaler.transform(test_multi_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']])


test_multi_features.head()
sentences_test_list = list(test_multi_features['sentence'])
test_tokens_list = list(test_multi_features['token'])

test_vectors = get_embeddings(sentences_test_list, test_tokens_list)
print(test_vectors.shape)

test_f_vectors = test_multi_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values
test_vectors = np.concatenate((test_vectors, test_f_vectors), axis=1)
print(test_vectors.shape)
# Linear Regression
reg = LinearRegression().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/linear_regression_baseline.csv", index=False, header=False)

# Gradient Boosting
reg = GradientBoostingRegressor(n_estimators=100).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/gradient_boosting_baseline.csv", index=False, header=False)

# AdaBoost
reg = AdaBoostRegressor().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/ada_boost_baseline.csv", index=False, header=False)

# SVM regressor
reg = SVR().fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/SVM_baseline.csv", index=False, header=False)

# MLP Regressor
reg = MLPRegressor(hidden_layer_sizes=(150)).fit(vectors, np.array(complexity_train_list))
y_pred = reg.predict(test_vectors)

pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":y_pred})
pred.to_csv(SUBMISSION_FOLDER+"/MLP_baseline.csv", index=False, header=False)
evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_multi_test_labelled_preprocessed.csv")<jupyter_output>For file linear_regression_baseline.csv
pearson  :  0.80162468109306
spearman :  0.7958259247466357
mae      :  0.07648070707269523
mse      :  0.009377504110086981
r2       :  0.6114905425566605

For file gradient_boosting_baseline.csv
pearson  :  0.7425749839115788
spearman :  0.7295822333985482
mae      :  0.08519289095213395
mse      :  0.011636223379959393
r2       :  0.517911932752478

For file ada_boost_baseline.csv
pearson  :  0.7205928299840861
spearman :  0.722019865932745
mae      :  0.09487093040550654
mse      :  0.013532488729031253
r2       :  0.43934976809887283

For file MLP_baseline.csv
pearson  :  0.5627625010663627
spearman :  0.5379100633691584
mae      :  0.11296352326760921
mse      :  0.02100409896531524
r2       :  0.1298013845365148

For file SVM_baseline.csv
pearson  :  0.81236069308495
spearman :  0.7997878887495248
mae      :  0.07301231575210401
mse      :  0.008358819519049489
r2       :  0.6536945867376867
{'fraction_non_alphanumeric': 0.14898953705030815, 'fraction_numerical': 0.04805073813960155, 'mean_word_length': 6.176394960143996, 'pattern_counts': {'":': 42, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '491095', 'license': 'MIT', 'n_tokens_mistral': 11117, 'n_tokens_neox': 10217, 'n_words': 1653, 'path': 'baselines_and_with_features.ipynb', 'repo_name': 'hvarS/CS60075-Team28-Semeval-Task-1', 'size': '27908'}