<jupyter_start><jupyter_text>Load the train data<jupyter_code># Su questo file ci sono i commenti raggruppati per autore e vettorizzati tramite w2v. Sono 5000 vettori di 200 
# componenti l'uno 
train_w2v = pd.read_pickle("TrainW2VXAuthor.csv")  
print (train_w2v.shape[0])
train_w2v.W2V.head()<jupyter_output>5000<jupyter_text>Model Selection<jupyter_code>from sklearn.preprocessing import StandardScaler # For scaling
from sklearn.model_selection import train_test_split # for creating valid set and train set 
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import KFold
from sklearn.svm import SVC, LinearSVC
#Dataset
X = np.array(list(map(np.array, train_w2v.W2V)))
y = np.array(list(map(np.array, train_w2v.gender)))

# Training set + validation set
SEED = 20003
x_train , x_validation, y_train , y_validation = train_test_split(X, y, test_size = 1000, 
                                                                 random_state = SEED)

# Il validation set è di dimensione 1000 (1/5 del totale --> rule of thumb) perchè dopo questa classificazione dobbiamo 
# fare una regressione logistica e quindi dobbiamo dividere i 1000 ulteriormente in 800 per il training e 200 per 
# validation della logistica 

# MLP : semplice e si allena in fretta -- alcuni accorgimenti sui parametri 

mlpClf = MLPClassifier(solver = 'adam', activation= 'relu' ,alpha = 0.05, verbose = True, early_stopping = True,
                         learning_rate = 'invscaling', max_iter = 400)

# Cross validation - 10 Fold 
kf = KFold(n_splits = 10)

for train_indices, test_indices in kf.split(x_train):
    mlpClf.fit(x_train[train_indices], y_train[train_indices])
    print(mlpClf.score(x_train[test_indices], y_train[test_indices]))

y_score = mlpClf.predict_proba(x_validation)[:,1]
# Roc Curve for validation data 

# Utilizziamo la metrica roc perchè avremo la stessa nell'upload della sfida 

from sklearn.metrics import roc_curve, auc
fpr, tpr, thresholds = roc_curve(y_validation, y_score)
roc_auc = auc(fpr, tpr) 

plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
         lw=lw, label='ROC curve (area = %0.4f)'% roc_auc )

plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()
score = pd.DataFrame({"y_validation":y_validation, "W2VGender":y_score})
score.to_csv('scoreW2VAugValidation.csv')<jupyter_output><empty_output><jupyter_text>Test Set<jupyter_code>test_data_prob = pd.read_pickle('TestW2VXAuthorNoBody.csv')
print (test_data_prob.shape[0])
test_data_prob.head()
listofzeros = [0]*200
# Problemi con emojis --> erano rimasti dei commenti vuoti che davano errore 

test_data_prob.sentences[11377] = listofzeros
test_data_prob.sentences[11460] = listofzeros
test_data_prob.sentences[12941] = listofzeros
test_data_prob.sentences[13161] = listofzeros
test_data_prob.sentences[14152] = listofzeros
X_test = np.array(list(map(np.array, test_data_prob.sentences)))
np.where(pd.isnull(test_data_prob.sentences))
y_scoreTest = mlpClf.predict_proba(X_test)[:,1]
y_scoreTest[11377] = 0.5
y_scoreTest[11460] = 0.5
y_scoreTest[12941] = 0.5
y_scoreTest[13161] = 0.5
y_scoreTest[14152] = 0.5
solutionW2V = pd.DataFrame({"author":test_data_prob.author, "W2VGender":y_scoreTest})
solutionW2V.head()
solutionW2V.to_csv('solutionW2V.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11228270162439441, 'fraction_numerical': 0.04189227700199487, 'mean_word_length': 5.335740072202166, 'pattern_counts': {'":': 4, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '161094', 'license': 'MIT', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1303, 'n_words': 285, 'path': '.ipynb_checkpoints/W2VXAuthor-checkpoint.ipynb', 'repo_name': 'pitmonticone/RedditTextClassification', 'size': '3509'}