<jupyter_start><jupyter_text>Baseline training on full features including content<jupyter_code>import numpy as np
import pandas as pd

from sklearn.dummy import DummyClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report
from sklearn.neural_network import MLPClassifier

from sklearn.preprocessing import Normalizer<jupyter_output><empty_output><jupyter_text>Data loading<jupyter_code># Data paths
TRAIN = "../../data/prepared/full_train_df.pkl"
VAL = "../../data/prepared/full_val_df.pkl"
TEST = "../../data/prepared/full_test_df.pkl"
train_df = pd.read_pickle(TRAIN)
train_df.info()
train_df.head()
train_df_labels = train_df.retweet_label
train_df.drop(["retweet_label", "id_str"], axis=1, inplace=True)
val_df = pd.read_pickle(VAL)
val_df.info()
val_df_labels = val_df.retweet_label
val_df.drop(["retweet_label", "id_str"], axis=1, inplace=True)
test_df = pd.read_pickle(TEST)
test_df.info()
test_df_labels = test_df.retweet_label
test_df.drop(["retweet_label", "id_str"], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Content vector scaling<jupyter_code>vec_cols = list(range(768))
train_df[vec_cols].describe()
scaler = Normalizer()
transformed = scaler.fit_transform(train_df[vec_cols].values)
train_df[vec_cols] = transformed

transformed_val = scaler.transform(val_df[vec_cols].values)
val_df[vec_cols] = transformed_val

transformed_test = scaler.transform(test_df[vec_cols].values)
test_df[vec_cols] = transformed_test
train_df[vec_cols].describe()<jupyter_output><empty_output><jupyter_text>MLP Training<jupyter_code>clf = MLPClassifier(
    max_iter=50,
    hidden_layer_sizes=(512,),
    random_state=1,
    verbose=True,
    early_stopping=True,
    validation_fraction=0.1,
    n_iter_no_change=5,
    learning_rate_init=0.001,
    batch_size=256,
    alpha=0.001)
clf.fit(train_df.values, train_df_labels.values)
clf.best_validation_score_
clf.score(train_df.values, train_df_labels.values)
clf.score(val_df.values, val_df_labels.values)
val_predictions = clf.predict(val_df.values)
val_predictions.shape
out = classification_report(val_df_labels.values, val_predictions, output_dict=False)
print(out)<jupyter_output>precision    recall  f1-score   support

           0       0.70      0.77      0.73     10954
           1       0.68      0.59      0.63      8989

    accuracy                           0.69     19943
   macro avg       0.69      0.68      0.68     19943
weighted avg       0.69      0.69      0.69     19943<jupyter_text>MLP Classifier (2)<jupyter_code>clf_2 = MLPClassifier(
    max_iter=50,
    hidden_layer_sizes=(1000,50,30),
    random_state=1,
    verbose=True,
    early_stopping=True,
    validation_fraction=0.1,
    n_iter_no_change=5,
    learning_rate_init=0.001,
    batch_size=32,
    alpha=0.001)
clf_2.fit(train_df.values, train_df_labels.values)
clf_2.best_validation_score_
clf_2.score(train_df.values, train_df_labels.values)
clf_2.score(val_df.values, val_df_labels.values)
val_predictions = clf_2.predict(val_df.values)
val_predictions.shape
out = classification_report(val_df_labels.values, val_predictions, output_dict=False)
print(out)<jupyter_output>precision    recall  f1-score   support

           0       0.69      0.78      0.73     10954
           1       0.68      0.58      0.62      8989

    accuracy                           0.69     19943
   macro avg       0.68      0.68      0.68     19943
weighted avg       0.69      0.69      0.68     19943<jupyter_text>Random Forest<jupyter_code>rf = RandomForestClassifier(
    n_estimators=200,
    max_depth=25,
    min_samples_split=0.00002,
    min_samples_leaf=0.00002,
    random_state=1,
    verbose=True,
    n_jobs=-1,
    max_samples=0.4,
    max_features=0.2,
    oob_score=True,
    class_weight="balanced",
    min_impurity_decrease=0.00008
    )
rf.fit(train_df.values, train_df_labels.values)
rf.score(train_df.values, train_df_labels.values)
rf.score(val_df.values, val_df_labels.values)
rf_val_predictions = rf.predict(val_df.values)
out = classification_report(val_df_labels.values, rf_val_predictions, output_dict=False, digits=3)
print(out)<jupyter_output>precision    recall  f1-score   support

           0      0.698     0.749     0.723     10954
           1      0.664     0.606     0.634      8989

    accuracy                          0.684     19943
   macro avg      0.681     0.677     0.678     19943
weighted avg      0.683     0.684     0.683     19943<jupyter_text>Dummy classifier<jupyter_code>s_dummy = DummyClassifier(strategy="stratified", random_state=1)
s_dummy.fit(train_df.values, train_df_labels.values)
dummy_val_predictions = s_dummy.predict(val_df.values)
out = classification_report(val_df_labels.values, dummy_val_predictions, output_dict=False)
print(out)<jupyter_output>precision    recall  f1-score   support

           0       0.55      0.58      0.56     10954
           1       0.45      0.41      0.43      8989

    accuracy                           0.51     19943
   macro avg       0.50      0.50      0.50     19943
weighted avg       0.50      0.51      0.50     19943
{'fraction_non_alphanumeric': 0.09571150097465887, 'fraction_numerical': 0.07115009746588694, 'mean_word_length': 3.2440033085194377, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '151939', 'license': 'MIT', 'n_tokens_mistral': 2052, 'n_tokens_neox': 1868, 'n_words': 297, 'path': 'notebooks/baselines/full_features_2_labels.ipynb', 'repo_name': 'InfoCoV/Multi-Cro-CoV-cseBERT', 'size': '5130'}