<jupyter_start><jupyter_text>Load Data<jupyter_code>data = pd.read_csv('/content/drive/My Drive/progetto_aml/dataset/train_prepocessed.csv')
data.head()<jupyter_output><empty_output><jupyter_text>Funzioni utilli<jupyter_code># Funzioni per calcolare ndcg

def dcg_score(y_true, y_score, k=5):
    order = np.argsort(y_score)[::-1]
    y_true = np.take(y_true, order[:k])

    gain = 2 ** y_true - 1

    discounts = np.log2(np.arange(len(y_true)) + 2)
    return np.sum(gain / discounts)


def ndcg_score(ground_truth, predictions, k=5):
    lb = LabelBinarizer()
    lb.fit(range(len(predictions) ))
    T = lb.transform(ground_truth)

    scores = []

    # Iterate over each y_true and compute the DCG score
    for y_true, y_score in zip(T, predictions):
        actual = dcg_score(y_true, y_score, k)
        best = dcg_score(y_true, y_true, k)
        score = float(actual) / float(best)
        scores.append(score)

    return np.mean(scores)



ndcg_scorer = make_scorer(ndcg_score, needs_proba=True, k=5)
# Top_n accuracy

def top_n_accuracy(y_true, y_pred_prob, n = 5):
    topn = np.argsort(y_pred_prob, axis = 1)[:,-n:]
    acc = np.mean(np.array([1 if y_true[k] in topn[k] else 0 for k in range(len(topn))]))
    return(acc)<jupyter_output><empty_output><jupyter_text>Classificatori  SVM (Molto lento, non verrà più sviluppata)<jupyter_code>from sklearn.svm import SVC

clf3 = SVC(kernel='rbf', class_weight='balanced',
           decision_function_shape='ovo', probability=True, cache_size = 2000)

clf3 = clf3.fit(X_train, y_train)
y_pred_prob = clf3.predict_proba(X_test.values)
ndcg_score(encoded_y_test, y_pred_prob, k = 5)<jupyter_output><empty_output><jupyter_text>XGboost<jupyter_code>data_prova = data.copy()
data_prova.drop("id", axis = 1, inplace = True)
X_train, X_test = train_test_split(data_prova, test_size = 0.2, stratify = data_prova["country_destination"], random_state = 42)
y_train = X_train.pop("country_destination")
y_test = X_test.pop("country_destination")
label_encoder = LabelEncoder()
encoded_y_train = label_encoder.fit_transform(y_train)
encoded_y_test = label_encoder.fit_transform(y_test)
xgb = XGBClassifier(max_depth=7, learning_rate=0.18, n_estimators=100,
                    objective='multi:softprob', colsample_bytree=1, 
                    seed=42, n_jobs = -1, tree_method = "gpu_hist", max_delta_step = 1)
xgb.fit(X_train.values, y_train, 
        eval_set = [(X_train.values, y_train), (X_test.values, y_test)], 
        eval_metric = "mlogloss",
        early_stopping_rounds=3)
results = xgb.evals_result()
epochs = len(results['validation_1']['mlogloss'])
x_axis = range(0, epochs)
# plot log loss
fig, ax = plt.subplots()
ax.plot(x_axis, results['validation_0']['mlogloss'], label='Train')
ax.plot(x_axis, results['validation_1']['mlogloss'], label='Test')
ax.legend()
plt.ylabel('Log Loss')
plt.title('XGBoost Log Loss')
plt.show()
y_pred_prob = xgb.predict_proba(X_test.values)
ndcg_score(encoded_y_test, y_pred_prob, k = 5)
top_n_accuracy(encoded_y_test, y_pred_prob)<jupyter_output><empty_output><jupyter_text>Prova con XGBoost e dataset bilanciato (peggio del normale, non verrà più sviluppata)<jupyter_code>data_prova = data.copy()
data_prova.pop("id")
# Per bilanciare il Dataset

num = data_prova.country_destination.value_counts()[-1]

df_majority = data_prova[data_prova.country_destination=="NDF"]
df_minority = data_prova[data_prova.country_destination=="US"]
df_minority1 = data_prova[data_prova.country_destination=="FR"]
df_minority2 = data_prova[data_prova.country_destination=="other"]
df_minority3 = data_prova[data_prova.country_destination=="ES"]
df_minority4 = data_prova[data_prova.country_destination=="IT"]
df_minority5 = data_prova[data_prova.country_destination=="GB"]
df_minority6 = data_prova[data_prova.country_destination=="CA"]
df_minority7 = data_prova[data_prova.country_destination=="DE"]
df_minority8 = data_prova[data_prova.country_destination=="NL"]
df_minority9 = data_prova[data_prova.country_destination=="AU"]
df_minority10 = data_prova[data_prova.country_destination=="PT"]
X_train = df_majority[:num].append([df_minority[:num], df_minority1[:num], 
                         df_minority2[:num], df_minority3[:num], df_minority4[:num],
                         df_minority5[:num], df_minority6[:num], df_minority7[:num],
                         df_minority8[:num], df_minority9[:num], df_minority10[:num]], ignore_index = True)
X_train.country_destination.value_counts()
X_train = X_train.sample(frac=1)
X_train, X_test = train_test_split(X_train, test_size = 0.2, stratify = X_train["country_destination"], random_state = 42)
y_train = X_train.pop("country_destination")
y_test = X_test.pop("country_destination")
label_encoder = LabelEncoder()
encoded_y_test = label_encoder.fit_transform(y_test)
xgb = XGBClassifier(max_depth=7, learning_rate=0.18, n_estimators=100,
                    objective='multi:softprob', colsample_bytree=1, seed=42, n_jobs = -1, tree_method='gpu_hist')
xgb.fit(X_train.values, y_train, 
        eval_set = [(X_train.values, y_train), (X_test.values, y_test)], 
        eval_metric = "mlogloss",
        early_stopping_rounds=3)
results = xgb.evals_result()
epochs = len(results['validation_1']['mlogloss'])
x_axis = range(0, epochs)
# plot log loss
fig, ax = plt.subplots()
ax.plot(x_axis, results['validation_0']['mlogloss'], label='Train')
ax.plot(x_axis, results['validation_1']['mlogloss'], label='Test')
ax.legend()
plt.ylabel('Log Loss')
plt.title('XGBoost Log Loss')
plt.show()
y_pred_prob = xgb.predict_proba(X_test.values)
ndcg_score(encoded_y_test, y_pred_prob, k = 5)
top_n_accuracy(encoded_y_test, y_pred_prob)<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>data_prova = data.copy()
data_prova.drop("id", axis = 1, inplace = True)
X_train, X_test = train_test_split(data_prova, test_size = 0.2, stratify = data_prova["country_destination"], random_state = 42)
y_train = X_train.pop("country_destination")
y_test = X_test.pop("country_destination")
label_encoder = LabelEncoder()

encoded_y_train = label_encoder.fit_transform(y_train)
encoded_y_test = label_encoder.fit_transform(y_test)
from sklearn.utils import class_weight

class_weights = class_weight.compute_class_weight('balanced',
                                                 np.unique(encoded_y_train),
                                                 encoded_y_train)
n_classes = len(np.unique(y_train))
n_features = X_train.shape[1]
model = models.Sequential()

model.add(layers.Dense(n_features, input_dim =  n_features, activation = "relu"))
#model.add(layers.Dense(64, activation = "relu"))
#model.add(layers.Dense(32, activation = "relu"))
#model.add(layers.Dense(128, activation = "elu"))
model.add(layers.Dense(n_classes, activation="softmax"))
model.compile(optimizer='adam', 
              loss='sparse_categorical_crossentropy',
              metrics=["acc"])
history = model.fit(X_train.values, encoded_y_train, 
                    batch_size = 16,
                    epochs = 10,
                    class_weight = class_weights, 
                    validation_split = 0.3)
x_plot = list(range(1,history.epoch[-1]+2))

def plot_history(network_history):
    plt.figure()
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.plot(x_plot, network_history.history['loss'])
    plt.plot(x_plot, network_history.history['val_loss'])
    plt.legend(['Training', 'Validation'])
    
    plt.figure()
    plt.xlabel('Epochs')
    plt.ylabel('Accuracy')
    plt.plot(x_plot, network_history.history['acc'])
    plt.plot(x_plot, network_history.history['val_acc'])
    plt.legend(['Training', 'Validation'])
    
plot_history(history)
y_pred_prob = model.predict(X_test.values)
ndcg_score(encoded_y_test, y_pred_prob)<jupyter_output><empty_output><jupyter_text>NN one_vs_all (Provata perchè inizialmente non funzionava la rete neurale classica, ora quindi non è più necessaria)<jupyter_code>data_prova = data.copy()
data_prova.drop("id", axis = 1, inplace = True)
X_train, X_test = train_test_split(data_prova, test_size = 0.2, stratify = data_prova["country_destination"], random_state = 42)
y_train = X_train.pop("country_destination")
y_test = X_test.pop("country_destination")
label_encoder = LabelEncoder()

encoded_y_train = label_encoder.fit_transform(y_train)
encoded_y_test = label_encoder.fit_transform(y_test)
n_classes = len(np.unique(y_train))
n_features = X_train.shape[1]
def dnn_model(n_comps):
    model = models.Sequential()
    model.add(layers.Dense(n_comps, input_dim=n_comps, activation='relu'))
    #model.add(layers.Dropout(0.3))
    model.add(layers.Dense(128, activation="relu"))
    #model.add(layers.Dropout(0.3))
    model.add(layers.Dense(2, activation='softmax'))

    # compile model
    model.compile(loss="sparse_categorical_crossentropy", 
                  optimizer="adam", 
                  metrics=[metrics.accuracy])
    return model

model = KerasClassifier(build_fn=dnn_model, n_comps=n_features, 
                        batch_size = 16,
                        epochs = 1,
                        validation_split = 0.3)
clf = OneVsRestClassifier(model)
clf.fit(X_train.values, encoded_y_train)
y_pred_prob = clf.predict_proba(X_test.values)
ndcg_score(encoded_y_test, y_pred_prob, k = 5)
top_n_accuracy(encoded_y_test, y_pred_prob)
#Cross validation

cval_scores = cross_val_score(clf, X_train.values, encoded_y_train, cv=5, scoring=ndcg_scorer)
test_score = clf.score(X=X_test.values, y=encoded_y_test)
cval_scores<jupyter_output><empty_output><jupyter_text>Random Forest (basato su decision tree come XGBoost), più lento e meno performante, non verrà sviluppato<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.preprocessing import LabelEncoder, LabelBinarizer
clf = RandomForestClassifier(n_estimators=100, max_depth=12, random_state=42)
clf.fit(X_train, y_train)
rf_prob = clf.predict_proba(X_test)
ndcg_score(encoded_y_test, rf_prob, k = 5)
top_n_accuracy(encoded_y_test, rf_prob)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12728174603174602, 'fraction_numerical': 0.013888888888888888, 'mean_word_length': 5.184662576687116, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '343747', 'license': 'CC0-1.0', 'n_tokens_mistral': 3816, 'n_tokens_neox': 3658, 'n_words': 572, 'path': 'AML/Project/Models.ipynb', 'repo_name': 'mode95/Data-Science', 'size': '10080'}