<jupyter_start><jupyter_text>Load the training data into feature matrix, class labels, and event ids:<jupyter_code>y, tX, ids = load_csv_data(DATA_TRAIN_PATH)<jupyter_output><empty_output><jupyter_text>Analyse Data Load data and format it<jupyter_code>pd.set_option('display.max_rows', 40)
pd.set_option('display.max_columns', 100)
pd.set_option('display.width', 1000)
data = pd.read_csv(DATA_TRAIN_PATH, header=0, na_values='-999', delimiter=',')
COLUMNS_NAMES = ['DER_mass_MMC', 'DER_mass_transverse_met_lep', 'DER_mass_vis', 'DER_pt_h', 'DER_deltaeta_jet_jet', 'DER_mass_jet_jet', 'DER_prodeta_jet_jet', 'DER_deltar_tau_lep', 'DER_pt_tot', 'DER_sum_pt', 'DER_pt_ratio_lep_tau', 'DER_met_phi_centrality', 'DER_lep_eta_centrality', 'PRI_tau_pt', 'PRI_tau_eta', 'PRI_tau_phi', 'PRI_lep_pt', 'PRI_lep_eta', 'PRI_lep_phi', 'PRI_met', 'PRI_met_phi', 'PRI_met_sumet', 'PRI_jet_num', 'PRI_jet_leading_pt', 'PRI_jet_leading_eta', 'PRI_jet_leading_phi', 'PRI_jet_subleading_pt', 'PRI_jet_subleading_eta', 'PRI_jet_subleading_phi', 'PRI_jet_all_pt']
indicesS = data[data['Prediction'].str.contains('s')].index
indicesB = pd.Int64Index(np.arange(len(data))).difference(indicesS)
data.loc[indicesS, 'Prediction'] = np.ones(len(indicesS))
data.loc[indicesB, 'Prediction'] = np.ones(len(indicesB))-2<jupyter_output><empty_output><jupyter_text>Data distribution analysis<jupyter_code>display(data.head())
display(data.describe())<jupyter_output><empty_output><jupyter_text>Null values analysis<jupyter_code>sns.heatmap(data.isna())<jupyter_output><empty_output><jupyter_text>Plot Correlation Matrix<jupyter_code>corr = data.drop(['Id'], axis=1).corr()
fig, (ax) = plt.subplots(1, 1, figsize=(15,10))

hm = sns.heatmap(corr, 
                 ax=ax,           # Axes in which to draw the plot, otherwise use the currently-active Axes.
                 cmap="coolwarm", # Color Map.
                 #square=True,    # If True, set the Axes aspect to “equal” so each cell will be square-shaped.
                 annot=True, 
                 fmt='.2f',       # String formatting code to use when adding annotations.
                 #annot_kws={"size": 14},
                 linewidths=.05)

fig.subplots_adjust(top=0.93)
fig.suptitle('Wine Attributes Correlation Heatmap', 
              fontsize=14, 
              fontweight='bold')<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code>data[~data.isna()].drop(['Id'], axis = 1).boxplot(figsize=(20, 10), whis=2.5)<jupyter_output><empty_output><jupyter_text>Clean the data<jupyter_code>yC, tXC, idsC = prepareData(y, tX, ids)<jupyter_output><empty_output><jupyter_text>Parameter Exploration (K-fold Cross validation) Simple cross validation on 1 parameter<jupyter_code>from mpl_toolkits.mplot3d import Axes3D  # noqa: F401 unused import
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import numpy as np
from plot import *

# Parameters
seed = 1
degree = 10
lambda_ = 1e-6 
degrees = range(1, 13)
lambdas = np.logspace(-12, -1, 12)
k_fold = 4

# define lists to store the accuracy of training data and test data
accuracies_tr = []
accuracies_te = []

# Test for each degree
for degree in degrees:
    accuracy_tr, accuracy_te = cross_validate(yC[0], tXC[0], k_fold, lambda_, degree, seed)
    accuracies_tr.append(accuracy_tr)
    accuracies_te.append(accuracy_te)

cross_validation_visualization(degrees, accuracies_tr, accuracies_te)<jupyter_output><empty_output><jupyter_text>Cross validation on degree and lambda<jupyter_code># Make data.
values = pd.DataFrame(columns=["Degree", "Lambda", "Accuracy"])
degrees = np.array(range(2, 13))
lambdas = np.logspace(-12, -1, 12)

for degree in degrees: 
    for lambda_ in lambdas:
        accuracy = cross_validate(yC[0], tXC[0], k_fold, lambda_, degree, seed)[0]
        values = values.append({'Degree' : degree , 'Lambda' : lambda_, 'Accuracy': accuracy}, ignore_index=True)

plot_3_variables(values)<jupyter_output><empty_output><jupyter_text>Testing Clean Data and split Dataset<jupyter_code># Models parameters
degree = 13
whis = 2.5

# Prepare data
clean_yC, clean_tX, clean_ids = prepareData(y, tX, ids, degree, whis)

# Split the data in test and train sets
txC_train = []
yC_train = []
txC_test = []
yC_test = []

for i in range(len(clean_tX)):
    tX_train, y_train, tX_validation, y_validation, tX_test, y_test = splitData(clean_yC[i], clean_tX[i], [0.7, 0.1])
    txC_train.append(tX_train)
    yC_train.append(y_train)
    txC_test.append(tX_test)
    yC_test.append(y_test)<jupyter_output><empty_output><jupyter_text>Test the models<jupyter_code># Models parameters
max_iters = 1000
gamma = 1e-4
lambda_ =  1e-4
whis = 2.5
gamma = 1e-9

initial_w = np.zeros(txC_train[0].shape[1])
weights_0, loss_0 = ridge_regression(yC_train[0], txC_train[0], lambda_)
gamma = 1e-9
initial_w = np.zeros(txC_train[1].shape[1])
weights_1, loss_1 = ridge_regression(yC_train[1], txC_train[1], lambda_)
gamma = 1e-9
initial_w = np.zeros(txC_train[2].shape[1])
weights_2, loss_2 = ridge_regression(yC_train[2], txC_train[2], lambda_)
gamma = 1e-8
initial_w = np.zeros(txC_train[3].shape[1])
weights_3, loss_3 = ridge_regression(yC_train[3], txC_train[3], lambda_)
                                     
#ridge_regression(yC_train[0], txC_train[0], lambda_)
#least_squares_GD(yC_train[0], txC_train[0], initial_w, max_iters, gamma)
#least_squares_SGD(yC_train[0], txC_train[0], initial_w, max_iters, gamma)
#least_squares(yC_train[0], txC_train[0])
#logistic_regression(yC_train[0], txC_train[0], initial_w, max_iters, gamma)
#reg_logistic_regression(yC_train[0], txC_train[0], lambda_, initial_w, max_iters, gamma)

predictions_0 = predict_labels(weights_0, txC_test[0])
predictions_1 = predict_labels(weights_1, txC_test[1])
predictions_2 = predict_labels(weights_2, txC_test[2])
predictions_3 = predict_labels(weights_3, txC_test[3])

print("", classification_report(yC_test[0],predictions_0))
print("", classification_report(yC_test[1],predictions_1))
print("", classification_report(yC_test[2],predictions_2))
print("", classification_report(yC_test[3],predictions_3))

y_pred = np.concatenate([predictions_0, predictions_1, predictions_2, predictions_3])
y_true = np.concatenate([yC_test[0], yC_test[1], yC_test[2], yC_test[3]])
print("Total: ", accuracy_score(y_true, y_pred))<jupyter_output>precision    recall  f1-score   support

        -1.0       0.86      0.93      0.90     14898
         1.0       0.73      0.57      0.64      5085

    accuracy                           0.84     19983
   macro avg       0.80      0.75      0.77     19983
weighted avg       0.83      0.84      0.83     19983

               precision    recall  f1-score   support

        -1.0       0.83      0.87      0.85      9964
         1.0       0.75      0.69      0.72      5546

    accuracy                           0.81     15510
   macro avg       0.79      0.78      0.79     15510
weighted avg       0.80      0.81      0.80     15510

               precision    recall  f1-score   support

        -1.0       0.84      0.82      0.83      4949
         1.0       0.83      0.85      0.84      5128

    accuracy                           0.84     10077
   macro avg       0.84      0.84      0.84     10077
weighted avg       0.84      0.84      0.84     10077

               p[...]<jupyter_text>Final Model<jupyter_code>lambda_ = 0.0001
degree = 13
whis = 2.5

clean_yC, clean_tX, clean_ids = prepareData(y, tX, ids, degree, whis)
    
weights_0, loss_0 = ridge_regression(clean_yC[0], clean_tX[0], lambda_)
weights_1, loss_1 = ridge_regression(clean_yC[1], clean_tX[1], lambda_)
weights_2, loss_2 = ridge_regression(clean_yC[2], clean_tX[2], lambda_)
weights_3, loss_3 = ridge_regression(clean_yC[3], clean_tX[3], lambda_)

predictions_0 = predict_labels(weights_0, clean_tX[0])
predictions_1 = predict_labels(weights_1, clean_tX[1])
predictions_2 = predict_labels(weights_2, clean_tX[2])
predictions_3 = predict_labels(weights_3, clean_tX[3])

print("", classification_report(clean_yC[0],predictions_0))
print("", classification_report(clean_yC[1],predictions_1))
print("", classification_report(clean_yC[2],predictions_2))
print("", classification_report(clean_yC[3],predictions_3))<jupyter_output><empty_output><jupyter_text>Generate predictions and save ouput in csv format for submission:<jupyter_code># Load the dataset to predict
y_test, tX_test, ids_test = load_csv_data(DATA_TEST_PATH)
# Prepare the data in the same way as the train dataset
test_yC, test_tX, test_ids = prepareData(y_test, tX_test, ids_test, degree, whis)
# Predict each class 
y_pred_0 = predict_labels(weights_0, test_tX[0])
y_pred_1 = predict_labels(weights_1, test_tX[1])
y_pred_2 = predict_labels(weights_2, test_tX[2])
y_pred_3 = predict_labels(weights_3, test_tX[3])
y_pred = np.concatenate([y_pred_0, y_pred_1, y_pred_2, y_pred_3])
test_ids = np.concatenate([test_ids[0], test_ids[1], test_ids[2], test_ids[3]])
# Output the results in a csv
create_csv_submission(test_ids, y_pred, OUTPUT_PATH)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12322431450280806, 'fraction_numerical': 0.047461733289285324, 'mean_word_length': 4.477683956574186, 'pattern_counts': {'":': 1, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '17478', 'license': 'MIT', 'n_tokens_mistral': 3622, 'n_tokens_neox': 3431, 'n_words': 650, 'path': 'scripts/project1.ipynb', 'repo_name': 'X4l1b1/ML_Project1', 'size': '9081'}