<jupyter_start><jupyter_text><jupyter_code># Colab console code to not disconnect - run it in the JS console
"""
function ClickConnect(){
console.log("Working"); 
document.querySelector("colab-toolbar-button#connect").click() 
}setInterval(ClickConnect,60000)
"""
# Start logging time
import time
start_time = time.time()
COLAB_IDX = 1
TEMPERATURE = "t02"
FORMAT = "repeat"
TESTING = False
COLAB = False

if COLAB:
    BASE_DIR = "/content/drive/My Drive/IC/mosquito-networking/"
else:
    BASE_DIR = "../"
if COLAB:
    from google.colab import drive
    drive.flush_and_unmount()
    drive.mount('/content/drive', force_remount=True)
import numpy as np
import pandas as pd
import joblib

from sklearn.metrics import classification_report, accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV, RepeatedKFold
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
np.random.seed(42)<jupyter_output><empty_output><jupyter_text>- - -   load data<jupyter_code>data = np.load(BASE_DIR + f"data/interim/all_wavs_{FORMAT}_{TEMPERATURE}_no_std.npz",
               allow_pickle=True)
# Parse and divide data
train_data = data["train"]
val_data = data["val"]
test_data = data["test"]

x_train = train_data[:, :-1]
x_val = val_data[:, :-1]
x_test = test_data[:, :-1]

y_train = train_data[:, -1]
y_val = val_data[:, -1]
y_test = test_data[:, -1]<jupyter_output><empty_output><jupyter_text>No need to scale data for LR  Prepare model and train functionWe can't run a normal GridSearchCV since we need to change the solver in case of l1/l2<jupyter_code>cv = RepeatedKFold(n_splits=5)
%%time
params = {"C": [0.1, 1, 10]}
solvers = ["liblinear", "lbfgs", "lbfgs"]
penalties = ["l1", "l2", "none"]

best = None
score = 0

for solver, penalty in zip(solvers, penalties):
    if penalty != "none":
        grid = GridSearchCV(LogisticRegression(solver=solver), 
                            params, cv=cv)
    else:
        grid = GridSearchCV(LogisticRegression(solver=solver, n_jobs=8), 
                            {}, cv=cv)
    grid.fit(x_train, y_train)
    if grid.best_score_ > score:
        score = grid.best_score_
        best = grid.best_params_
        best["penalty"] = penalty
        best["solver"] = solver

print(f"Best score was {score} with parameters {best}")
MODEL = "log_reg"
model_name = f"{MODEL}_{FORMAT}_{TEMPERATURE}"<jupyter_output><empty_output><jupyter_text>Retrain with best parameters<jupyter_code>model = LogisticRegression(**best, n_jobs=8)
model.fit(x_train, y_train)
for test_x, test_y, name in [(x_train, y_train, "train"), 
                             (x_val, y_val, "val"), 
                             (x_test, y_test, "test")]:
    print(f"For {name}, the results are:")
    preds = model.predict(test_x)
    print(classification_report(test_y, preds))
    print(f"And total acc is {accuracy_score(test_y, preds)}")
    print("\n\n")<jupyter_output>For train, the results are:
              precision    recall  f1-score   support

           0       1.00      1.00      1.00      1525
           1       1.00      1.00      1.00      1447

    accuracy                           1.00      2972
   macro avg       1.00      1.00      1.00      2972
weighted avg       1.00      1.00      1.00      2972

And total acc is 1.0



For val, the results are:
              precision    recall  f1-score   support

           0       0.65      0.73      0.69       347
           1       0.74      0.65      0.69       396

    accuracy                           0.69       743
   macro avg       0.69      0.69      0.69       743
weighted avg       0.70      0.69      0.69       743

And total acc is 0.6904441453566622



For test, the results are:
              precision    recall  f1-score   support

           0       0.54      0.60      0.57      1760
           1       0.56      0.50      0.53      1808

    accuracy                          [...]<jupyter_text>Save model<jupyter_code>joblib.dump(model, f"{BASE_DIR}models/{model_name}.joblib")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11026709139916688, 'fraction_numerical': 0.048027444253859346, 'mean_word_length': 2.42162615255658, 'pattern_counts': {'":': 2, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '176838', 'license': 'FTL', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1385, 'n_words': 343, 'path': 'notebooks_tcc/1.7-BrunoGomesCoelho_Logistic_regression.ipynb', 'repo_name': 'BrunoGomesCoelho/mosquito-networking', 'size': '4081'}