<jupyter_start><jupyter_text>Data location<jupyter_code>data_path = "data/clean_data.csv"
save_directory = "current_accuracies/logistic_regression/logistic_regression_weights"
accuracies_file_path = "current_accuracies/logistic_regression/LogisticRegression_accuracies.pkl"

if not os.path.isdir(save_directory):
    os.mkdir(save_directory)<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>df = pd.read_csv(data_path)
df.shape
df.columns
df.iloc[:, 1: 23].columns, df.iloc[:, 23: 31].columns
print(df.iloc[:, 1: 23].shape[1], df.iloc[:, 23: 31].shape[1])
data = np.loadtxt(data_path, delimiter=",", dtype=np.float32, skiprows=1)
amax_label = np.amax(data[:, 23:31], axis=1)
hist,bins = np.histogram(amax_label) 

plt.hist(amax_label, bins = bins) 
plt.title("any disease") 
plt.show()
df<jupyter_output><empty_output><jupyter_text>Prepare the data<jupyter_code>TEST_SPLIT = 0.2
VALIDATION_SPLIT = 0.21
TRAIN_SPLIT = 1 - TEST_SPLIT - VALIDATION_SPLIT

split_proportions = [TRAIN_SPLIT, VALIDATION_SPLIT, TEST_SPLIT]<jupyter_output><empty_output><jupyter_text>Data verification<jupyter_code>heart_dataset = HeartDiseaseDataset(data_path, any_disease=False)
train_dataloader_verif, val_dataloader_verif, test_dataloader_verif = create_dataloaders(heart_dataset, batch_size=1, split_proportions=split_proportions)
participant = next(iter(train_dataloader_verif))
print('input ==>\n', participant[0])
print('label ==>\n', participant[1])
nb_epochs = 10
trainers = train_labels(df.iloc[:, 23: 31].columns, data_path, 
                        df.iloc[:, 23: 31].columns, split_proportions, 
                        save_directory, nb_epochs)<jupyter_output>==== Training DEATH ====

Epoch 1 / 10
6860/6860 [==============================] - 8s 1ms/step - loss: 0.2894 - acc: 0.7137 - lr: 0.0500
Validation
2442/2442 [==============================] - 2s 888us/step - loss: 0.3429 - acc: 0.7525 - lr: 0.0500
Model saved. Acc updated: -inf -> 0.757
Epoch 2 / 10
6860/6860 [==============================] - 8s 1ms/step - loss: 0.2637 - acc: 0.7378 - lr: 0.0500
Validation
2442/2442 [==============================] - 2s 894us/step - loss: 0.0277 - acc: 0.7697 - lr: 0.0500
Model saved. Acc updated: 0.757 -> 0.766
Epoch 3 / 10
6860/6860 [==============================] - 8s 1ms/step - loss: 0.2529 - acc: 0.7610 - lr: 0.0500
Validation
2442/2442 [==============================] - 2s 904us/step - loss: 0.0912 - acc: 0.7491 - lr: 0.0500
Epoch 4 / 10
6860/6860 [==============================] - 8s 1ms/step - loss: 0.2485 - acc: 0.7697 - lr: 0.0500
Validation
2442/2442 [==============================] - 2s 948us/step - loss: 0.0842 - acc: 0.7884 - lr: 0.05[...]<jupyter_text>Test<jupyter_code>accuracies = prediction_analyse_labels(df.iloc[:, 23: 31].columns, trainers, df.iloc[:, 23: 31].columns, data_path, split_proportions)
save_accuracies_pkl(accuracies_file_path, accuracies)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21553198768388643, 'fraction_numerical': 0.09613410879233664, 'mean_word_length': 5.384279475982533, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '297471', 'license': 'MIT', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1115, 'n_words': 201, 'path': 'logistic_regression.ipynb', 'repo_name': 'ninous-dev/heart-prediction', 'size': '2923'}