<jupyter_start><jupyter_text>Dataset cleansing<jupyter_code>import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>1. Load the (standardized) Data<jupyter_code>patients = pd.read_csv("../../Data/standardized_patients.csv",index_col=0, header=0)
patients.dropna(inplace=True)
patients.head()
patients = patients.sample(frac=1).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>2. Create the Three dataset possible: Men, Women not in Menopause condition, Women in Menopause condition  2.1 Men<jupyter_code>male_patients = patients[patients['sex']>0]
male_patients.drop(columns=['menopause','HRT','sex'], axis=1, inplace=True)
male_patients.head()<jupyter_output><empty_output><jupyter_text>Create the X and y<jupyter_code>male_patients_X = male_patients.copy()
male_patients_X.drop("Class", axis=1, inplace=True)
print(male_patients.shape)

male_patients_y = male_patients[['Class']].copy()
print(male_patients_y.shape)<jupyter_output>(79177, 14)
(79177, 1)<jupyter_text>Split the data into Train, Test and Validation The ratio in the splitting, in percentage, is 60/20/20<jupyter_code>male_patients_X_train_and_val, male_patients_X_test, male_patients_y_train_and_val, male_patients_y_test = train_test_split(male_patients_X, male_patients_y, test_size=0.2, random_state=42)
male_patients_X_train, male_patients_X_val, male_patients_y_train, male_patients_y_val = train_test_split(male_patients_X_train_and_val, male_patients_y_train_and_val, test_size=0.25, random_state=42)
male_patients_y_val[male_patients_y_val['Class']==1].shape
male_patients_y_train[male_patients_y_train['Class']==1].shape
male_patients_y_test[male_patients_y_test['Class']==1].shape<jupyter_output><empty_output><jupyter_text>Save to Files<jupyter_code>patients.to_csv("../../Data/male_patients/standardized_patients.csv")

male_patients_X_train.to_csv("../../Data/male_patients/X_train_total.csv")
male_patients_y_train.to_csv("../../Data/male_patients/y_train_total.csv")

male_patients_X_test.to_csv("../../Data/male_patients/X_test.csv")
male_patients_y_test.to_csv("../../Data/male_patients/y_test.csv")

male_patients_X_val.to_csv("../../Data/male_patients/X_val.csv")
male_patients_y_val.to_csv("../../Data/male_patients/y_val.csv")<jupyter_output><empty_output><jupyter_text>2.2 Female patients not in Menopause condition<jupyter_code>female_patients_no_menopause = patients[patients['menopause']<1]
female_patients_no_menopause = female_patients_no_menopause[female_patients_no_menopause['sex']<0]
female_patients_no_menopause.drop(columns=['menopause','HRT','sex'], axis=1, inplace=True)
female_patients_no_menopause.head()<jupyter_output><empty_output><jupyter_text>Create the X and y<jupyter_code>female_patients_no_menopause_X = female_patients_no_menopause.copy()
female_patients_no_menopause_X.drop("Class", axis=1, inplace=True)
print(female_patients_no_menopause.shape)

female_patients_no_menopause_y = female_patients_no_menopause[['Class']].copy()
print(female_patients_no_menopause_y.shape)<jupyter_output>(24492, 14)
(24492, 1)<jupyter_text>Split the data into Train, Test and Validation The ratio in the splitting, in percentage, is 60/20/20<jupyter_code>female_patients_no_menopause_X_train_and_val, female_patients_no_menopause_X_test, female_patients_no_menopause_y_train_and_val, female_patients_no_menopause_y_test = train_test_split(female_patients_no_menopause_X, female_patients_no_menopause_y, test_size=0.2, random_state=16)
female_patients_no_menopause_X_train, female_patients_no_menopause_X_val, female_patients_no_menopause_y_train, female_patients_no_menopause_y_val = train_test_split(female_patients_no_menopause_X_train_and_val, female_patients_no_menopause_y_train_and_val, test_size=0.25, random_state=16)
female_patients_no_menopause_y_val[female_patients_no_menopause_y_val['Class']==1].shape
female_patients_no_menopause_y_train[female_patients_no_menopause_y_train['Class']==1].shape
female_patients_no_menopause_y_test[female_patients_no_menopause_y_test['Class']==1].shape<jupyter_output><empty_output><jupyter_text>Save to Files<jupyter_code>patients.to_csv("../../Data/female_patients_no_menopause/standardized_patients.csv")

female_patients_no_menopause_X_train.to_csv("../../Data/female_patients_no_menopause/X_train_total.csv")
female_patients_no_menopause_y_train.to_csv("../../Data/female_patients_no_menopause/y_train_total.csv")

female_patients_no_menopause_X_test.to_csv("../../Data/female_patients_no_menopause/X_test.csv")
female_patients_no_menopause_y_test.to_csv("../../Data/female_patients_no_menopause/y_test.csv")

female_patients_no_menopause_X_val.to_csv("../../Data/female_patients_no_menopause/X_val.csv")
female_patients_no_menopause_y_val.to_csv("../../Data/female_patients_no_menopause/y_val.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11612638489946656, 'fraction_numerical': 0.01600328272466147, 'mean_word_length': 11.828947368421053, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '162142', 'license': 'MIT', 'n_tokens_mistral': 2018, 'n_tokens_neox': 1765, 'n_words': 172, 'path': '1. Data Preprocessing/2. Data Preparation/3.3.1. Data_splitting (Male,Female in menopause, Female non Menopause and Train, Test, and Validation).ipynb', 'repo_name': 'TimeParadox89/MSc-Thesis', 'size': '4874'}