<jupyter_start><jupyter_text>Preprocessing<jupyter_code>import numpy as np
import pandas as pd
import missingno as msno
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
import os<jupyter_output><empty_output><jupyter_text>Inspect Data<jupyter_code>directory = os.getcwd()
data = pd.read_csv('patientdata.csv', na_values=["na", "--", "unknown"], index_col='patient_id')
data

msno.matrix(data)<jupyter_output><empty_output><jupyter_text>Handling Missing Data<jupyter_code>def missing_mean(column_name, missing_data):
    imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
    missing_data[column_name] = imp_mean.fit_transform(missing_data[column_name].values.reshape(-1, 1)).astype(int)

def missing_constant(constant, column_name, missing_data):
    imp_constant = SimpleImputer(missing_values=np.nan, strategy='constant', fill_value=constant)
    missing_data[column_name] = imp_constant.fit_transform(missing_data[column_name].values.reshape(-1, 1))

# city
data = data.drop('city', axis=1)

# gender 
data = data.replace({'gender': {'male': 'm', 'female': 'f'}})
data = data[(data['gender'] == 'm') | (data['gender'] == 'f')]

# age_yrs
missing_mean('age_yrs', data)

# weight_kg
missing_mean('weight_kg', data)

# height_cm
missing_mean('height_cm', data)

# diabetes
missing_constant('no', 'diabetes', data)

# heart_disease
missing_constant('no', 'heart_disease', data)

# lung_disease
missing_constant('no', 'lung_disease', data)

msno.matrix(data)<jupyter_output><empty_output><jupyter_text>OneHotEncoder<jupyter_code>def encode_strings(column_name, data):    
    label_encoder = LabelEncoder()
    data[column_name] = label_encoder.fit_transform(data[column_name])

# gender
encode_strings('gender', data)    

# diabetes
encode_strings('diabetes', data)

# heart_disease
encode_strings('heart_disease', data)

# lung_disease
encode_strings('lung_disease', data)

data<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>data_scaled = preprocessing.scale(data)
df = pd.DataFrame(data_scaled, 
        index=data.index,
        columns=data.columns)
df<jupyter_output><empty_output><jupyter_text>Export to Excel<jupyter_code>df.to_excel('patients.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13394415357766143, 'fraction_numerical': 0.002181500872600349, 'mean_word_length': 8.359183673469389, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '142555', 'license': 'MIT', 'n_tokens_mistral': 828, 'n_tokens_neox': 804, 'n_words': 121, 'path': '2_ Preprocessing/patients/patients.ipynb', 'repo_name': 'DavidStahl97/Predictive-Analytics', 'size': '2292'}