<jupyter_start><jupyter_text>Prepare data set for use in models1. One Hot Encode categorical features2. Split data into test and training sets3. Prepare StandardScaler for use in models<jupyter_code>df = pd.read_csv("../data/01_raw/heart.csv")
df_encoded = df.copy()
df_encoded.to_csv("../data/02_interim/encoded.csv")
df_unencoded = df.copy()
df_unencoded.to_csv("../data/02_interim/unencoded.csv")
onehot = OneHotEncoder()
transformer = ColumnTransformer([('onehot', onehot, cat_cols)],remainder='passthrough' )
df_encoded = transformer.fit_transform(df_encoded)
df_encoded = pd.DataFrame(df_encoded, columns=transformer.get_feature_names())
df_encoded.to_csv("../data/02_interim/encoded.csv")
df_encoded.shape
features = df_encoded.iloc[:,0:30]
target = df_encoded['output']
features_unenc = df_unencoded.iloc[:,0:13]
target_unenc = df_unencoded['output']
features_unenc
features.to_csv("../data/02_interim/features.csv")
target.to_csv("../data/02_interim/target.csv")
X_train, X_test, y_train, y_test = train_test_split(features, target, train_size=0.80, stratify=target, random_state=10)
print("X_train:", X_train.shape)
print("X_test:", X_test.shape)
print("y_train:", y_train.shape)
print("y_test:", y_test.shape)
X_train.to_csv("../data/03_processed/X_train.csv")
X_test.to_csv("../data/03_processed/X_test.csv")
y_train.to_csv("../data/03_processed/y_train.csv")
y_test.to_csv("../data/03_processed/y_test.csv")
X_train_unenc, X_test_unenc, y_train_unenc, y_test_unenc = train_test_split(features_unenc, target_unenc, train_size=0.80, stratify=target, random_state=10)
X_train_unenc.to_csv("../data/03_processed/X_train_unenc.csv")
X_test_unenc.to_csv("../data/03_processed/X_test_unenc.csv")
y_train_unenc.to_csv("../data/03_processed/y_train_unenc.csv")
y_test_unenc.to_csv("../data/03_processed/y_test_unenc.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15525606469002695, 'fraction_numerical': 0.025336927223719677, 'mean_word_length': 7.474885844748859, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '249295', 'license': 'MIT', 'n_tokens_mistral': 749, 'n_tokens_neox': 718, 'n_words': 90, 'path': 'notebooks/02_DataPrep.ipynb', 'repo_name': 'tlchampion/hattack', 'size': '1855'}