<jupyter_start><jupyter_text>Splitting<jupyter_code>#Adding categorical age column for proportional/stratified train/test split
AGE_GROUP_AMOUNT = 8
full_df['AGE_GROUP'] = pd.cut(full_df['AGE'], bins=AGE_GROUP_AMOUNT,
                              labels=range(AGE_GROUP_AMOUNT))

#Making train test split with proportional age groups
split = StratifiedShuffleSplit(n_splits=1, test_size=0.1,
                               random_state=RAND_STATE)
for train_index, test_index in split.split(full_df, full_df['AGE_GROUP']):
    strat_train_set = full_df.iloc[train_index]
    strat_test_set = full_df.iloc[test_index]
#TODO: implment StandardScaler in pytorch (take a look at nn.Flatten) and maybe the whole preprocessing pipeline in PyTorch
drop_cols = ['SEQN', 'AGE_GROUP', 'AGE', 'AlkPhos_UL']
cat_cols = ["GENDER"]

full_transform = ColumnTransformer([
    ('encoder', OrdinalEncoder(), cat_cols),
    ('scaler', StandardScaler(), [x for x in strat_train_set.columns if x not in drop_cols + cat_cols]),
])
x = full_transform.fit_transform(strat_train_set)
pd.DataFrame(x, columns=[x for x in strat_train_set.columns if x not in drop_cols])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10485933503836317, 'fraction_numerical': 0.0034100596760443308, 'mean_word_length': 5.051546391752577, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '136059', 'license': 'MIT', 'n_tokens_mistral': 407, 'n_tokens_neox': 395, 'n_words': 92, 'path': 'PyTorch.ipynb', 'repo_name': 'cocosjsu/AgingClock', 'size': '1173'}