<jupyter_start><jupyter_text>Convert input data to desired format<jupyter_code>import pickle
import pandas as pd
import numpy as np
from collections import Counter
regiondata_url = "./gmql_data/Datasets/gs_breast_finale.pickle"
regiondata = pickle.load(open(regiondata_url, "rb"))
regiondata<jupyter_output><empty_output><jupyter_text>Get labels from the index<jupyter_code>lbls = regiondata.index.get_level_values('manually_curated__tissue_status')
len(lbls)
lbls= ["1" if x=="tumoral" else "0" for x in lbls]
for i in range(len(lbls)):
    lbls[i] = int(lbls[i])<jupyter_output><empty_output><jupyter_text>Dump the unnecessery columns<jupyter_code>regiondata = regiondata.normalized_count
regiondata.index = regiondata.index.get_level_values(-1)
regiondata.head()
regiondata = regiondata.fillna(regiondata.mean())
regiondata = regiondata.dropna(axis=1, how='all')
regiondata
regiondata = (regiondata - regiondata.mean())
regiondata = regiondata / (regiondata.max() - regiondata.min())
regiondata<jupyter_output><empty_output><jupyter_text>Save base data to pickel file<jupyter_code>pickle_file = "./gmql_data/Datasets/breast_data_all_w_labels.p"

try:
    
    f = open(pickle_file, 'wb')
    save = {
    'data': regiondata,
    'labels': lbls,

    }
    pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
    f.close()
except Exception as e:
    print('Unable to save data to', pickle_file, ':', e)
    raise<jupyter_output><empty_output><jupyter_text>Split the data for train and test<jupyter_code>from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import StratifiedShuffleSplit
import pickle
import numpy as np
pickle_file = "./gmql_data/Datasets/breast_data_all_w_labels.p"
with open(pickle_file, 'rb') as f:
    save = pickle.load(f)
    data = save['data']
    labels = save['labels']
    del save  # hint to help gc free up memory<jupyter_output><empty_output><jupyter_text>Set labels to y parameter<jupyter_code>x = np.array(data)
y = np.array(labels)
sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
sss.get_n_splits(x, y)
sss
for train_index, test_index in sss.split(x, y):
    x_train, x_test = x[train_index], x[test_index]
    y_train, y_test = y[train_index], y[test_index]
x_train
x_test
y_train
y_test<jupyter_output><empty_output><jupyter_text>Check if the input raito is preserved. Check wit y data since its easier to see Create validation dataset from training data<jupyter_code>x = x_train
z = y_train

sss = StratifiedShuffleSplit(n_splits=3, test_size=0.25, random_state=0)
sss.get_n_splits(x, z)
sss
for train_index, valid_index in sss.split(x, z):
    x_train, x_valid = x[train_index], x[valid_index]
    z_train, z_valid = z[train_index], z[valid_index]
x_valid<jupyter_output><empty_output><jupyter_text>Save the data for future use<jupyter_code>pickle_file = "./gmql_data/Datasets/breast_data_all_divided_4.pickle"

try:
  f = open(pickle_file, 'wb')
  save = {
    'train_dataset': x_train,
    'train_labels': z_train,
    'test_dataset': x_test,
    'test_labels': y_test,
    'valid_dataset': x_valid,
    'valid_labels': z_valid,
    }
  pickle.dump(save, f, pickle.HIGHEST_PROTOCOL)
  f.close()
except Exception as e:
  print('Unable to save data to', pickle_file, ':', e)
  raise<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11434637549287231, 'fraction_numerical': 0.004246284501061571, 'mean_word_length': 5.943157894736842, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '388831', 'license': 'MIT', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1255, 'n_words': 257, 'path': 'PrepareDataSet.ipynb', 'repo_name': 'acanakoglu/genomic-ladder', 'size': '3297'}