<jupyter_start><jupyter_text>MELANOMA<jupyter_code># experiment source
experiment = 'exper_melanoma'
location = 'external'
# the output location
loc_output = os.path.join(src.DIR_DATA, location, experiment)
src.define_folder(loc_=loc_output)<jupyter_output>FOLDER information,  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_melanoma/<jupyter_text>reference<jupyter_code>dataset = 'reference.rds'
df_raw = pyreadr.read_r(os.path.join(src.DIR_DATA, location, experiment, dataset))[None]
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))
print('Experiment datasets are exported into ', os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_melanoma/reference.pck<jupyter_text>query<jupyter_code>dataset = 'query.rds'
df_raw = pyreadr.read_r(os.path.join(src.DIR_DATA, location, experiment, dataset))[None]
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))
print('Experiment datasets are exported into ', os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_melanoma/query.pck<jupyter_text>MOUSE<jupyter_code># experiment source
experiment = 'exper_mouse'
location = 'external'
# the output location
loc_output = os.path.join(src.DIR_DATA, location, experiment)
src.define_folder(loc_=loc_output)<jupyter_output>FOLDER information,  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_mouse/<jupyter_text>learning<jupyter_code>df_raw = pd.read_csv(os.path.join(src.DIR_DATA, location, experiment, '1-3_integrated_NNtraining.txt')
                             , sep='\t'
                             , index_col=0).T.drop(columns='Weight')#.reset_index()
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, '1-3_integrated_NNtraining.pck'))
print('Experiment datasets are exported into ', os.path.join(loc_output, '1-3_integrated_NNtraining.pck' ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_mouse/1-3_integrated_NNtraining.pck<jupyter_text>retrieval<jupyter_code>df_raw = pd.read_csv(os.path.join(src.DIR_DATA, location, experiment, '3-33_integrated_retrieval_set.txt'), sep='\t').T
# Create a new variable called 'header' from the first row of the dataset
header = df_raw.iloc[0]
# Rename the dataframe's column values with the header variable
df_raw.columns = list(header.values)
# Replace the dataframe with a new one which does not contain the first row
df_raw = df_raw[1:]
# Delete 'Dataset' column
df_raw.drop(columns=['Dataset'], inplace=True)
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, '3-33_integrated_retrieval_set.pck'))
print('Experiment datasets are exported into ', os.path.join(loc_output, '3-33_integrated_retrieval_set.pck' ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_mouse/3-33_integrated_retrieval_set.pck<jupyter_text>PBMC<jupyter_code># experiment source
experiment = 'exper_pbmc'
location = 'external'
# the output location
loc_output = os.path.join(src.DIR_DATA, location, experiment)
src.define_folder(loc_=loc_output)<jupyter_output>FOLDER information,  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_pbmc/<jupyter_text>Dataset from Github<jupyter_code>dataset = 'Immune.20210314.rds.gz'
df_raw = pyreadr.read_r(os.path.join(src.DIR_DATA, location, experiment, dataset))[None]
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))
print('Experiment datasets are exported into ', os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_pbmc/Immune.pck<jupyter_text>Data with Magic implemented<jupyter_code>dataset = 'Immune.20210314.magic.tsv.tar.gz'
df_raw = pd.read_csv(os.path.join(src.DIR_DATA, location, experiment, dataset), sep='\t')
print(df_raw.shape)
print(df_raw.info())
df_raw
# Exporting preprocessed dataset
df_raw.to_pickle(os.path.join(loc_output, (dataset.split('.')[0]+'_magic.pck') ))
print('Experiment datasets are exported into ', os.path.join(loc_output, (dataset.split('.')[0]+'_magic.pck') ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_pbmc/Immune_magic.pck<jupyter_text>IMMUNE<jupyter_code># experiment source
experiment = 'exper_immune'
location = 'external'
# the output location
loc_output = os.path.join(src.DIR_DATA, location, experiment)
src.define_folder(loc_=loc_output)
dataset = 'Fig3g.data.rds.gz'
df_raw = pyreadr.read_r(os.path.join(src.DIR_DATA, location, experiment, dataset))[None]
df_raw = df_raw.iloc[:, :-2]
print(df_raw.shape)
print(df_raw.info())
df_raw
df_raw.to_pickle(os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))
print('Experiment datasets are exported into ', os.path.join(loc_output, (dataset.split('.')[0]+'.pck') ))<jupyter_output>Experiment datasets are exported into  /home/pgundogdu/projects/signalization_prior_knowledge_based_nn/data/external/exper_immune/Fig3g.pck
{'fraction_non_alphanumeric': 0.13551566349876804, 'fraction_numerical': 0.009151707145371349, 'mean_word_length': 7.623672230652504, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '402820', 'license': 'RSA-MD', 'n_tokens_mistral': 2123, 'n_tokens_neox': 2051, 'n_words': 290, 'path': 'notebooks/1.0-pg-converting-datasets-into-pck.ipynb', 'repo_name': 'pelingundogdu/mlfpm-esr9-m1.16', 'size': '5682'}