<jupyter_start><jupyter_text>Concatenating and calculating index on microglia dataset from [this publication](https://www.cell.com/immunity/fulltext/S1074-7613(18)30485-0?_returnURL=https%3A%2F%2Flinkinghub.elsevier.com%2Fretrieve%2Fpii%2FS1074761318304850%3Fshowall%3Dtrue), data can be found [here](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE121654)<jupyter_code>import numpy as np
import pandas as pd
import os
### okay, so we want ot concatenate all of the files together

files = os.listdir('E:\\DATA\\rna_seq_datasets\\stevens_developmental_microglia_2019')

index = pd.read_csv('E:\\DATA\\rna_seq_datasets\\stevens_developmental_microglia_2019\\' + files[3], delimiter = '\t', index_col = 0).index

df_output = pd.DataFrame(index = index)

for file in files:
    if '.txt' in file:
        df = pd.read_csv('E:\\DATA\\rna_seq_datasets\\stevens_developmental_microglia_2019\\' + file, delimiter = '\t', index_col = 0)
        df_final = pd.DataFrame(df.mean(axis = 1))
        df_final.rename(columns = {0:file}, inplace = True)
        df_output = df_output.join(df_final)
i = -1

columns = list(df_output.columns)

for column in columns:
    i = i + 1
    columns[i] = column[11:-8]
    
df_output.columns = columns
df_output
import developmental_index as dvp
scaled_df = dvp.scale_expression(df_output)
df_index = pd.read_csv('C:\\Users\\Ben\\Dropbox\\bilbo_lab_spr2020\\microglia-seq_website\\microglia-seq\\mdi\\unique_data_index_gene_list.csv', index_col = 0)
df_index.index = df_index.index.str.capitalize()
merged = dvp.import_index(scaled_df, df_index)
raw_index = dvp.generate_index(merged, merged.columns[0:-2])
index = dvp.scale_index(raw_index)
samples = list(index['index'].values)
### i think i might want to wait to build these labels until after the index is calculated....

age = ['Unknown'] * len(samples)
sex = ['Unknown'] * len(samples)
tx = ['N/A'] * len(samples)

i = -1

for sample in samples:
    i = i + 1
    if 'E14' in sample:
        age[i] = 'E14'
    if 'P4' in sample:
        age[i] = 'P4'
    if 'P5' in sample:
        age[i] = 'P4'
    if 'P30' in sample:
        age[i] = 'P30'
    if 'P100' in sample:
        age[i] = 'P100'
    if 'Old' in sample:
        age[i] = 'Old'
        
i = -1
for sample in samples:
    i = i + 1
    if 'F' in sample:
        sex[i] = 'F'
    if 'M' in sample:
        sex[i] = 'M'
    if 'Male' in sample:
        sex[i] = 'M'
    if 'male' in sample:
        sex[i] = 'M'
    if 'female' in sample:
        sex[i] = 'F'
    if 'Female' in sample:
        sex[i] = 'F'
        
        
i = -1
for sample in samples:
    i = i + 1
    if 'SALINE' in sample:
        tx[i] = 'Sal'
    if 'LPC' in sample:
        tx[i] = 'Lpc'
index['age'] = age
index['sex'] = sex
index['tx'] = tx
index.rename(columns = {0:''})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1485148514851485, 'fraction_numerical': 0.03536067892503536, 'mean_word_length': 3.1420204978038067, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '603578', 'license': 'MIT', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1081, 'n_words': 254, 'path': 'stevens_developmental_microglia_2019/calculating_index_200609.ipynb', 'repo_name': 'bendevlin18/developmental_index_rna-seq', 'size': '2828'}