<jupyter_start><jupyter_text>Human Protien Atlas Data Processing Here we are pulling data in and processing it. * [Here](http://www.sciencemag.org/content/347/6220/1260419.full) is the paper by Uhlen et al. on the dataset * The data was obtained from [proteinatlas.org](http://www.proteinatlas.org/)<jupyter_code>%matplotlib inline
import pandas as pd
path = '/cellar/users/agross/Data/Protein_Atlas/'<jupyter_output><empty_output><jupyter_text>Cancer File<jupyter_code>cancer = pd.read_csv(path + 'cancer.csv')
cancer.head()
cancer['Expression type'].value_counts()
cancer.Level.value_counts()
cancer.Tumor.value_counts()<jupyter_output><empty_output><jupyter_text>Just double checking that the counts add up to the 'Total patients' column.<jupyter_code>gb = cancer.groupby(['Gene','Tumor'])
assert all(gb['Count patients'].sum() == gb['Total patients'].first())
stacked = cancer.set_index(['Gene','Tumor','Level'])['Count patients']
stacked = stacked.unstack('Level')
stacked.head()
cancer = stacked<jupyter_output><empty_output><jupyter_text>Normal Tissue File<jupyter_code>normals = pd.read_csv(path + 'normal_tissue.csv')
normals.head()
normals['Expression type'].value_counts()
del normals['Expression type']
normals['Reliability'].value_counts()
normals['Level'].value_counts()
normals.Tissue.unique()
normals['Cell type'].unique()
normals[['Tissue','Cell type']].drop_duplicates().shape<jupyter_output><empty_output><jupyter_text>Subcellular location<jupyter_code>loc = pd.read_csv(path + 'subcellular_location.csv')
loc.head()
loc['Expression type'].value_counts()
loc.Reliability.value_counts()
loc['Main location'].value_counts().head(10)<jupyter_output><empty_output><jupyter_text>RNA<jupyter_code>rna = pd.read_csv(path + 'rna.csv')
rna.head()
rna.Unit.value_counts()
del rna['Unit']
rna.groupby('Abundance').Value.agg({'mean':'mean','count':'count'}).sort('mean')
rna_df = rna.set_index(['Gene','Sample']).Value.unstack()
rna_df.shape<jupyter_output><empty_output><jupyter_text>Metadata<jupyter_code>meta = pd.read_table(path + 'proteinatlas.tab')
meta.head(3).T
meta.Gene.value_counts().value_counts()
mapping = meta.set_index('Ensembl')['Gene']
meta['Subcellular location'].value_counts().head()<jupyter_output><empty_output><jupyter_text>Mapping tumor to normals<jupyter_code>tn_map = {'breast cancer': [('breast','glandular cells')],
          'carcinoid': [('pancreas','islets of Langerhans')],
          'cervical cancer': [('cervix, uterine','squamous epithelial cells'),
                              ('cervix, uterine','glandular cells')],
          'colorectal cancer': [('colon','glandular cells'),
                                ('rectum','glandular cells')],
          'endometrial cancer': [('endometrium 1', 'glandular cells'),
                                 ('endometrium 2', 'glandular cells')],
          'glioma':[('cerebral cortex', 'glial cells')],
          'head and neck cancer': [('oral mucosa', 'squamous epithelial cells'),
                                   ('tonsil', 'squamous epithelial cells'),
                                   ('salivary gland', 'glandular cells')],
          'liver cancer': [('liver', 'bile duct cells'),
                           ('liver', 'hepatocytes')],
          'lung cancer': [('bronchus', 'respiratory epithelial cells'),
                          ('lung', 'pneumocytes')],
          'lymphoma': [('lymph node', 'germinal center cells'),
                       ('lymph node', 'non-germinal center cells')],
          'melanoma': [('skin 1', 'melanocytes')],
          'pancreatic cancer': [('pancreas', 'exocrine glandular cells')],
          'prostate cancer': [('prostate', 'glandular cells')],
          'renal cancer': [('kidney', 'cells in tubules')],
          'skin cancer': [('skin 1', 'keratinocytes')],
          'stomach cancer': [('stomach 1', 'glandular cells'),
                             ('stomach 2', 'glandular cells')],
          'testis cancer': [('testis', 'cells in seminiferous ducts')],
          'urothelial cancer': [('urinary bladder', 'urothelial cells')]}                                  
 
cancer.index.get_level_values('Tumor').unique()
normals[['Tissue','Cell type']].drop_duplicates().as_matrix()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16953548691346382, 'fraction_numerical': 0.005423249233671304, 'mean_word_length': 4.0802395209580835, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '32367', 'license': 'MIT', 'n_tokens_mistral': 1456, 'n_tokens_neox': 1339, 'n_words': 252, 'path': 'Notebooks/Purgatory/Protein_Atlas.ipynb', 'repo_name': 'theandygross/TCGA_differential_expression', 'size': '4241'}