<jupyter_start><jupyter_text>This notebook is the scratch work behind my post on microbiome data availability.<jupyter_code>import pandas as pd

fname = 'Disease datasets list - data blog post.tsv'
df = pd.read_csv(fname, sep='\t')
df.head()
# All rows in "data availability" column are filled out
df.shape, df.dropna(subset=['Data availability']).shape
df['full_data'] = ['AWS' in str(i) for i in df['Status'].values]
print(len([i for i in df['Status'].dropna().values if 'AWS' in i]))
#df.dropna(subset=['Manual meta analysis?'])[['dataset_id', 'Manual meta analysis?', 'Data availability', 'Status']]
df.groupby('full_data').size()
df['Data availability'] = df['Data availability'].str.lower()
df.groupby(['full_data', 'Data availability']).size()<jupyter_output><empty_output><jupyter_text>Have full data<jupyter_code>df.query('full_data == True').sort_values(by='Data availability')
internet = ['ena', 'sra', 'mg-rast', 'author website']
df = df.rename(columns={'Data availability': 'data_avail'})
df.query('data_avail == @internet').sort_values(by='Clinical metadata availability')<jupyter_output><empty_output><jupyter_text>Have full data, got data from internet<jupyter_code>df.query('full_data == True')\
    .query('data_avail == @internet')\
    .groupby(['data_avail', 'Clinical metadata availability'])\
    .size()
df.query('full_data == True').query('data_avail == @internet').shape<jupyter_output><empty_output><jupyter_text>Have full data, didn't get data from internet<jupyter_code>df.query('full_data == True')\
    .query('data_avail != @internet')\
    .groupby(['data_avail', 'Clinical metadata availability'])\
    .size()<jupyter_output><empty_output><jupyter_text>Don't have full data<jupyter_code>df.query('full_data == False').groupby('data_avail').size().sort_values()<jupyter_output><empty_output><jupyter_text>Don't have full data, but raw data on SRA<jupyter_code>df.query('full_data == False').query('data_avail == "sra"')
df.query('full_data == False').query('data_avail == "author website"')<jupyter_output><empty_output><jupyter_text>Don't have full data, emailed<jupyter_code>df['data_avail'] = df['data_avail'].replace('would need to email', 'emailed')
df.query('full_data == False').query('data_avail == "emailed"').shape
df.query('full_data == False').query('data_avail == "emailed"').groupby("If emailed, response").size()
got_data = ['Got data from authors', 'Responded, sent via ftp']
df = df.rename(columns={"If emailed, response": 'email_response'})

keepcols = ['Manual meta analysis?', 'data_avail', 'email_response', 
            'Comments on email exchange/data', 'Clinical metadata availability',
            'Clinical metadata notes', 'Processing information',
            'Other comments', u'Status', u'Name in AWS', u'Email sent out?',
            'Disease', 'First and last authors']

df.query('full_data == False').query('data_avail == "emailed"').query('email_response == @got_data')[keepcols]
df.query('full_data == False').query('data_avail == "emailed"').query('email_response == "data was being put up on SRA"')[keepcols + ['Year']]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17782761935277155, 'fraction_numerical': 0.0, 'mean_word_length': 6.8639798488664985, 'pattern_counts': {'":': 1, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '511685', 'license': 'MIT', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1026, 'n_words': 219, 'path': '_jupyter/2018-04-17-data-availability/2018-04-17-data-availability.ipynb', 'repo_name': 'jasonhector/jasonhector.github.io', 'size': '3121'}