<jupyter_start><jupyter_text>- TCGA (cancer genomic data with associated clinical data) hosted on google cloud and accessible via python. Example notebooks and scripts in the [github repository](https://github.com/isb-cgc/examples-Python).- [Google Genomics](https://cloud.google.com/genomics/docs/quickstart)- [List of publically available clinical datasets](https://github.com/EpistasisLab/ClinicalDataSources) from the Epistasis lab at UPenn- [Open data](https://avillach-lab.hms.harvard.edu/access-data/open-data) from the Avillach lab at harvard.- [Columbia Open Health Data](https://www.nature.com/articles/sdata2018273)- [openFDA](https://open.fda.gov/)- [Public datasets on github](https://github.com/awesomedata/awesome-public-datasets)- [eICU MIT clinical data](https://github.com/MIT-LCP/eicu-code)- [MGH/MF Waveform data](https://alpha.physionet.org/content/mghdb/1.0.0/)- [PhysioNet clinical datasets](https://alpha.physionet.org/about/database/image)- [CityHealth Dashboard](https://nyumc.qualtrics.com/jfe/form/SV_8IZgcZ9tifgNibH)- [MIMIC Critical Care Database](https://mimic.physionet.org/gettingstarted/access/) - Requires training- [CT Medical Images](https://www.kaggle.com/kmader/siim-medical-images) - [NIH Cancer CT Images](https://www.nih.gov/news-events/news-releases/nih-clinical-center-releases-dataset-32000-ct-images)- [Github list of medical image datasets](https://github.com/sfikas/medical-imaging-datasets)- [Broad Institute Cancer Datasets](http://portals.broadinstitute.org/cgi-bin/cancer/datasets.cgi)  Columbia Open Health DataTa, <NAME>.; <NAME>; <NAME>; <NAME>; <NAME> (2018): Columbia Open Health Data, a database of EHR prevalence and co-occurrence of conditions, drugs, and procedures. figshare. Collection.<jupyter_code>import wget
# more dataset links at https://figshare.com/collections/Columbia_Open_Health_Data_a_database_of_EHR_prevalence_and_co-occurrence_of_conditions_drugs_and_procedures/4151252/1
url_dict = {
'5year_paired_concepts' : 'https://ndownloader.figshare.com/files/12272924',
'ifetime_paired_concept_deviations': 'https://ndownloader.figshare.com/files/13154816',
'5year_single_concept_deviations' : 'https://ndownloader.figshare.com/files/13154819',
'lifetime_single_concept_deviations' : 'https://ndownloader.figshare.com/files/13154756'
}
output_directory = 'data'
filename = wget.download(url_dict['5year_paired_concepts'], out=output_directory)
import pandas as pd
pd.read_csv(filename,sep='\t',
            names=['concept_id_1','concept_id_2', 'prevalence']
           ).head()<jupyter_output><empty_output><jupyter_text>[Quantitative Dehydration Estimation](https://alpha.physionet.org/content/qde/1.0.0/)<jupyter_code>import wget
output_directory = 'data'
filename = wget.download('https://alpha.physionet.org/files/qde/1.0.0/dehydration_estimation.csv?download',out=output_directory)
import pandas as pd
f = pd.read_csv(filename,sep=',')
display(f.shape)
f.head()<jupyter_output><empty_output><jupyter_text>[Tappy Keystroke Data](https://alpha.physionet.org/content/tappy/1.0.0/)<jupyter_code>import wget
output_directory='data'
filename = wget.download('https://alpha.physionet.org/static/published-projects/tappy/tappy-keystroke-data-1.0.0.zip',out=output_directory)
filename
import zipfile
with zipfile.ZipFile(filename, 'r') as zip_ref:
    zip_ref.extractall(output_directory)
extracted_dir = filename.split('/')[1].split('.z')[0]
full_extracted_dir = output_directory+'/'+extracted_dir
import os
zip_files = os.listdir(full_extracted_dir)[:2]
for f in zip_files:
    with zipfile.ZipFile(full_extracted_dir+'/'+f, 'r') as zip_ref:
        zip_ref.extractall(full_extracted_dir)
os.listdir(full_extracted_dir)
data_folders = os.listdir(full_extracted_dir)[1:3]
data_folders
data_files = {}
for d in data_folders:
    data_files[d] = [full_extracted_dir+'/'+d+'/'+x for x in os.listdir(full_extracted_dir+'/'+d)]
data_files['Archived users'][0]
import yaml
with open(data_files['Archived users'][0], 'r') as stream:
    data_loaded = yaml.safe_load(stream)
data_loaded
data_files['Tappy Data'][0]
import pandas as pd
f = pd.read_csv(data_files['Tappy Data'][0],
                sep='\t',header=None,
                names=['UserKey','Date','Timestamp','Hand',
                       'Hold_time','Direction','Latency_time',
                       'Flight_time'])
f.head()
#https://stackoverflow.com/questions/18857352/python-remove-very-last-character-in-file

import os


def truncate_utf8_chars(filename, count, ignore_newlines=True):
    """
    Truncates last `count` characters of a text file encoded in UTF-8.
    :param filename: The path to the text file to read
    :param count: Number of UTF-8 characters to remove from the end of the file
    :param ignore_newlines: Set to true, if the newline character at the end of the file should be ignored
    """
    with open(filename, 'rb+') as f:
        last_char = None

        size = os.fstat(f.fileno()).st_size

        offset = 1
        chars = 0
        while offset <= size:
            f.seek(-offset, os.SEEK_END)
            b = ord(f.read(1))

            if ignore_newlines:
                if b == 0x0D or b == 0x0A:
                    offset += 1
                    continue

            if b & 0b10000000 == 0 or b & 0b11000000 == 0b11000000:
                # This is the first byte of a UTF8 character
                chars += 1
                if chars == count:
                    # When `count` number of characters have been found, move current position back
                    # with one byte (to include the byte just checked) and truncate the file
                    f.seek(-1, os.SEEK_CUR)
                    f.truncate()
                    return
            offset += 1
import csv
filename=data_files['Tappy Data'][0]
truncate_utf8_chars(filename, 1)
col_names=['UserKey','Date','Timestamp','Hand','Hold_time','Direction','Latency_time','Flight_time','']
all_rows = []
with open(filename, newline = '') as rows:
    row_reader = csv.reader(rows, delimiter='\t')
    for row in row_reader:
        all_rows.append(row)
df = pd.DataFrame(all_rows,columns=col_names).drop('',axis=1)
print(df.shape)
df.head()<jupyter_output>(6547, 8)<jupyter_text>[openFDA](https://open.fda.gov/downloads/)<jupyter_code>import wget
output_directory = 'data'
filename = wget.download('https://download.open.fda.gov/device/recall/device-recall-0001-of-0001.json.zip',out=output_directory)
filename
import json
z = zipfile.ZipFile(filename,"r")
filename=z.namelist()[0]
data = z.open(filename).read()
d = json.loads(data.decode("utf-8"))
import pandas as pd
from pandas.io.json import json_normalize
all_df = json_normalize(d)
all_df
meta_df = all_df.drop('results',axis=1)
meta_df.columns = [x.split('.')[1] for x in meta_df.columns]
meta_df
res = all_df[['results']].values
len(res)
len(res[0])
len(res[0][0])
len(res[0][0][2])
#takes a few minutes
dfs = []
for j in res[0][0]:
    dfs.append(json_normalize(j))
#takes a few minutes
cdfs = pd.concat(dfs,sort=True)
cdfs.columns = [x if len(x.split('.'))==1 else x.split('.')[1] for x in cdfs.columns]
print(cdfs.shape)
cdfs.head()
def myfunc(x):
    if type(x)!=list:
        return x
    else:
        if len(x)==0:
            return ''
        if len(x)==1:
            return x[0]
    
cdfs_clean = cdfs.applymap(myfunc)
cdfs_clean.head()<jupyter_output><empty_output><jupyter_text>[UC Irvine open datasets](https://archive.ics.uci.edu/ml/index.php)<jupyter_code>import wget
out_directory = "data"
filename = wget.download('https://archive.ics.uci.edu/ml/machine-learning-databases/lung-cancer/lung-cancer.names',out=out_directory)
filename
import pandas as pd
lines = [line.rstrip('\n') for line in open(filename)]
for l in lines:
    print(l)
filename = wget.download('https://archive.ics.uci.edu/ml/machine-learning-databases/lung-cancer/lung-cancer.data',out=out_directory)
import pandas as pd
pd.read_csv(filename,header=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14239075841285787, 'fraction_numerical': 0.021597187343043698, 'mean_word_length': 4.222950819672131, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 30, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '428988', 'license': 'MIT', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2681, 'n_words': 524, 'path': 'Clinical_and_Genomic_Data_Sources.ipynb', 'repo_name': 'sohilsshah91/hacknights', 'size': '8009'}