<jupyter_start><jupyter_text>Download galectin data Data was chosen using data surveyor (<EMAIL>:rxncor/cfg-data-chart.git).See the [saved search page](../data/galectin-3/galectin-3.pdf) and saved [galectin json data](../data/galectin-3_5.0_human.json )Relevant papers may be https://www.ncbi.nlm.nih.gov/pubmed/24814672https://www.ncbi.nlm.nih.gov/pubmed/18216021  House keeping tasks  import all required dependencies<jupyter_code># standard imports
import urllib2
import os
import json
import StringIO
import pickle

# dataframe and numerical
import pandas as pd
import numpy as np

# plotting
import matplotlib.pyplot as plt
%matplotlib inline

#scipy
from scipy import stats
from scipy.special import erf
from scipy import sqrt


from IPython.display import HTML
def addToggle():
    return '''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.'''
HTML(addToggle())<jupyter_output><empty_output><jupyter_text>load dataset and download all data write xls to results directory<jupyter_code># load the sample data from the json file
samples_in="../data/galectin-3/galectin-3_5.0_human.json"
results_dir = "../results/galectin-3/"
dataframe_out=results_dir+"dataframes_galectin.pkl"
subdir="./"
if not os.path.isfile(samples_in):
    print "You need to include a samples file, generate one using <EMAIL>:rxncor/cfg-data-chart.git"
    exit(1)
# get json array metadata from previous search of CFG
with open(os.path.join(subdir, samples_in)) as f:
    datajson = json.load(f)
datajson 

# list which metadata are available

metadata_tags= datajson[0].keys()
metadata_tags

# HINT: never print datajson[i] once the excel files are loaded. rather use datajson[i].keys() and only print the necessary
# download xls

for sample in datajson:
    try:
        sample["xlsdata"]= urllib2.urlopen(sample["rawdatalink"][0]).read()
    except Exception as e:
        sample["xlsdata"]=None
        print e
# check xlsdata exists for samples, did the download go wrong somehow or was the data unavailable.
tmp_check_xls=[]
for sample in datajson:
    if sample["xlsdata"] is not None:
        tmp_check_xls.append("OK")
if len(tmp_check_xls)!=len(datajson):
    print "Missing .xls - check what went wrong"
# write the sample xls data to results directory

if os.path.isdir(results_dir):
    pass
else:
    os.mkdir(results_dir)


for sample in datajson:
    # prettify the outputname
    excel_out_name = sample["sample"].encode('utf-8')
    excel_out_name_ = excel_out_name.replace('\xc2\xb5', "micro")
    excel_out_name_ = excel_out_name_.replace('/', " per ")
    excel_out_name_ = excel_out_name_.replace(' ', "_")
    excel_out_name_ = excel_out_name_.replace(',', "")
    write_excel = open(results_dir+excel_out_name_+".xls",'wb')
    # write the excel file
    write_excel.write(datajson[0]["xlsdata"])
# confirm the array version 

total_samples=len(datajson)

# check the array versions 
array_version_stats ={}# knew those from the CFG anyway but do it again
for sample in datajson:
    try:
        array_version_stats[sample["experimentarrayversion"]]+=1
    except:
        array_version_stats[sample["experimentarrayversion"]]=1
key_max, value_max = max(array_version_stats.iteritems(), key=lambda x:x[1])
print key_max, value_max<jupyter_output><empty_output><jupyter_text>load xls into dataframe and add dataframe to sample list  write out as a pickle<jupyter_code># load xls into dataframe and add to sample dict

## Set parameters for this analysis
skiprows=0
columnlist=[0,1,2,3,4]
sheetname='Core PA Results'


for sample in datajson:
    handle = StringIO.StringIO(sample["xlsdata"])
    xlsx = pd.ExcelFile(handle)
    sn=xlsx.sheet_names
    #if len(sn)>1:
    #    for name in sn:
    #        if 'Cancer' in name or 'Sample' in name or 'Serum' in name:
    #            sheetname=name
    #else:
    #    sheetname=sn[0]
    print "Available sheets:", sn
    sample["dataframe"] = pd.read_excel(xlsx, sheetname, skiprows=skiprows, parse_cols=columnlist)
## pickle as json breaks on excel and sometimes dataframes
import pickle
pck=open(dataframe_out,'w')
pickle.dump(datajson,pck)
pck.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11235955056179775, 'fraction_numerical': 0.010337078651685393, 'mean_word_length': 4.59874213836478, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '351813', 'license': 'MIT', 'n_tokens_mistral': 1525, 'n_tokens_neox': 1478, 'n_words': 414, 'path': 'notebooks/download_cfg_for_galectin.ipynb', 'repo_name': 'chrisbarnettster/cfg_array_analysis_notebooks', 'size': '4470'}