<commit_before>import requests
import simplejson as json
import os
import fnmatch
import re

def get_remote_study(study):
    response = requests.get(url)
    

def get_local_studies():
    local_studies = []
    for root, dirnames, filenames in os.walk('phylesystem-1/study'):
      for filename in fnmatch.filter(filenames, '*.json'):
            local_studies.append(os.path.join(filename))


    return local_studies

def id_missing_studies(remote_studies, local_studies):
    missing_studies = []
    for s in remote_studies:
        if s.replace(".json", "") not in local_studies:
            missing_studies.append(s)
    return missing_studies


def _decode_list(data):
    rv = []
    for item in data:
        if isinstance(item, unicode):
            item = item.encode('utf-8')
        elif isinstance(item, list):
            item = _decode_list(item)
        elif isinstance(item, dict):
            item = _decode_dict(item)
        rv.append(item)
    return rv

def _decode_dict(data):
    rv = {}
    for key, value in data.iteritems():
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        elif isinstance(value, dict):
            value = _decode_dict(value)
        rv[key] = value
    return rv

#response = requests.get('http://api.opentreeoflife.org/phylesystem/v1/study_list')
#remote_studies = json.loads(response.text)
#local_studies = get_local_studies()
#missing_studies = id_missing_studies(remote_studies, local_studies)

response = requests.get('http://api.opentreeoflife.org/phylesystem/v1/study/pg_100.json')
json_data = json.loads(response.text, object_hook=_decode_dict)

otus = []
for otu in json_data['data']['nexml']['otus']['otu']:
     otus.append(otu['@id'])

print len(otus)<commit_msg>Finished base functionality. Added functions to calculate unique otus for both all studies and studies in synth.<commit_after>import requests
import simplejson as json
import csv
import time
import timeit

def _decode_list(data):
    rv = []
    for item in data:
        if isinstance(item, unicode):
            item = item.encode('utf-8')
        elif isinstance(item, list):
            item = _decode_list(item)
        elif isinstance(item, dict):
            item = _decode_dict(item)
        rv.append(item)
    return rv

def _decode_dict(data):
    rv = {}
    for key, value in data.iteritems():
        if isinstance(key, unicode):
            key = key.encode('utf-8')
        if isinstance(value, unicode):
            value = value.encode('utf-8')
        elif isinstance(value, list):
            value = _decode_list(value)
        elif isinstance(value, dict):
            value = _decode_dict(value)
        rv[key] = value
    return rv

def get_remote_otus(study):
    url = 'http://api.opentreeoflife.org/phylesystem/v1/study/%s.json' %study
    response = requests.get(url)
    json_data = json.loads(response.text, object_hook=_decode_dict)
    otus = []
    for otu in json_data['data']['nexml']['otus']['otu']:
        otus.append(otu['@id'])

    return otus

def save_otu_count(otu_count, study_count, run_time, total_otus, synth_study_count, unique_synth_otus, total_synth_otus):

    with open('otu-count.csv', 'w+') as csvfile:
        csvwriter = csv.writer(csvfile, delimiter="\t", quotechar="'")
        csvwriter.writerow(['Date:'+time.strftime("%x")] + ['UniqueOTUS:'+otu_count] + ['TotalOTUS:'+total_otus] + ['TotalStudies:'+study_count] + ['UniqueSynthOTUS:'+unique_synth_otus] + ['TotalSynthOTUS:'+total_synth_otus] + ['SynthStudyCount:'+synth_study_count] + ['Runtime:'+run_time])


def parse_synth_study_ids(synthesis_list):
    synth_study_list = []
    for s in synthesis_list:
        if 'taxonomy' not in s:
            id = s.split("_")[1]
            if "ot" or "pg" not in id:
                id = "pg_" + str(id)
        synth_study_list.append(id)

    return synth_study_list



if __name__ == "__main__":
    
    start_time = timeit.default_timer()

    ''' Get list of studies in synthesis, and process for otus '''

    response = requests.post('http://api.opentreeoflife.org/treemachine/v1/getSynthesisSourceList')
    synthesis_list = json.loads(response.text, object_hook=_decode_dict)
    synth_study_list = parse_synth_study_ids(synthesis_list) 
    
    ''' Get all OTUS in all NeXSON files from phylesystem
        and calculate unique otus in synthesis and overall
    '''
    response = requests.get('http://api.opentreeoflife.org/phylesystem/v1/study_list')
    study_list = json.loads(response.text)
    #study_list = ['pg_1000', 'pg_100']  # short study list for testing purposes
    all_unique_otus = []
    total_otus = 0
    total_synth_otus = 0
    unique_synth_otus = []
    count = 1
    for s in study_list:
        #print "Getting OTUS for study %s, %s / %s..." %(s, count, len(study_list)) # tracker for testing purposes
        otus = get_remote_otus(s)
        for o in otus:
            if o not in all_unique_otus:
                all_unique_otus.append(o)
        total_otus = total_otus + len(otus)
        if s in synth_study_list:
            for o in otus:
                if o not in unique_synth_otus:
                    unique_synth_otus.append(o)
            total_synth_otus = total_synth_otus + len(otus)                
        count += 1
    

 
    ## process it all, and save it to to a csv file
    stop_time = timeit.default_timer()
    run_time = stop_time - start_time
    save_otu_count(str(len(all_unique_otus)), str(len(study_list)), str(run_time), str(total_otus), str(len(synth_study_list)), str(unique_synth_otus), str(total_synth_otus))
{'fraction_non_alphanumeric': 0.11684027777777778, 'fraction_numerical': 0.0046875, 'mean_word_length': 2.970365265334252, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2426872', 'n_tokens_mistral': 1962, 'n_tokens_neox': 1880, 'n_words': 418}