<commit_before>import urllib
import time
import sys
import json
from xml.etree.ElementTree import fromstring

TAXON_URL_BASE="http://www.ncbi.nlm.nih.gov/taxonomy"

BASE_SEARCH_QUARY = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
BASE_SUMMARY_QUARY = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"

BAD_XML_ERROR = "NCBI XML could not be parsed: %s\n (quary=%s)"
MP_ID_ERROR = "We expect one ID back from NCBI, but we got more than one ID. Oops!"
HTML_ERROR = "HTML error in accessing NCBI"

def search_NCBI_for_ids(search_term):

    # Search the taxanomy DB of NCBI for a given term
    parameters={"db":'taxonomy',"term":search_term}

    url = "%s?%s" %(BASE_SEARCH_QUARY,urllib.urlencode(parameters))# URL just for outputting errors
    # Search taxonomy for a given term. In case of an error, wait a second and try again. 
    # Do this for a maximum of 20 times
    succ = False
    tries=0
    while not succ and tries < 20:
        if tries > 0:
            time.sleep(1) # seems like NCBI has trouble with too many calls in short time periods
            print >>sys.stderr, "Error for %s, trying one more time: %d" %(search_term,tries)
        # make the call
        f = urllib.urlopen(BASE_SEARCH_QUARY,urllib.urlencode(parameters))    
        tries += 1
        # If HTTP error, try again
        succ = f.getcode() == 200
        if not succ:
            continue
        data = f.read()
        f.close()
        
        # Parse the resulting XML file (no automatic validation for now. We do have DTDs that we can later use to do validation) 
        dom = fromstring(data)
        
        # If error in the XML file, try again
        succ = dom.find("ERROR") is None or dom.find("ERROR").text is None

    if f.getcode() != 200:
        raise Exception (HTML_ERROR, f.getcode())
    if not succ:
        raise Exceptoin ("Unable to resolve %s after 20 tries" %search_term)

    # Find IdList and make sure it is not empty
    idListElement = dom.findall("IdList")    
    if len(idListElement) != 1:
        raise Exception(BAD_XML_ERROR %("We expec only on IdList item",url),data)
    
    # Find the retrieved IDs
    idList = idListElement[0].findall("Id")

        
    if len(idList) == 0:
        return None
    elif len(idList) > 1:
        raise MP_ID_ERROR
    else:
        id = idList[0].text
        return id

def get_name_for_ids(ids):

    idToName = {}

    # Get Taxon summary from  NCBI taxonomy for a given list of IDs
    parameters={"db":'taxonomy',"id":",".join(ids)}

    # Build URL and send make the request
    f = urllib.urlopen(BASE_SUMMARY_QUARY,urllib.urlencode(parameters))
    data = f.read()
    f.close()

    # Parse the resulting XML file (no automatic validation for now. We do have DTDs that we can later use to do validation) 
    dom = fromstring(data)<commit_msg>Add a tool att to ncbi adapter<commit_after>import urllib
import time
import sys
import json
from xml.etree.ElementTree import fromstring

TAXON_URL_BASE="http://www.ncbi.nlm.nih.gov/taxonomy"

BASE_SEARCH_QUARY = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi"
BASE_SUMMARY_QUARY = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi"

BAD_XML_ERROR = "NCBI XML could not be parsed: %s\n (quary=%s)"
MP_ID_ERROR = "We expect one ID back from NCBI, but we got more than one ID. Oops!"
HTML_ERROR = "HTML error in accessing NCBI"

def search_NCBI_for_ids(search_term):

    # Search the taxanomy DB of NCBI for a given term
    parameters={"db":'taxonomy',"term":search_term,"tool":"tnrastic"}

    url = "%s?%s" %(BASE_SEARCH_QUARY,urllib.urlencode(parameters))# URL just for outputting errors
    # Search taxonomy for a given term. In case of an error, wait a second and try again. 
    # Do this for a maximum of 20 times
    succ = False
    tries=0
    while not succ and tries < 20:
        if tries > 0:
            time.sleep(1) # seems like NCBI has trouble with too many calls in short time periods
            print >>sys.stderr, "Error for %s, trying one more time: %d" %(search_term,tries)
        # make the call
        f = urllib.urlopen(BASE_SEARCH_QUARY,urllib.urlencode(parameters))    
        tries += 1
        # If HTTP error, try again
        succ = f.getcode() == 200
        if not succ:
            continue
        data = f.read()
        f.close()
        
        # Parse the resulting XML file (no automatic validation for now. We do have DTDs that we can later use to do validation) 
        dom = fromstring(data)
        
        # If error in the XML file, try again
        succ = dom.find("ERROR") is None or dom.find("ERROR").text is None

    if f.getcode() != 200:
        raise Exception (HTML_ERROR, f.getcode())
    if not succ:
        raise Exceptoin ("Unable to resolve %s after 20 tries" %search_term)

    # Find IdList and make sure it is not empty
    idListElement = dom.findall("IdList")    
    if len(idListElement) != 1:
        raise Exception(BAD_XML_ERROR %("We expec only on IdList item",url),data)
    
    # Find the retrieved IDs
    idList = idListElement[0].findall("Id")

        
    if len(idList) == 0:
        return None
    elif len(idList) > 1:
        raise MP_ID_ERROR
    else:
        id = idList[0].text
        return id

def get_name_for_ids(ids):

    idToName = {}

    # Get Taxon summary from  NCBI taxonomy for a given list of IDs
    parameters={"db":'taxonomy',"id":",".join(ids),"tool":"tnrastic"}

    # Build URL and send make the request
    f = urllib.urlopen(BASE_SUMMARY_QUARY,urllib.urlencode(parameters))
    data = f.read()
    f.close()

    # Parse the resulting XML file (no automatic validation for now. We do have DTDs that we can later use to do validation) 
    dom = fromstring(data)
{'fraction_non_alphanumeric': 0.09960090230782578, 'fraction_numerical': 0.007287870900572618, 'mean_word_length': 2.7452891487979207, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2401494', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1826, 'n_words': 621}