<jupyter_start><jupyter_text>Read the species names<jupyter_code># Put the names into dataframe
names = pd.read_table('data/CEH_taxa.txt')
# tab delimited, has header line, its content is unimportant, one line per species

# Some of the names are of the form "genus (subgenus/superspecies) species"
# So, the next sections makes the alternatives dictionary 
# where the full names are keys, "genus species" is 'binom'
# and "subgenus/superspecies species" is 'alt_binom'
# This script will also work is all the names are plain binomials

alternatives = {}

for row in names.iterrows():
    
    # original name is the name as appears in the table
    original_name = "%s %s"%(row[1][0], row[1][1])
    
    # genus is the first name in the genus field ie without the one in brackets if exists
    genus = row[1][0].split(' (')[0]
    
    # alt_genus is the braced name in the genus field, if exists
    alt_genus = None
    if len(row[1][0].split(' (')) > 1:
        alt_genus = row[1][0].split(' (')[1].split(')')[0]
        
    # species is the species field
    species = row[1][1]
    
    # binom is genus+species
    binom = "%s %s"%(genus, species)
    
    #alt_binom is alt_genus+species
    alt_binom = None
    if alt_genus:
        alt_binom = "%s %s"%(alt_genus, species)

        
    #arrange all the possible combinations under a single key, the original full name
    alternatives[original_name]={'binom': binom,
                                 'alt_binom': alt_binom}<jupyter_output><empty_output><jupyter_text>Count cox1 records in genebank<jupyter_code>from Bio import Entrez

# If you want to search other genes, see dependencies section below


count_matches = {}
j = 0

results = open('results/coi_record_counts.txt','wt')

for original_name in alternatives.keys():
    if j%100 == 0:
        print '.'
    j += 1
    # search using:
    
    # (original name [orgn] OR binom [orgn] OR alt_binom [orgn])
    # AND
    # (COX1 [GENE] OR COX1 [PROT] OR coi [GENE] OR coi [PROT] OR ...)
    
    # See above definitions of original name, binom, alt_binom
    # See complete list of cox1 synonyms in the Dependencies section below

    result = search_genbank(original_name, alternatives[original_name])
    count_matches[original_name] = result
    results.write("%s\t%i\t%s\n"%(original_name, result[0], result[1]))

            
results.close()

# The stdout is just progress. One dot per 100 species searched
# Save a pickle file of the counts

pickle_file_name = 'results/coi_record_counts.pkl'
output = open(pickle_file_name,'wb')
pickle.dump(count_matches, output)
output.close()<jupyter_output><empty_output><jupyter_text>Dependencies<jupyter_code># orig == The original name
# bin == binom
# alt == alt_binom
# See first cell for expelnation on 'original name', 'binom' and 'alt_binom'

def search_genbank(orig, names_dict, email="your email here",
                   use_orig=True, use_bin=True, use_alt=True,
                   search_gene=True, search_product=True, search_all=False):
    
    from Bio import Entrez

    # these are coi synonyms. If you want to count other genes
    # this is the place to add them.
    cox1_synonyms = ['MT-CO1','cox1','cox 1','COX1','COX 1','COI','CO I','coi','COXi', 'COX i',
                     'COXI','COX I', 'co i','coI','co I','cox I','coxI',
                     'cytochome oxidase 1','Cytochome oxidase 1','Cytochome Oxidase 1',
                     'cytochome oxidase I','Cytochome oxidase I', 'Cytochome Oxidase I',
                     'cytochome oxidase subunit 1','Cytochome oxidase subunit 1','Cytochome Oxidase subunit 1',
                     'Cytochome Oxidase Subunit 1',
                     'cytochome oxidase subunit I','Cytochome oxidase subunit I','Cytochome Oxidase subunit I',
                     'Cytochome Oxidase Subunit I',
                     'cytochrome c oxidase subunit I', 'Cytochrome c oxidase subunit I', 'Cytochrome C oxidase subunit I',
                     'Cytochrome C Oxidase subunit I', 'Cytochrome C Oxidase Subunit I',
                     'cytochrome c oxidase subunit 1', 'Cytochrome c oxidase subunit 1', 'Cytochrome C oxidase subunit 1',
                     'Cytochrome C Oxidase subunit 1', 'Cytochrome C Oxidase Subunit 1']  
    
    
    ## This will write the search phrase
    search_phrase = '('
    
    # First all the possible species names
    if use_orig:
        search_phrase += '%s [orgn] '%orig
    if len(search_phrase) > 1 and not search_phrase[-3:] == 'OR ':
        search_phrase += 'OR '
    if use_bin:
        search_phrase += '%s [orgn] '%names_dict['binom']
    if len(search_phrase) > 1 and not search_phrase[-3:] == 'OR ':
        search_phrase += 'OR '
    if use_alt and names_dict['alt_binom']:
        search_phrase += '%s [orgn] '%names_dict['alt_binom']
    if len(search_phrase) > 1 and search_phrase[-3:] == 'OR ':
        search_phrase = search_phrase[:-3]+') '
    elif len(search_phrase) > 1:
        search_phrase += ') '       
    else:
        raise RuntimeError('No species name passed')
    

    # Then all the cox1 synonyms (or your other genes)    
    search_phrase += 'AND ('
    
    for syn in cox1_synonyms:
        if search_all:
            search_phrase += "%s OR "%syn
        elif search_gene and search_product:
            search_phrase += "%s [GENE] OR %s [PROT] OR "%(syn,syn)
        elif search_gene:
            search_phrase += "%s [GENE] OR "%syn
        elif search_product:
            search_phrase += "%s [PROT] OR "%syn
        else:
            raise RuntimeError('Must specify fields to search')
            
    search_phrase = search_phrase[:-3]+')'
            
    # This will query genbank
    Entrez.email = email
    handle = Entrez.esearch(db="nucleotide", term=search_phrase)
    record = Entrez.read(handle)
    
    # Returning the number of records and the search phrase
    return [int(record['Count']), search_phrase]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1111111111111111, 'fraction_numerical': 0.010070493454179255, 'mean_word_length': 2.7883026064844247, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '312645', 'license': 'CC0-1.0', 'n_tokens_mistral': 2009, 'n_tokens_neox': 1835, 'n_words': 587, 'path': 'Count cox1 occurance in GenBank for a list of species.ipynb', 'repo_name': 'davelunt/barcode-db-builder', 'size': '5958'}