<commit_before>import urllib2
import Bio.Phylo as bp
from Bio.Phylo import Newick
import os
import tarfile


col_delimiter = '\t|\t'
row_delimiter = '\t|\n'
url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz'
tree_filename = 'ncbi_taxonomy.newick'
tree_format = 'newick'

# download the taxonomy archive
filename = url.split('/')[-1]
if os.path.exists(filename):
    print 'Using existing copy of %s' % filename
else:
    print 'Downloading %s...' % filename
    r = urllib2.urlopen(urllib2.Request(url))
    assert r.geturl() == url
    with open(filename, 'wb') as output_file:
        output_file.write(r.read())
    r.close()

# extract the text dump
for filename in ('nodes.dmp', 'names.dmp'):
    if os.path.exists(filename):
        print 'Using existing copy of %s' % filename
    else:
        print 'Extracting %s...' % filename
        archive = tarfile.open(name=filename, mode='r:gz')
        archive.extract(filename)

# get names for all tax_ids from names.dmp
print 'Getting names...'
scientific_names = {}
common_names = {}
with open('names.dmp') as names_file:
    for line in names_file:
        line = line.rstrip(row_delimiter)
        values = line.split(col_delimiter)
        tax_id, name_txt, _, name_type = values[:4]
        if name_type == 'scientific name':
            scientific_names[tax_id] = name_txt
        elif name_type == 'common name':
            common_names[tax_id] = name_txt

# read all node info from nodes.dmp
print 'Reading taxonomy...'
nodes = {}
with open('nodes.dmp') as nodes_file:
    for line in nodes_file:
        line = line.rstrip(row_delimiter)
        values = line.split(col_delimiter)
        tax_id, parent_id = values[:2]
        this_node = Newick.Clade(name=scientific_names[tax_id])
        nodes[tax_id] = this_node
        this_node.parent = parent_id

# create tree from nodes dictionary
print 'Building tree...'
for node_id, this_node in nodes.items():
    if node_id == this_node.parent:
        root_node = this_node
        print 'Found root.'
    else:
        parent_node = nodes[this_node.parent]
        parent_node.clades.append(this_node)
    del this_node.parent

tree = Newick.Tree(root=root_node)

# write tree to file
print 'Writing %s tree to %s...' % (tree_format, tree_filename)
bp.write([tree], tree_filename, tree_format)

print 'Done!'<commit_msg>Add common name as node comment.<commit_after>import urllib2
import Bio.Phylo as bp
from Bio.Phylo import Newick
import os
import tarfile


col_delimiter = '\t|\t'
row_delimiter = '\t|\n'
url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump.tar.gz'
tree_filename = 'ncbi_taxonomy.newick'
tree_format = 'newick'

# download the taxonomy archive
filename = url.split('/')[-1]
if os.path.exists(filename):
    print 'Using existing copy of %s' % filename
else:
    print 'Downloading %s...' % filename
    r = urllib2.urlopen(urllib2.Request(url))
    assert r.geturl() == url
    with open(filename, 'wb') as output_file:
        output_file.write(r.read())
    r.close()

# extract the text dump
for filename in ('nodes.dmp', 'names.dmp'):
    if os.path.exists(filename):
        print 'Using existing copy of %s' % filename
    else:
        print 'Extracting %s...' % filename
        archive = tarfile.open(name=filename, mode='r:gz')
        archive.extract(filename)

# get names for all tax_ids from names.dmp
print 'Getting names...'
scientific_names = {}
common_names = {}
with open('names.dmp') as names_file:
    for line in names_file:
        line = line.rstrip(row_delimiter)
        values = line.split(col_delimiter)
        tax_id, name_txt, _, name_type = values[:4]
        if name_type == 'scientific name':
            scientific_names[tax_id] = name_txt
        elif name_type == 'common name':
            common_names[tax_id] = name_txt

# read all node info from nodes.dmp
print 'Reading taxonomy...'
nodes = {}
with open('nodes.dmp') as nodes_file:
    for line in nodes_file:
        line = line.rstrip(row_delimiter)
        values = line.split(col_delimiter)
        tax_id, parent_id = values[:2]
        this_node = Newick.Clade(name=scientific_names[tax_id])
        if tax_id in common_names:
            this_node.comment = common_names[tax_id]
        nodes[tax_id] = this_node
        this_node.parent = parent_id

# create tree from nodes dictionary
print 'Building tree...'
for node_id, this_node in nodes.items():
    if node_id == this_node.parent:
        root_node = this_node
        print 'Found root.'
    else:
        parent_node = nodes[this_node.parent]
        parent_node.clades.append(this_node)
    del this_node.parent

tree = Newick.Tree(root=root_node)

# write tree to file
print 'Writing %s tree to %s...' % (tree_format, tree_filename)
bp.write([tree], tree_filename, tree_format)

print 'Done!'
{'fraction_non_alphanumeric': 0.11950864043306267, 'fraction_numerical': 0.0024984384759525295, 'mean_word_length': 3.1521175453759724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4464405', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1568, 'n_words': 410}