<commit_before>#!/usr/bin/env python
#coding=utf-8
"""Download all Glottolog Newick Files"""

import os
import re
import codecs

try:
    import requests
except ImportError:
    raise ImportError("please install the requests library")

try:
    from bs4 import BeautifulSoup as bs
except ImportError:
    raise ImportError("please install the beautifulsoup4 library")

try:
    from ete2 import Tree
except ImportError:
    raise ImportError("please install the ete2 library")

IS_ISOCODE = re.compile(r"""'.* <\w{4}\d{4}><(\w{3})>'$""")


def clean_tree(tree):
    """Removes taxa with no ISO-639 codes"""
    to_keep = []
    for leaf in tree.iter_leaves():
        iso = IS_ISOCODE.findall(leaf.name)
        if len(iso) == 1:
            leaf.name = iso[0]  # rename to ISO code
            to_keep.append(leaf.name)
    
    if len(to_keep) <= 3:  # not worth keeping.
        return None
    tree.prune(to_keep)
    return tree

GLOTTOLOG_FAMILIES = "http://glottolog.org/glottolog/language.atom?type=families&sSearch_1=Top-level+unit"

if __name__ == '__main__':
    urls = {}
    
    for entry in bs(requests.get(GLOTTOLOG_FAMILIES).text).find_all('entry'):
        urls[entry.find('title').text] = entry.find('id').text
    
    for family in sorted(urls):
        url = urls[family]
        filename = "%s.glotto.trees" % family<commit_msg>Editing get-1.py to get glottolog trees with glottocodes at the tips<commit_after>#!/usr/bin/env python
#coding=utf-8
"""Download all Glottolog Newick Files"""

import os
import re
import codecs

try:
    import requests
except ImportError:
    raise ImportError("please install the requests library")

try:
    from bs4 import BeautifulSoup as bs
except ImportError:
    raise ImportError("please install the beautifulsoup4 library")

try:
    from ete2 import Tree
except ImportError:
    raise ImportError("please install the ete2 library")

IS_ISOCODE = re.compile(r"""'.* <\w{4}\d{4}><(\w{3})>'$""")
IS_GLOTTOCODE = re.compile(r"""'.* <(\w{4}\d{4})><\w{3}>'$""")

def clean_tree(tree):
    """Previously, removed taxa with no ISO-639 codes.
    Now renames taxa to Glottocodes."""
    to_keep = []
    for leaf in tree.iter_leaves():
        #iso = IS_ISOCODE.findall(leaf.name)
        glotto = IS_GLOTTOCODE.findall(leaf.name)
        if len(glotto) == 1:
            leaf.name = glotto[0]  # rename to glotto code
            to_keep.append(leaf.name)
    
    if len(to_keep) <= 3:  # not worth keeping.
        return None
    tree.prune(to_keep)
    return tree

GLOTTOLOG_FAMILIES = "http://glottolog.org/glottolog/language.atom?type=families&sSearch_1=Top-level+unit"

if __name__ == '__main__':
    urls = {}
    
    for entry in bs(requests.get(GLOTTOLOG_FAMILIES).text).find_all('entry'):
        urls[entry.find('title').text] = entry.find('id').text
    
    for family in sorted(urls):
        url = urls[family]
        filename = "%s.glotto.trees" % family
{'fraction_non_alphanumeric': 0.13349265961078866, 'fraction_numerical': 0.011608057357459884, 'mean_word_length': 3.258720930232558, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5414777', 'n_tokens_mistral': 1037, 'n_tokens_neox': 1005, 'n_words': 246}