<commit_before>import os
import tldextract

def extract_urls(path):
    urls = []
    for (dirpath, dirnames, filenames) in os.walk(path):
        for filename in filenames: 
            with open(os.path.join(dirpath, filename), 'r') as file:
                header = file.readline()
                url = header.split(';')[0].strip('"')
                urls.append(url)
        break # no subdirectories
    return urls            
                
def extract_tlds(path):
    urls = extract_urls(path)
    tlds = set()
    for url in urls:    
        tld = tldextract.extract(url)
        tld = tld.domain + '.' + tld.suffix
        tlds.add(tld)
    return tlds
    <commit_msg>Extract documents by TLD.<commit_after>import os
import tldextract
import shutil

def extract_urls(path):
    urls = []
    for (dirpath, dirnames, filenames) in os.walk(path):
        for filename in filenames: 
            with open(os.path.join(dirpath, filename), 'r') as file:
                header = file.readline()
                url = header.split(';')[0].strip('"')
                urls.append(url)
        break # no subdirectories
    return urls            


def extract_tld(url):
    tld = tldextract.extract(url)
    tld = tld.domain + '.' + tld.suffix
    return tld


def extract_tlds(path):
    return {extract_tld(url) for url in extract_urls(path)}
    

def extract_documents_by_tlds(tlds, input_path, output_path):
    for (dirpath, dirnames, filenames) in os.walk(input_path):
        for filename in filenames: 
            with open(os.path.join(input_path, filename), 'r') as file:
                header = file.readline()
                url = header.split(';')[0].strip('"')
                if extract_tld(url) in tlds:
                    print('Copying ' + filename)
                    shutil.copy2(os.path.join(dirpath, filename), output_path)
        break # no subdirectories
    

{'fraction_non_alphanumeric': 0.11368421052631579, 'fraction_numerical': 0.002105263157894737, 'mean_word_length': 2.046474358974359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7170133', 'n_tokens_mistral': 622, 'n_tokens_neox': 607, 'n_words': 142}