<commit_before>import urltools
from hashlib import md5
import dateutil
import datetime

def get_domain(url_str):
    "Extract domain from URL"
    return urltools.parse(url_str).domain

def get_path(url_str):
    "Extract path component from URL"
    return urltools.parse(url_str).path

def _parse(url):
    "Parse URL with urltools"
    return urltools.parse(url)

def md5_hash(input_str):
    "Compuate MD5 hash"
    return md5(input_str.encode()).hexdigest()

def normalize(url, strip=True):
    "Optionally removing query string & RFC3986 normalize URL"
    p = urltools.parse(url)
    if strip:
        url = p.scheme + '://' + p.subdomain + p.domain + p.path
    return urltools.normalize(url)

def _clean_split(div_str, sep=', ', case=None):
    sl = []
    for div in div_str.split(sep):
        div = div.strip()
        if case == 'upper':
            div = div.upper()
        elif case == 'lower':
            div = div.lower()
        
        if div != '':
            sl.append(div)
    return sl

def find_path_date(url_path):
    "Extract date from URL page if exists."
    url_path_parts = _clean_split(url_path, sep='/', case='lower')
    match = []
    for part in url_path_parts:
        try:
            dateutil.parser.parse(part) # is part is date-like?
            match.append(part)
        except:
            next
    date_str = '/'.join(match)
    if len(date_str) < 4: # need at least 4 digits
        return None
    try:
        url_date = dateutil.parser.parse(date_str)
        if url_date >= dateutil.parser.parse('1982-06-01') and url_date <= (datetime.datetime.utcnow() + datetime.timedelta(weeks=3)):
            return url_date.strftime('%Y-%m-%d')
    except:
        return None

def url_path_features(url=None, url_path=None):
    "Extract features from URL"
    d = {}
    if url_path:
        d['path'] = url_path 
    else:
        d['path'] = get_path(url)
    d['path_parts'] = _clean_split(d['path'], sep='/', case='lower')
    url_date = find_path_date(d['path'])
    if url_date != None:
        d['url_date'] = url_date
    path_part_lengths = list(map(len, d['path_parts']))
    d['max_part_size'] = max(path_part_lengths)
    d['path_title'] = d['path_parts'][path_part_lengths.index(d['max_part_size'])]
    d['path_title_words'] = len(d['path_title'].split('-'))
    return d<commit_msg>updated / removed functions<commit_after>import urltools
from hashlib import md5 as _md5
import dateutil as _dateutil
import datetime as _datetime

def find_domain(url_str):
    "Extract domain from URL"
    return urltools.parse(url_str).domain

def find_path(url_str):
    "Extract path component from URL"
    return urltools.parse(url_str).path

def find_query(url_str):
    "Extract query component from URL"
    return urltools.parse(url_str).query

def find_fragment(url_str):
    "Extract fragment component from URL"
    return urltools.parse(url_str).fragment

def md5_hash(input_str):
    "Compuate MD5 hash"
    return _md5(input_str.encode()).hexdigest()

def normalize(url, strip=True):
    "RFC3986 normalize URL & Optionally removing url-query/fragment string (default)"
    p = urltools.parse(url)
    if strip:
        url = p.scheme + '://' + p.subdomain + p.domain + p.path
    return urltools.normalize(url)

def extract_date(url_path, url=None):
    "Extract date from URL page if exists."
    if url_path is None and url:
        url_path = get_path(url)
    url_path_parts = _clean_split(url_path, sep='/', case='lower')
    date_parts = []
    for part in url_path_parts:
        try:
            _dateutil.parser.parse(part) # is part date-like?
            date_parts.append(part)
        except:
            next
    date_str = '/'.join(date_parts)
    if len(date_str) < 4: # require  >=4 digits
        return
    url_date = _dateutil.parser.parse(date_str)
    if url_date <= (_datetime.datetime.utcnow() + _datetime.timedelta(weeks=1)):
        return url_date.strftime('%Y-%m-%d')

def _clean_split(div_str, sep=', ', case=None):
    sl = []
    for div in div_str.split(sep):
        div = div.strip()
        if case == 'upper':
            div = div.upper()
        elif case == 'lower':
            div = div.lower()
        if div != '':
            sl.append(div)
    return sl
{'fraction_non_alphanumeric': 0.12561634186428738, 'fraction_numerical': 0.007278703921108241, 'mean_word_length': 2.820627802690583, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4762705', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1415, 'n_words': 339}