<commit_before>import hashlib
import urllib.parse
import re

def hash(url):
    """Returns hash of url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    return hashlib.sha1(url.encode('utf-8')).hexdigest()

def clean(url):
    """Remove last backslash from url"""
    return url.rstrip('/')

def add_scheme(url):
    """Add missing scheme to url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    scheme = "http"
    netloc = path
    path = ""
    return urllib.parse.urlunsplit((scheme, netloc, path, qs, anchor))

def domain(url):
    """Return domain of the url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)

    if not scheme:
        url = add_scheme(url)
        scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)

    return netloc

def generate_regex(url):
    """Generate regex for url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    netloc = netloc.replace(".", "\.")

    return re.compile("^(https?:\/\/)?([a-z0-9]+[.])*"+netloc+".*$")<commit_msg>TODO comment added<commit_after>import hashlib
import urllib.parse
import re

def hash(url):
    """Returns hash of url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    return hashlib.sha1(url.encode('utf-8')).hexdigest()

def clean(url):
    """Remove last backslash from url"""
    return url.rstrip('/')

def add_scheme(url):
    """Add missing scheme to url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    scheme = "http"
    netloc = path
    path = ""
    return urllib.parse.urlunsplit((scheme, netloc, path, qs, anchor))

def domain(url):
    """Return domain of the url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)

    #TODO - Maybe implement in higher layer
    if not scheme:
        url = add_scheme(url)
        scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)

    return netloc

def generate_regex(url):
    """Generate regex for url"""
    scheme, netloc, path, qs, anchor = urllib.parse.urlsplit(url)
    netloc = netloc.replace(".", "\.")

    return re.compile("^(https?:\/\/)?([a-z0-9]+[.])*"+netloc+".*$")
{'fraction_non_alphanumeric': 0.16444648598989434, 'fraction_numerical': 0.0036747818098300414, 'mean_word_length': 3.435845213849287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4407530', 'n_tokens_mistral': 763, 'n_tokens_neox': 762, 'n_words': 197}