<commit_before>import urllib, urllib2, os

opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
urllib2.install_opener(opener)

HEADER = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0',
          'Accept-Language': 'de',
          'Accept-Encoding': 'utf-8'}

def absUrl(site, href):
    href = href.replace("\\", "/")
    if href.startswith("http://") or href.startswith("https://"):
        return href
    comps = href.split("/")
    if href[:1] == "/":
        comps[0:1] = site.split("/")[:3]
    else:
        comps[0:0] = site.split("/")[:-1]
    i = 2
    while i < len(comps):
        if comps[i] in ('.'):
            del comps[i]
        elif comps[i] == '..':
            if i > 0 and comps[i-1] != '..':
                del comps[i-1:i+1]
                i -= 1
            else:
                i += 1
        else:
            i += 1
    return "/".join(comps)

def getResponse(url, postData=None):
    if(postData is not None):
        postData = urllib.urlencode(postData)
    req = urllib2.Request(url, postData, HEADER)
    return urllib2.urlopen(req)

def safe_getResponse(url, postData=None):
    try:
        return getResponse(url, postData=postData)
    except urllib2.HTTPError, e:
        print 'Error Code:', e.code
    except ValueError, e:
        print "invalid URL:" + url
    except urllib2.URLError, e:
        print 'Reason:', e.reason
    return None<commit_msg>Fixed Bug trailing '/' again<commit_after>import urllib, urllib2, os

opener = urllib2.build_opener(urllib2.HTTPCookieProcessor())
urllib2.install_opener(opener)

HEADER = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0',
          'Accept-Language': 'de',
          'Accept-Encoding': 'utf-8'}

def absUrl(site, href):
    href = href.replace("\\", "/")
    if href.startswith("http://") or href.startswith("https://"):
        return href
    comps = href.split("/")
    if href[:1] == "/":
        comps[0:1] = site.split("/")[:3]
    else:
        comps[0:0] = site.split("/")[:-1]
    i = 2
    while i < len(comps):
        if comps[i] == '.':
            del comps[i]
        elif comps[i] == '..':
            if i > 0 and comps[i-1] != '..':
                del comps[i-1:i+1]
                i -= 1
            else:
                i += 1
        else:
            i += 1
    return "/".join(comps)

def getResponse(url, postData=None):
    if(postData is not None):
        postData = urllib.urlencode(postData)
    req = urllib2.Request(url, postData, HEADER)
    return urllib2.urlopen(req)

def safe_getResponse(url, postData=None):
    try:
        return getResponse(url, postData=postData)
    except urllib2.HTTPError, e:
        print 'Error Code:', e.code
    except ValueError, e:
        print "invalid URL:" + url
    except urllib2.URLError, e:
        print 'Reason:', e.reason
    return None
{'fraction_non_alphanumeric': 0.16183152743795876, 'fraction_numerical': 0.02097168822090178, 'mean_word_length': 2.1312910284463893, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453479', 'n_tokens_mistral': 1040, 'n_tokens_neox': 1025, 'n_words': 227}