<commit_before>import hashlib
import os
import urllib2


def check_md5(file_path, target_md5):
    file_md5 = hashlib.md5(open(file_path, 'rb').read()).hexdigest()
    return file_md5 == target_md5


def download(source_url, dest_file_path):
    d = urllib2.urlopen(source_url)
    output = open(dest_file_path, 'wb')
    output.write(d.read())
    output.close()


def fetch_url(source_url, dest_filename, md5=None):
    """
    Fetch the file from source_url, save it to the current working directory, and check MD5.
    If the file is already downloaded to cache, check its MD5.
    """
    dest_file_path = dest_filename

    if os.path.isfile(dest_file_path):
        if md5 is None:
            print('{f} already exists - No MD5 checksum given - redownloading file.'.format(
                f=dest_filename))
            # return dest_file_path
            os.remove(dest_file_path)
        elif check_md5(dest_file_path, md5):
            print('{fn} already exists - MD5 of cached file matches {m} - Using cached copy of {fn}'.format(m=md5,
                                                                                                            fn=dest_file_path))
            return dest_file_path
        else:
            print('Cached copy of {fn} failed MD5 check.'.format(fn=dest_file_path))
            os.remove(dest_file_path)

    print('Downloading new copy of {fn} from {url}'.format(fn=dest_file_path, url=source_url))<commit_msg>Modify HTTP download function to die on a 404 error.<commit_after>import hashlib
import os
import urllib2


def check_md5(file_path, target_md5):
    file_md5 = hashlib.md5(open(file_path, 'rb').read()).hexdigest()
    return file_md5 == target_md5


def download(source_url, dest_file_path):
    try:
        d = urllib2.urlopen(source_url)
    except urllib2.HTTPError, e:
        if e.code == 404:
            print ("ERROR - {u} was 404. Check the exact spelling of the pack name and pack version.".format(u=source_url))
            quit()
    # urllib doesn't always seem to throw a HTTPError in the case of a 404. Seems quite happy to open
    # a HTML 'Sorry, 404' page if the server presents one. Hence this additional check.
    if d.getcode() == 404:
        print ("ERROR - {u} was 404. Check the exact spelling of the pack name and pack version.".format(u=source_url))
        quit()
    # Getting 'text/html' usually indicates something is buggered.
    # i.e. NodeCDN gives a 404 html page but no 404 response code.
    if d.headers.getheaders('Content-Type')[0] == 'text/html':
        print ("""ERROR - requested {u}.
However! server responded with Content-Type 'text/html' instead of an actual file.
Maybe a 404 page?. Check the exact spelling of the pack name and pack version.""".format(u=source_url))
        quit()
    with open(dest_file_path, 'wb') as output:
        output.write(d.read())
        output.close()


def fetch_url(source_url, dest_filename, md5=None):
    """
    Fetch the file from source_url, save it to the current working directory, and check MD5.
    If the file is already downloaded to cache, check its MD5.
    """
    dest_file_path = dest_filename

    if os.path.isfile(dest_file_path):
        if md5 is None:
            print('{f} already exists - No MD5 checksum given - redownloading file.'.format(
                f=dest_filename))
            # return dest_file_path
            os.remove(dest_file_path)
        elif check_md5(dest_file_path, md5):
            print('{fn} already exists - MD5 of cached file matches {m} - Using cached copy of {fn}'.format(m=md5,
                                                                                                            fn=dest_file_path))
            return dest_file_path
        else:
            print('Cached copy of {fn} failed MD5 check.'.format(fn=dest_file_path))
            os.remove(dest_file_path)

    print('Downloading new copy of {fn} from {url}'.format(fn=dest_file_path, url=source_url))
{'fraction_non_alphanumeric': 0.09640070475711049, 'fraction_numerical': 0.017115529826327712, 'mean_word_length': 2.399486740804106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6463185', 'n_tokens_mistral': 1223, 'n_tokens_neox': 1172, 'n_words': 366}