<commit_before>from __future__ import print_function, division

import hashlib
import os
import json


def generate_file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    with open(filename, "rb") as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update(buf)
    return m.hexdigest()

def check_hashes(d):
    """
    Parameter
    -----------
    d : dictionary in which keys are file names and the items are hashes
        corresponding to the keys.
    
    Return
    --------
    check whether the has of each file key in 'd' matches the corresponding
    hash item.
    """

    all_good = True
    for k, v in d.items():
        digest = generate_file_md5(k)
        if v == digest:
            print("The file {0} has the correct hash.".format(k))
        else:
            print("ERROR: The file {0} has the WRONG hash!".format(k))
            all_good = False
    return all_good

def get_hash(directory):
    """
    Parameters
    -----------
    directory: the path of the directory you want to make a hash list for.
    Return:
    a list of hashes for each path in directory.
    """
    file_hashes = {}
    for path, dirs, files in os.walk(directory):
        for file_name in files:
            file_path = os.path.join(path, file_name)
            file_hashes[file_path] = generate_file_md5(file_path)
    return file_hashes


#get hashes for all files in all subdirectories of the decompressed 
#ds115_sub001 directory.
if __name__ == "__main__":
    file_hashes = get_ash('sub001')
    with open('sub001_hashes.json', 'w') as out:
        json.dump(file_hashes,out)
    check_hashes(file_hashes)

<commit_msg>another sol<commit_after>from __future__ import print_function, division

import hashlib
import os
import json


def generate_file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    with open(filename, "rb") as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update(buf)
    return m.hexdigest()

def check_hashes(d):
    """
    Parameter
    -----------
    d : dictionary in which keys are file names and the items are hashes
        corresponding to the keys.
    
    Return
    --------
    check whether the has of each file key in 'd' matches the corresponding
    hash item.
    """

    all_good = True
    for k, v in d.items():
        digest = generate_file_md5(k)
        if v == digest:
            print("The file {0} has the correct hash.".format(k))
        else:
            print("ERROR: The file {0} has the WRONG hash!".format(k))
            all_good = False
    return all_good

def get_hash(directory):
    """
    Parameters
    -----------
    directory: the path of the directory you want to make a hash list for.
    Return:
    a list of hashes for each path in directory.
    """
    file_paths = []
    for path, subdirs, files in os.walk(directory):
        for name in files:
            file_paths.append(os.path.join(path, name))
    dictionary = {path: generate_file_md5(path) for path in file_paths}
    with open('sub001_hashes.json', 'w') as out:
        json.dump(dictionary, out)
    return dictionary
    

#get hashes for all files in all subdirectories of the decompressed 
#ds115_sub001 directory.
if __name__ == "__main__":
    file_hashes = get_hash('sub001')
    check_hashes(file_hashes)


{'fraction_non_alphanumeric': 0.10242194339072075, 'fraction_numerical': 0.01225561715786402, 'mean_word_length': 2.206735266604303, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2298742', 'n_tokens_mistral': 1063, 'n_tokens_neox': 1034, 'n_words': 346}