<commit_before>import sys
import os
import hashlib


# Function to read file in chunks of given size
def read_chunks(file, size = 1024):
    while True:
        chunk = file.read(size)
        if not chunk:
            return
        yield chunk


# Function to get unique hashes of files
# Use sha256 by default
def get_hash(file, first_kilobyte = False, hash = hashlib.sha256):
    hash_obj = hash()
    f = open(file, "rb")

    if first_kilobyte:
        hash_obj.update(f.read(1024))
    else:
        for chunk in read_chunks(f):
            hash_obj.update(chunk)

    hashed = hash_obj.digest()

    f.close()
    return hashed


# Function to compare file sizes
#  - Returns a dictionary of lists containing possible duplicate files by file size
# Return a list of possible duplicate paths instead?
def size_comparison(paths):
    ret = {}

    # Iterate over files in paths:
    for path in paths:
        for dirpath, dirnames, filenames in os.walk(path):
            for filename in filenames:
                full_path = os.path.join(dirpath, filename)

                try:
                    full_path = os.path.realpath(full_path)     # Eliminate symbolic links - get the file itself
                    file_size = os.path.getsize(full_path)      # Get size in bytes
                except Exception:                               # If file is not accessible or something goes wrong >> continues
                    print("Oops! There was a problem accessing a file:", full_path)
                    continue

                has_key = ret.get(file_size)                    # Check if list of files of given size exist

                if has_key:
                    ret[file_size].append(full_path)            # Add file path to list
                else:
                    ret[file_size] = []                         # Create file list for this particular file size
                    ret[file_size].append(full_path)            # Add file path to list

    return ret


def first_kilobyte_comparison(paths):
    ret = {}
    
    for size in paths:
        # Unique file size, continue
        if (len(paths[size]) < 2):
            continue

        for filename in paths[size]:
            try:
                hash_1k = get_hash(filename, first_kilobyte=True)
            except Exception:
                print("Oops! There was a problem accessing a file:", full_path)
                continue

            has_key = ret.get(hash_1k)

            if has_key:
                ret[hash_1k].append(filename)
            else:
                ret[hash_1k] = []          # create the list for this 1k hash
                ret[hash_1k].append(filename)

    return ret


def full_file_comparison(paths):
    hash_full = {}

    for hash_1 in paths:
        if (len(paths[hash_1]) < 2):
            continue

        for filename in paths[hash_1]:
            try: 
                full_hash = get_hash(filename, first_kilobyte = False)
            except Exception:
                print("Oops! There was a problem accessing a file:", full_path)
                continue

            duplicate = hash_full.get(full_hash)

            if duplicate:
                print("Duplicate found: %s and %s" % (filename, duplicate))
                if (os.path.exists(duplicate)):
                    continue
                    #os.remove(duplicate) # Use commented section to remove duplicates
            else:
                hash_full[full_hash] = filename

    return hash_full


# Function to find duplicate files
def check_for_duplicates(paths):
    hashes_by_size = size_comparison(paths)
    hashes_on_1k = first_kilobyte_comparison(hashes_by_size)    # For all files with the same file size, get their hash on the 1st 1024 bytes
    hashes_full = full_file_comparison(hashes_on_1k)            # For all files with the hash on the 1st 1024 bytes, get their hash on the full file - collisions will be duplicates


#
# # Run
#

if sys.argv[1:]:
    check_for_duplicates(sys.argv[1:])
else:
    print("Please pass the paths to check as parameters to the script")<commit_msg>Change printing<commit_after># Finds duplicate files, prints results on screen

import sys
import os
import hashlib


# Function to read file in chunks of given size
def read_chunks(file, size = 1024):
    while True:
        chunk = file.read(size)
        if not chunk:
            return
        yield chunk


# Function to get unique hashes of files
# Use sha256 by default
def get_hash(file, first_kilobyte = False, hash = hashlib.sha256):
    hash_obj = hash()
    f = open(file, "rb")

    if first_kilobyte:
        hash_obj.update(f.read(1024))
    else:
        for chunk in read_chunks(f):
            hash_obj.update(chunk)

    hashed = hash_obj.digest()

    f.close()
    return hashed


# Function to compare file sizes
#  - Returns a dictionary of lists containing possible duplicate files by file size
# Return a list of possible duplicate paths instead?
def size_comparison(paths):
    ret = {}

    # Iterate over files in paths:
    for path in paths:
        for dirpath, dirnames, filenames in os.walk(path):
            for filename in filenames:
                full_path = os.path.join(dirpath, filename)

                try:
                    full_path = os.path.realpath(full_path)     # Eliminate symbolic links - get the file itself
                    file_size = os.path.getsize(full_path)      # Get size in bytes
                except Exception:                               # If file is not accessible or something goes wrong >> continues
                    print("Oops! There was a problem accessing a file:", full_path)
                    continue

                has_key = ret.get(file_size)                    # Check if list of files of given size exist

                if has_key:
                    ret[file_size].append(full_path)            # Add file path to list
                else:
                    ret[file_size] = []                         # Create file list for this particular file size
                    ret[file_size].append(full_path)            # Add file path to list

    return ret


def first_kilobyte_comparison(paths):
    ret = {}
    
    for size in paths:
        # Unique file size, continue
        if (len(paths[size]) < 2):
            continue

        for filename in paths[size]:
            try:
                hash_1k = get_hash(filename, first_kilobyte=True)
            except Exception:
                print("Oops! There was a problem accessing a file:", full_path)
                continue

            has_key = ret.get(hash_1k)

            if has_key:
                ret[hash_1k].append(filename)
            else:
                ret[hash_1k] = []          # create the list for this 1k hash
                ret[hash_1k].append(filename)

    return ret


def full_file_comparison(paths):
    hash_full = {}
    ret = {}

    for hash_1 in paths:
        if (len(paths[hash_1]) < 2):
            continue

        for filename in paths[hash_1]:
            try: 
                full_hash = get_hash(filename, first_kilobyte = False)
            except Exception:
                print("Oops! There was a problem accessing a file:", full_path)
                continue

            duplicate = hash_full.get(full_hash)

            if duplicate:
                ret[full_hash].append(filename)
            else:
                hash_full[full_hash] = filename
                ret[full_hash] = []
                ret[full_hash].append(filename)

    return ret


# Function to find duplicate files
def check_for_duplicates(paths):
    hashes_by_size = size_comparison(paths)
    hashes_on_1k = first_kilobyte_comparison(hashes_by_size)    # For all files with the same file size, get their hash on the 1st 1024 bytes
    duplicates = full_file_comparison(hashes_on_1k)             # For all files with the hash on the 1st 1024 bytes, get their hash on the full file >> duplicates are found

    # Print duplicate files grouped:
    for i in duplicates.values():
        if (len(i) > 1):                                        # If list length is 1, file is unique
            print(str(len(i)) + " Identical:")
            for j in i:
                print("  -" + str(j))                           # Print identical files
            print()
        else:
            continue


#
# # Run
#

if sys.argv[1:]:
    check_for_duplicates(sys.argv[1:])
else:
    print("Please pass the paths to check as parameters to the script")
{'fraction_non_alphanumeric': 0.07214922571562647, 'fraction_numerical': 0.009385265133740028, 'mean_word_length': 1.7714564369310792, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5786661', 'n_tokens_mistral': 2447, 'n_tokens_neox': 2253, 'n_words': 761}