<commit_before>#!/usr/bin/python3
import sys
import hashlib
from pathlib import Path
from datetime import datetime 
import argparse

BLOCKSIZE = 65536
def get_file_hash(file_path):
    hasher = hashlib.sha256()
    with open(file_path, 'rb') as f:
        buf = f.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = f.read(BLOCKSIZE)
    return hasher.hexdigest()


class DupeTracker():
    def __init__(self,suffix=None, verbose=None):
        self.file_map = {}
        self.suffix = suffix
        self.verbose = verbose

    def map_file(self, file_to_map):
        if self.suffix and file_to_map.suffix.lower() != self.suffix.lower():
            return
        if self.verbose:
            print('Scanning file ' + file_to_map.as_posix())

        file_hash = get_file_hash(file_to_map.as_posix())
        if file_hash in self.file_map:
            self.file_map[file_hash].append(file_to_map.as_posix())
        else:
            self.file_map[file_hash] = [file_to_map.as_posix()]

    def scan_dir(self, directory=None, root=True):
        path = Path(directory) if directory else Path.cwd()

        if root:
            print('Scanning %s for duplicate %s files' % (path.as_posix(), self.suffix))
            print('Press Control+C to cancel')
            start_time = datetime.now()

        for child in path.iterdir():
            if child.is_dir():
                self.scan_dir(child, root=False)
            else:
                self.map_file(child)

        if not self.verbose:
            print('.', end='',flush=True)

        if root:
            elapsed_seconds = (datetime.now() - start_time).total_seconds()
            print('\nScanning %s took %s seconds' % (path.as_posix(), elapsed_seconds))

    def trim_file_map(self):
        print('Cleaning up file map with %s hashes' % len(self.file_map))
        single_hashes = []
        for file_hash in self.file_map:
            if len(self.file_map[file_hash]) == 1:
                single_hashes.append(file_hash)

        for single_hash in single_hashes:
            del self.file_map[single_hash]

        # list comprehensions seem fancier, yet I still find they impact readability
        #[single_hashes.append(file_hash) for file_hash in file_map if len(file_map[file_hash]) == 1]
        #[del file_map[single_hash] for single_hash in single_hashes]

        print('File map now has %s hashes' % len(self.file_map))

    def print_dupe_report(self):
        print('The following duplicate files have been found:')
        for file_hash in self.file_map:
            if len(self.file_map[file_hash]) == 1:
                continue
            print('\nHash %s is repeated in:' % file_hash)
            print('    ' + '\n    '.join(self.file_map[file_hash]))
            
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--extension', help='Extension to scan for')
    parser.add_argument('-d', '--directory', help='Root of the directories to scan')
    parser.add_argument('-v', '--verbose', 
                        action='store_true', help='List all the files being scanned')
    args = parser.parse_args()
    
    tracker = DupeTracker(args.extension, args.verbose)
    tracker.scan_dir(args.directory)
    print()
    #tracker.trim_file_map()<commit_msg>Slightly better dictionary usage<commit_after>#!/usr/bin/python3
from collections import defaultdict
import sys
import hashlib
from pathlib import Path
from datetime import datetime 
import argparse

BLOCKSIZE = 65536
def get_file_hash(file_path):
    hasher = hashlib.sha256()
    with open(file_path, 'rb') as f:
        buf = f.read(BLOCKSIZE)
        while len(buf) > 0:
            hasher.update(buf)
            buf = f.read(BLOCKSIZE)
    return hasher.hexdigest()


class DupeTracker():
    def __init__(self,suffix=None, verbose=None):
        self.file_map = defaultdict(list)
        self.suffix = suffix
        self.verbose = verbose

    def map_file(self, file_to_map):
        if self.suffix and file_to_map.suffix.lower() != self.suffix.lower():
            return
        if self.verbose:
            print('Scanning file ' + file_to_map.as_posix())

        file_hash = get_file_hash(file_to_map.as_posix())
        self.file_map[file_hash].append(file_to_map.as_posix())

    def scan_dir(self, directory=None, root=True):
        path = Path(directory) if directory else Path.cwd()

        if root:
            print('Scanning %s for duplicate %s files' % (path.as_posix(), self.suffix))
            print('Press Control+C to cancel')
            start_time = datetime.now()

        for child in path.iterdir():
            if child.is_dir():
                self.scan_dir(child, root=False)
            else:
                self.map_file(child)

        if not self.verbose:
            print('.', end='',flush=True)

        if root:
            elapsed_seconds = (datetime.now() - start_time).total_seconds()
            print('\nScanning %s took %s seconds' % (path.as_posix(), elapsed_seconds))

    def trim_file_map(self):
        print('Cleaning up file map with %s hashes' % len(self.file_map))
        single_hashes = []
        for file_hash in self.file_map:
            if len(self.file_map[file_hash]) == 1:
                single_hashes.append(file_hash)

        for single_hash in single_hashes:
            del self.file_map[single_hash]

        # list comprehensions seem fancier, yet I still find they impact readability
        #[single_hashes.append(file_hash) for file_hash in file_map if len(file_map[file_hash]) == 1]
        #[del file_map[single_hash] for single_hash in single_hashes]

        print('File map now has %s hashes' % len(self.file_map))

    def print_dupe_report(self):
        print('The following duplicate files have been found:')
        for file_hash, dupe_list in self.file_map.items():
            if len(dupe_list) == 1:
                continue
            print('\nHash %s is repeated in:' % file_hash)
            print('    ' + '\n    '.join(dupe_list))
            
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--extension', help='Extension to scan for')
    parser.add_argument('-d', '--directory', help='Root of the directories to scan')
    parser.add_argument('-v', '--verbose', 
                        action='store_true', help='List all the files being scanned')
    args = parser.parse_args()
    
    tracker = DupeTracker(args.extension, args.verbose)
    tracker.scan_dir(args.directory)
    print()
    #tracker.trim_file_map()
{'fraction_non_alphanumeric': 0.11313868613138686, 'fraction_numerical': 0.003953771289537713, 'mean_word_length': 2.4488725747246987, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7339953', 'n_tokens_mistral': 2069, 'n_tokens_neox': 1990, 'n_words': 483}