<commit_before>#!/usr/bin/python

# Methods to handle local files


import hashlib
import os
import random


class LocalContainer(object):
    def __init__(self, name):
        self.name = name
        self.path = self.name.replace('file://', '')
        self.region = 'local'

    def get_directory(self, path):
        return LocalDirectory(self.region, self.path, path)


class LocalDirectory(object):
    def __init__(self, region, parent, path):
        self.region = region
        if not path:
            self.path = parent
        else:
            self.path = os.path.join(parent, path)

    def listdir(self):
        ents = os.listdir(self.path)
        random.shuffle(ents)
        for ent in ents:
            yield ent

    def get_file(self, path):
        fullpath = os.path.join(self.path, path)
        return LocalFile(self.region, fullpath)


class LocalFile(object):
    def __init__(self, region, path):
        self.region = region
        self.path = path
        self.cache = {}

    def checksum(self):
        if 'checksum' in self.cache:
            return self.cache['checksum']

        h = hashlib.sha512()
        with open(self.path, 'r') as f:
            d = f.read(1024 * 1204)
            while d:
                h.update(d)
                d = f.read(1024 * 1024)
        self.cache['checksum'] = h.hexdigest()
        return self.cache['checksum']

    def size(self):
        if 'size' in self.cache:
            return self.cache['size']

        self.cache['size'] = os.path.getsize(self.path)
        return self.cache['size']

    def isdir(self):
        return os.path.isdir(self.path)

    def islink(self):<commit_msg>Handle a sync to local<commit_after>#!/usr/bin/python

# Methods to handle local files


import datetime
import hashlib
import os
import random
import shutil


class LocalContainer(object):
    def __init__(self, name):
        self.name = name
        self.path = self.name.replace('file://', '')
        self.region = 'local'

    def get_directory(self, path):
        return LocalDirectory(self.region, self.path, path)


class LocalDirectory(object):
    def __init__(self, region, parent, path):
        self.region = region
        if not path:
            self.path = parent
        else:
            self.path = os.path.join(parent, path)

    def listdir(self):
        ents = os.listdir(self.path)
        random.shuffle(ents)
        for ent in ents:
            yield ent

    def get_file(self, path):
        fullpath = os.path.join(self.path, path)
        return LocalFile(self.region, fullpath)

    def update_shalist(self, path, value):
        pass

    def write_shalist(self):
        pass


class LocalFile(object):
    def __init__(self, region, path):
        self.region = region
        self.path = path
        self.cache = {}

    def checksum(self):
        if 'checksum' in self.cache:
            return self.cache['checksum']

        h = hashlib.sha512()
        with open(self.path, 'r') as f:
            d = f.read(1024 * 1204)
            while d:
                h.update(d)
                d = f.read(1024 * 1024)
        self.cache['checksum'] = h.hexdigest()
        return self.cache['checksum']

    def size(self):
        if 'size' in self.cache:
            return self.cache['size']

        self.cache['size'] = os.path.getsize(self.path)
        return self.cache['size']

    def store(self, local_file):
        d = os.path.dirname(self.path)
        if not os.path.exists(d):
            os.makedirs(d)
        print '%s Renaming %s to %s' %(datetime.datetime.now(), local_file, self.path)
        shutil.copy(local_file, self.path)

    def isdir(self):
        return os.path.isdir(self.path)

    def islink(self):
{'fraction_non_alphanumeric': 0.1198181331906927, 'fraction_numerical': 0.010163145225996255, 'mean_word_length': 2.0161290322580645, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4363637', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1136, 'n_words': 285}