<commit_before>import os
import bisect
import hashlib
import binascii
import RandomIO
import partialhash
from datetime import datetime
from dataserv_client import control
from dataserv_client import common


logger = common.logging.getLogger(__name__)


class Builder:

    def __init__(self, address, shard_size, max_size, on_generate_shard=None,
                 use_folder_tree=False):
        self.target_height = int(max_size / shard_size)
        self.address = address
        self.shard_size = shard_size
        self.max_size = max_size
        self.use_folder_tree = use_folder_tree
        self.on_generate_shard = on_generate_shard

    @staticmethod
    def sha256(content):
        """Finds the SHA-256 hash of the content."""
        content = content.encode('utf-8')
        return hashlib.sha256(content).hexdigest()

    def _build_all_seeds(self, height):
        """Includes seed for height 0."""
        seed = self.sha256(self.address)
        seeds = [seed]
        for i in range(height):
            seed = self.sha256(seed)
            seeds.append(seed)
        return seeds

    def build_seeds(self, height):
        """Deterministically build seeds."""
        return self._build_all_seeds(height)[:height]

    def build_seed(self, height):
        """Deterministically build a seed."""
        return self._build_all_seeds(height).pop()

    def _get_shard_path(self, store_path, seed, create_needed_folders=False):
        if self.use_folder_tree:
            folders = os.path.join(*control.util.chunks(seed, 3))
            store_path = os.path.join(store_path, folders)
            if create_needed_folders:
                control.util.ensure_path_exists(store_path)
        return os.path.join(store_path, seed)

    def generate_shard(self, seed, store_path, cleanup=False):
        """
        Save a shard, and return its SHA-256 hash.

        :param seed: Seed pased to RandomIO to generate file.
        :param store_path: What path to store the file.
        :param cleanup: Delete the file after generation.
        :return: SHA-256 hash of the file.
        """

        # save the shard
        path = self._get_shard_path(store_path, seed,
                                    create_needed_folders=True)
        RandomIO.RandomIO(seed).genfile(self.shard_size, path)

        # get the file hash
        with open(path, 'rb') as f:
            file_hash = hashlib.sha256(f.read()).hexdigest()

        # remove file if requested
        if cleanup:
            os.remove(path)

        return file_hash

    def filter_to_resume_point(self, store_path, enum_seeds):<commit_msg>retry write once on IOError<commit_after>import os
import bisect
import hashlib
import binascii
import RandomIO
import partialhash
from datetime import datetime
from dataserv_client import control
from dataserv_client import common


logger = common.logging.getLogger(__name__)


class Builder:

    def __init__(self, address, shard_size, max_size, on_generate_shard=None,
                 use_folder_tree=False):
        self.target_height = int(max_size / shard_size)
        self.address = address
        self.shard_size = shard_size
        self.max_size = max_size
        self.use_folder_tree = use_folder_tree
        self.on_generate_shard = on_generate_shard

    @staticmethod
    def sha256(content):
        """Finds the SHA-256 hash of the content."""
        content = content.encode('utf-8')
        return hashlib.sha256(content).hexdigest()

    def _build_all_seeds(self, height):
        """Includes seed for height 0."""
        seed = self.sha256(self.address)
        seeds = [seed]
        for i in range(height):
            seed = self.sha256(seed)
            seeds.append(seed)
        return seeds

    def build_seeds(self, height):
        """Deterministically build seeds."""
        return self._build_all_seeds(height)[:height]

    def build_seed(self, height):
        """Deterministically build a seed."""
        return self._build_all_seeds(height).pop()

    def _get_shard_path(self, store_path, seed, create_needed_folders=False):
        if self.use_folder_tree:
            folders = os.path.join(*control.util.chunks(seed, 3))
            store_path = os.path.join(store_path, folders)
            if create_needed_folders:
                control.util.ensure_path_exists(store_path)
        return os.path.join(store_path, seed)

    def generate_shard(self, seed, store_path, cleanup=False):
        """
        Save a shard, and return its SHA-256 hash.

        :param seed: Seed pased to RandomIO to generate file.
        :param store_path: What path to store the file.
        :param cleanup: Delete the file after generation.
        :return: SHA-256 hash of the file.
        """

        # save the shard
        path = self._get_shard_path(store_path, seed,
                                    create_needed_folders=True)
        try:
            RandomIO.RandomIO(seed).genfile(self.shard_size, path)
        except IOError as e:
            msg = "Failed to write shard, will try once more! '{0}'"
            logger.error(msg.format(repr(e)))
            time.sleep(2)
            RandomIO.RandomIO(seed).genfile(self.shard_size, path)

        # get the file hash
        with open(path, 'rb') as f:
            file_hash = hashlib.sha256(f.read()).hexdigest()

        # remove file if requested
        if cleanup:
            os.remove(path)

        return file_hash

    def filter_to_resume_point(self, store_path, enum_seeds):
{'fraction_non_alphanumeric': 0.08876811594202899, 'fraction_numerical': 0.010144927536231883, 'mean_word_length': 2.444167186525265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4907521', 'n_tokens_mistral': 1695, 'n_tokens_neox': 1605, 'n_words': 412}