<commit_before>import logging
from time import time

import numpy as np


class HDF5CacheBlock(object):
    def __init__(self, shape, fillvalue, dtype):
        self.has_new_data = True
        self.flush_time = time()
        self.data = np.full(shape, fillvalue, dtype)


class HDF5UnlimitedCache(object):
    """ An object to represent a cache used by an HDF5Dataset """

    def __init__(self, name, dtype, fillvalue, block_size, stale_time, shape=None):
        self._name = name
        self._dtype = dtype
        self._fillvalue = fillvalue
        self._block_size = block_size
        self._blocks = {}
        self._highest_index = 0
        if shape is None:
            self._shape = block_size
        else:
            self._shape = (block_size,) + shape
        self._blocks[0] = HDF5CacheBlock(self._shape, self._fillvalue, self._dtype)
        self._stale_time = stale_time
        self._logger = logging.getLogger("HDF5UnlimitedCache")
        self._logger.debug(
            "Created unlimited HDF5UnlimitedCache for {}".format(self._name)
        )

    def add_value(self, value, offset):
        if offset > self._highest_index:
            self._highest_index = offset
        block_index, value_index = divmod(offset, self._block_size)
        if block_index not in self._blocks:
            self._logger.debug("New cache block required")
            self._blocks[block_index] = HDF5CacheBlock(
                self._shape, self._fillvalue, self._dtype
            )
        self._blocks[block_index].has_new_data = True
        self._blocks[block_index].data[value_index] = value

    def purge_blocks(self):<commit_msg>Extend add_value API to allow append by calling without an offset<commit_after>import logging
from time import time

import numpy as np


class HDF5CacheBlock(object):
    def __init__(self, shape, fillvalue, dtype):
        self.has_new_data = True
        self.flush_time = time()
        self.data = np.full(shape, fillvalue, dtype)


class HDF5UnlimitedCache(object):
    """ An object to represent a cache used by an HDF5Dataset """

    def __init__(self, name, dtype, fillvalue, block_size, stale_time, shape=None):
        self._name = name
        self._dtype = dtype
        self._fillvalue = fillvalue
        self._block_size = block_size
        self._blocks = {}
        self._highest_index = -1
        if shape is None:
            self._shape = block_size
        else:
            self._shape = (block_size,) + shape
        self._blocks[0] = HDF5CacheBlock(self._shape, self._fillvalue, self._dtype)
        self._stale_time = stale_time
        self._logger = logging.getLogger("HDF5UnlimitedCache")
        self._logger.debug(
            "Created unlimited HDF5UnlimitedCache for {}".format(self._name)
        )

    def add_value(self, value, offset=None):
        if offset is None:
            offset = self._highest_index + 1
        if offset > self._highest_index:
            self._highest_index = offset
        block_index, value_index = divmod(offset, self._block_size)
        if block_index not in self._blocks:
            self._logger.debug("New cache block required")
            self._blocks[block_index] = HDF5CacheBlock(
                self._shape, self._fillvalue, self._dtype
            )
        self._blocks[block_index].has_new_data = True
        self._blocks[block_index].data[value_index] = value

    def purge_blocks(self):
{'fraction_non_alphanumeric': 0.08942139099941554, 'fraction_numerical': 0.005552308591466978, 'mean_word_length': 2.4786585365853657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2761218', 'n_tokens_mistral': 1013, 'n_tokens_neox': 982, 'n_words': 242}