<commit_before>from proton_decay_study.generators.base import BaseDataGenerator
import h5py
import logging


class SingleFileDataGenerator(BaseDataGenerator):
  """
    Creates a generator for a single file
  """
  logger = logging.getLogger("pdk.gen.single")

  def __init__(self, datapath, dataset, labelset, batch_size=10):
    self._file = h5py.File(datapath,'r')
    self._dataset = self._file[dataset]
    self._labelset = self._file[labelset]
    self.current_index=0
    self.batch_size = batch_size

  def __len__(self):
    return self._dataset.shape[0]


  def next(self):
    if self.current_index>= len(self):
        self.logger.info("Reusing Data at Size: {}".format(len(self)))
        self.current_index = 0
    if self.current_index+self.batch_size>= len(self):
        self.batch_size = len(self)-current_index
    x = self._dataset[self.current_index:self.current_index+self.batch_size]
    y = self._labelset[self.current_index:self.current_index+self.batch_size]
    self.current_index+=self.batch_size
    return (x,y)<commit_msg>Bugfix<commit_after>from proton_decay_study.generators.base import BaseDataGenerator
import h5py
import logging


class SingleFileDataGenerator(BaseDataGenerator):
  """
    Creates a generator for a single file
  """
  logger = logging.getLogger("pdk.gen.single")

  def __init__(self, datapath, dataset, labelset, batch_size=10):
    self._file = h5py.File(datapath,'r')
    self._dataset = self._file[dataset]
    self._labelset = self._file[labelset]
    self.current_index=0
    self.batch_size = batch_size

  def __len__(self):
    return self._dataset.shape[0]


  def next(self):
    if self.current_index>= len(self):
        self.logger.info("Reusing Data at Size: {}".format(len(self)))
        self.current_index = 0
    if self.current_index+self.batch_size>= len(self):
        self.batch_size = len(self)-self.current_index
    x = self._dataset[self.current_index:self.current_index+self.batch_size]
    y = self._labelset[self.current_index:self.current_index+self.batch_size]
    self.current_index+=self.batch_size
    return (x,y)
{'fraction_non_alphanumeric': 0.11644486692015209, 'fraction_numerical': 0.006653992395437262, 'mean_word_length': 4.236318407960199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '123815', 'n_tokens_mistral': 703, 'n_tokens_neox': 684, 'n_words': 123}