<commit_before>from pylearn2.utils import serial #rm
import numpy as N#rm

class DefaultTrainingAlgorithm:
    def __init__(self, batch_size, batches_per_iter , monitoring_batches = - 1, monitoring_dataset = None):
        self.batch_size, self.batches_per_iter = batch_size, batches_per_iter
        if monitoring_dataset is None:
            assert monitoring_batches == -1
        self.monitoring_dataset, self.monitoring_batches = monitoring_dataset, monitoring_batches
    #

    def train(self, model, dataset):

        if len(model.error_record) == 0 and self.monitoring_dataset:
            self.monitor(model)
        #

        for i in xrange(self.batches_per_iter):
            model.learn(dataset, self.batch_size)
        #

        if self.monitoring_dataset:
            self.monitor(model)
        #

        return True
    #
<commit_msg>default training algorithm now supports models that have a fixed batch size<commit_after>from pylearn2.utils import serial #rm
import numpy as N#rm

class DefaultTrainingAlgorithm:
    def __init__(self, batch_size = None , batches_per_iter = 1000 , monitoring_batches = - 1, monitoring_dataset = None):
        """
        if batch_size is None, reverts to the force_batch_size field of the model
        """

        self.batch_size, self.batches_per_iter = batch_size, batches_per_iter
        if monitoring_dataset is None:
            assert monitoring_batches == -1
        self.monitoring_dataset, self.monitoring_batches = monitoring_dataset, monitoring_batches
    #

    def train(self, model, dataset):

        if self.batch_size is None:
            batch_size = model.force_batch_size
        else:
            batch_size = self.batch_size
            if hasattr(model.force_batch_size):
                assert model.force_batch_size <= 0 or batch_size == model.force_batch_size

        if len(model.error_record) == 0 and self.monitoring_dataset:
            self.monitor(model)
        #

        for i in xrange(self.batches_per_iter):
            model.learn(dataset, batch_size)
        #

        if self.monitoring_dataset:
            self.monitor(model)
        #

        return True
    #

{'fraction_non_alphanumeric': 0.07136279926335175, 'fraction_numerical': 0.005985267034990792, 'mean_word_length': 2.521880064829822, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6026977', 'n_tokens_mistral': 635, 'n_tokens_neox': 606, 'n_words': 152}