<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

################################################################################
# Copyright (c) 2020 ContinualAI Research                                      #
# Copyrights licensed under the CC BY 4.0 License.                             #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 1-05-2020                                                              #
# Author(s): <NAME>                                                 #
# E-mail: <EMAIL>                                              #
# Website: clair.continualai.org                                               #
################################################################################

""" Continual Data Loader for the MNIST continual learning benchmark. """

# Python 2-3 compatible
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

# other imports
import logging
from avalanche.benchmarks.datasets_envs import MNIST
from avalanche.benchmarks.utils import remove_some_labels


class CMNIST(object):
    """ Continuous MNIST benchmark data loader. """

    def __init__(self, bp=None, num_batch=10, mode='perm', task_sep=True,
                 eval_protocol=None):

        """" Initialize Object. mode={perm|split|rot}. """

        # checks
        if num_batch > 5 and mode == "split":
            raise Exception("Split mode cannot have more than 5 tasks!")

        self.bp = bp
        self.num_batch = num_batch
        self.iter = 0
        self.task_sep = task_sep
        self.eval_protocol = eval_protocol

        # Getting root logger
        self.log = logging.getLogger('mylogger')

        if self.bp is None:
            self.mnist = MNIST()
        else:
            self.mnist = MNIST(data_loc=bp)

        self.train_set, self.test_set = self.mnist.get_data()

        # to be filled
        self.all_train_sets = []
        self.all_test_sets = []
        self.tasks_id = []

        print("preparing CL benchmark...")
        for i in range(self.num_batch):

            if mode == "perm":

                train_x, test_x = self.mnist.permute_mnist(seed=i)
                train_y, test_y = self.train_set[1], self.test_set[1]
            elif mode == "rot":

                train_x, test_x = self.mnist.rotate_mnist(rotation=i*5)
                train_y, test_y = self.train_set[1], self.test_set[1]
            elif mode == "split":

                train_x, train_y = remove_some_labels(
                    self.train_set, [j for j in range(10)
                                         if j not in [i * 2, i * 2 + 1]]
                )
                test_x, test_y = remove_some_labels(
                    self.test_set, [j for j in range(10)
                                        if j not in [i * 2, i * 2 + 1]]
                )
            else:
                raise NotImplemented

            self.all_train_sets.append([train_x, train_y])
            self.all_test_sets.append([test_x, test_y])
            if self.task_sep:
                self.tasks_id.append(i)
            else:
                self.tasks_id.append(0)

    def __iter__(self):
        return self

    def __next__(self):
        """ Next batch based on the object parameter which can be also changed
            from the previous iteration. """

        if self.iter == self.num_batch:
            raise StopIteration

        train_set = self.all_train_sets[self.iter]

        # get ready for next iter
        self.iter += 1

        return train_set[0], train_set[1], self.tasks_id[self.iter-1]

    def get_full_testset(self):
        """
        Return the test set (the same for each inc. batch).
        """<commit_msg>Minor bug fix in raise Error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

################################################################################
# Copyright (c) 2020 ContinualAI Research                                      #
# Copyrights licensed under the CC BY 4.0 License.                             #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 1-05-2020                                                              #
# Author(s): <NAME>                                                 #
# E-mail: <EMAIL>                                              #
# Website: clair.continualai.org                                               #
################################################################################

""" Continual Data Loader for the MNIST continual learning benchmark. """

# Python 2-3 compatible
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

# other imports
import logging
from avalanche.benchmarks.datasets_envs import MNIST
from avalanche.benchmarks.utils import remove_some_labels


class CMNIST(object):
    """ Continuous MNIST benchmark data loader. """

    def __init__(self, bp=None, num_batch=10, mode='perm', task_sep=True,
                 eval_protocol=None):

        """" Initialize Object. mode={perm|split|rot}. """

        # checks
        if num_batch > 5 and mode == "split":
            raise Exception("Split mode cannot have more than 5 tasks!")

        self.bp = bp
        self.num_batch = num_batch
        self.iter = 0
        self.task_sep = task_sep
        self.eval_protocol = eval_protocol

        # Getting root logger
        self.log = logging.getLogger('mylogger')

        if self.bp is None:
            self.mnist = MNIST()
        else:
            self.mnist = MNIST(data_loc=bp)

        self.train_set, self.test_set = self.mnist.get_data()

        # to be filled
        self.all_train_sets = []
        self.all_test_sets = []
        self.tasks_id = []

        print("preparing CL benchmark...")
        for i in range(self.num_batch):

            if mode == "perm":

                train_x, test_x = self.mnist.permute_mnist(seed=i)
                train_y, test_y = self.train_set[1], self.test_set[1]
            elif mode == "rot":

                train_x, test_x = self.mnist.rotate_mnist(rotation=i*5)
                train_y, test_y = self.train_set[1], self.test_set[1]
            elif mode == "split":

                train_x, train_y = remove_some_labels(
                    self.train_set, [j for j in range(10)
                                         if j not in [i * 2, i * 2 + 1]]
                )
                test_x, test_y = remove_some_labels(
                    self.test_set, [j for j in range(10)
                                        if j not in [i * 2, i * 2 + 1]]
                )
            else:
                raise NotImplementedError()

            self.all_train_sets.append([train_x, train_y])
            self.all_test_sets.append([test_x, test_y])
            if self.task_sep:
                self.tasks_id.append(i)
            else:
                self.tasks_id.append(0)

    def __iter__(self):
        return self

    def __next__(self):
        """ Next batch based on the object parameter which can be also changed
            from the previous iteration. """

        if self.iter == self.num_batch:
            raise StopIteration

        train_set = self.all_train_sets[self.iter]

        # get ready for next iter
        self.iter += 1

        return train_set[0], train_set[1], self.tasks_id[self.iter-1]

    def get_full_testset(self):
        """
        Return the test set (the same for each inc. batch).
        """
{'fraction_non_alphanumeric': 0.12864892792560062, 'fraction_numerical': 0.010591578403513303, 'mean_word_length': 1.5155945419103314, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5649418', 'n_tokens_mistral': 2131, 'n_tokens_neox': 1974, 'n_words': 535}