<jupyter_start><jupyter_text>Network<jupyter_code>import math
from torch.nn import init

class DownsampleA(nn.Module):
    def __init__(self, nIn, nOut, stride):
        super(DownsampleA, self).__init__()
        assert stride == 2
        self.avg = nn.AvgPool2d(kernel_size=1, stride=stride)

    def forward(self, x):
        x = self.avg(x)
        return torch.cat((x, x.mul(0)), 1)

class ResNetBasicblock(nn.Module):
    expansion = 1
    """
    RexNet basicblock (https://github.com/facebook/fb.resnet.torch/blob/master/models/resnet.lua)
    """

    def __init__(self, inplanes, planes, stride=1, downsample=None, relu=True):
        super(ResNetBasicblock, self).__init__()

        self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn_a = nn.BatchNorm2d(planes)

        self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn_b = nn.BatchNorm2d(planes)

        self.downsample = downsample
        self.relu = relu

    def forward(self, x):
        residual = x

        basicblock = self.conv_a(x)
        basicblock = self.bn_a(basicblock)
        basicblock = F.relu(basicblock, inplace=True)

        basicblock = self.conv_b(basicblock)
        basicblock = self.bn_b(basicblock)

        if self.downsample is not None:
            residual = self.downsample(x)

        y = residual + basicblock

        if self.relu:
            y = F.relu(y)

        return y


class CifarResNet(nn.Module):
    """
    ResNet optimized for the Cifar Dataset, as specified in
    https://arxiv.org/abs/1512.03385.pdf
    """

    def __init__(self, block=ResNetBasicblock, depth=32, num_classes=100, channels=3):
        """ Constructor
        Args:
          depth: number of layers.
          num_classes: number of classes
          base_width: base width
        """
        super(CifarResNet, self).__init__()

        # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
        assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
        layer_blocks = (depth - 2) // 6

        self.num_classes = num_classes

        self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn_1 = nn.BatchNorm2d(16)

        self.inplanes = 16
        self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
        self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
        self.stage_3 = self._make_layer(block, 64, layer_blocks, 2, last=True)
        self.avgpool = nn.AvgPool2d(8)
        self.linear = nn.Linear(64, num_classes)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                init.kaiming_normal_(m.weight, nonlinearity='relu')
                # m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.weight.data.normal_(0, math.sqrt(1. / 64.))
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1, last=False):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = DownsampleA(self.inplanes, planes * block.expansion, stride)

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        if last:
            for i in range(1, blocks - 1):
                layers.append(block(self.inplanes, planes))
            layers.append(block(self.inplanes, planes, relu=False))

        else:
            for i in range(1, blocks):
                layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):

        x = self.conv_1_3x3(x)
        x = self.bn_1(x)
        x = F.relu(x)
        x = self.stage_1(x)
        x = self.stage_2(x)
        x = self.stage_3(x)
        x = self.avgpool(x)
        x = x.view(x.size(0), -1)

        return x

    def update_means(self, x, y):
        self.linear.update_means(x, y)

    def predict(self, x):
        out = self.linear(x)
        return out<jupyter_output><empty_output><jupyter_text>ICARL<jupyter_code>## parameters ##
LR = 2.
MEM_SIZE = 20*100
DECAY = 0.00001
EPOCHS = 70
LR_FACTOR = 5.

DEVICE = 'cpu'
if torch.cuda.is_available():
    DEVICE = 'cuda'

    
class ICarl:

    def __init__(self, network, n_classes=100, mem_size=MEM_SIZE,
                 lr_init=LR, decay=DECAY, epochs=EPOCHS, device=DEVICE):
        """
        :param network: the backbone neural network of the model
        :param n_classes: Number of classes of the dataset
        :param dictionary_size: Number of example in each class (must be equal for all classes)
        :param mem_size: Number of prototypes to be stored
        :param lr_init: Initial learning rate (decayed by 5 after 0.7 and 0.9 epochs
        :param decay: weight decay for SGD
        :param epochs: number of epochs to train the model
        :param device: "cuda" or "cpu"
        """
        self.network = network.to(device)
        self.network2 = self.network

        self.loss = nn.BCEWithLogitsLoss(reduction='mean')

        self.device = device
        self.dataset = None
        self.nb_cl = None

        self.lr_init = lr_init
        self.mem_size = mem_size
        self.n_classes = n_classes
        self.decay = decay
        self.epochs = epochs
        self.lr_strat = [round(epochs*0.7), round(epochs*0.9)]
        self.lr_factor = LR_FACTOR

        self.prototypes = None

        self.alpha_dr_herding = [np.array([0]) for i in range(n_classes)]

    def fit(self, dataset, nb_cl, checkpoint=None, epochs=None):

        self.alpha_dr_herding = [np.array([0]) for i in range(self.n_classes)]
        self.dataset = dataset
        self.nb_cl = nb_cl
        if epochs is not None:
            self.epochs = epochs

        x_protoset = None
        y_protoset = None

        cumulative_accuracies = []

        start_iter = 0

        if checkpoint is not None:
            check_dict = torch.load(checkpoint)
            start_iter = check_dict['iteration']
            self.network.load_state_dict(check_dict['network'])
            self.network2.load_state_dict(check_dict['network2'])
            x_protoset = check_dict['X']
            y_protoset = check_dict['Y']

        for iteration in range(start_iter, self.n_classes // nb_cl):

            # Prepare the training data for the current batch of classes
            data_loader = dataset.next_iteration(x_protoset, y_protoset)

            # TRAIN THIS ITERATION #
            print('Batch of classes number {0} arrives ...'.format(iteration + 1))

            self.incremental_fit(iteration, data_loader)  # train for N epochs (after each epoch validate)

            # END OF TRAINING FOR THIS ITERATION #

            # UPDATE EXEMPLARS #
            print('Updating exemplar set...')
            x_protoset, y_protoset = self.update_exemplars(iteration)

            # Save training checkpoint
            torch.save({
                'iteration': iteration,
                'network': self.network.state_dict(),
                'network2': self.network2.state_dict(),
                'X': x_protoset,
                'Y': y_protoset
            }, "checkpoint/iter_" + str(iteration) + "_checkpoint.pth.tar")

            means = self.compute_means(iteration)
            # COMPUTE ACCURACY ##
            acc_cum = self.test(iteration, means)

            print("Cumulative results")
            print("  top 1 accuracy iCaRL          :\t{:.2f} %".format(acc_cum[0]))
            print("  top 1 accuracy Hybrid 1       :\t{:.2f} %".format(acc_cum[1]))
            print("  top 1 accuracy NCM            :\t{:.2f} %".format(acc_cum[2]))

            acc_new = self.test(iteration, means, cumulative=False)

            print("New batch results")
            print("  top 1 accuracy iCaRL          :\t{:.2f} %".format(acc_new[0]))
            print("  top 1 accuracy Hybrid 1       :\t{:.2f} %".format(acc_new[1]))
            print("  top 1 accuracy NCM            :\t{:.2f} %".format(acc_new[2]))

            acc_base = self.test(0, means)

            print("First batch results")
            print("  top 1 accuracy iCaRL          :\t{:.2f} %".format(acc_base[0]))
            print("  top 1 accuracy Hybrid 1       :\t{:.2f} %".format(acc_base[1]))
            print("  top 1 accuracy NCM            :\t{:.2f} %".format(acc_base[2]))

            cumulative_accuracies.append(acc_cum)

            print("")

        torch.save(
            {
             "network": self.network.state_dict(),
             "accuracy": cumulative_accuracies
             },
            f"models/icarl{datetime.now().isoformat()}.pth"
        )

        return cumulative_accuracies

    def incremental_fit(self, iteration, data_loader):
        new_lr = self.lr_init
        optimizer = optim.SGD(filter(lambda p: p.requires_grad, self.network.parameters()), lr=new_lr, momentum=0.9,
                              weight_decay=self.decay, nesterov=False)

        for epoch in range(self.epochs):
            self.network.train()
            train_loss = 0
            correct = 0
            total = 0

            # In each epoch, we do a full pass over the training data:
            for inputs, targets_prep in data_loader:

                targets = np.zeros((inputs.shape[0], 100), np.float32)  # 100 = classes of cifar
                targets[range(len(targets_prep)), targets_prep.type(torch.int32)] = 1.  # prepare target for CE loss

                inputs = inputs.to(self.device)

                optimizer.zero_grad()
                outputs = self.network.forward(inputs)  # feature vector only
                prediction = self.network.predict(outputs)  # make the prediction with sigmoid, making g_y(xi)
                targets = tensor(targets).to(outputs.device)
                targets_prep = torch.LongTensor(targets_prep).to(outputs.device)

                if iteration > 0:  # apply distillation
                    outputs_old = self.network2.forward(inputs)
                    prediction_old = self.network2.predict(outputs_old)
                    targets[:, np.array(self.dataset.order[range(0, iteration * self.nb_cl)])] = \
                        torch.sigmoid(prediction_old[:, np.array(self.dataset.order[range(0, iteration * self.nb_cl)])])

                loss_bx = self.loss(prediction, targets)  # joins classification and distillation losses
                loss_bx.backward()
                optimizer.step()

                train_loss += loss_bx.item()
                _, predicted = prediction.max(1)
                total += targets.size(0)
                correct += predicted.eq(targets_prep).sum().item()

            # END loop minibatches
            # self.network.eval()
            # test_loss = 0
            # correct = 0
            # total = 0
            # # count = 0
            # for inputs, targets_prep in self.dataset.minibatches(train=False):
            #     # count += 1
            #
            #     targets = np.zeros((inputs.shape[0], 100), np.float32)
            #     targets[range(len(targets_prep)), targets_prep.type(torch.int32)] = 1.
            #
            #     inputs = inputs.to(self.device)
            #
            #     outputs = self.network.forward(inputs)  # make the embedding
            #     outputs = self.network.predict(outputs)  # make the prediction with sigmoid, making g_y(xi)
            #
            #     targets = torch.tensor(targets).to(outputs.device)
            #     loss_bx = self.loss(outputs, targets)
            #     test_loss += loss_bx.item()
            #
            #     targets_prep = torch.LongTensor(targets_prep).to(outputs.device)
            #     _, predicted = outputs.max(1)
            #     correct += predicted.eq(targets_prep).sum().item()
            #
            #     total += targets.size(0)
            #

            acc = 100. * correct / total

            #if (epoch+1) % (self.epochs//10) == 0:
            #    print(f"Epoch {epoch + 1} : Train Loss {train_loss / total:.8f}, Train Acc {acc:.2f}")

            # adjust learning rate
            if (epoch + 1) in self.lr_strat:
                new_lr = new_lr / self.lr_factor
                print("New LR:" + str(new_lr))
                optimizer = optim.SGD(filter(lambda p: p.requires_grad, self.network.parameters()), lr=new_lr, momentum=0.9,
                                      weight_decay=self.decay, nesterov=False)

        # Duplicate current network to distillate info
        self.network2 = copy.deepcopy(self.network)
        self.network2.eval()

    def update_exemplars(self, iteration):
        nb_protos_cl = int(np.ceil(self.mem_size / self.compute_num_classes(iteration)))  # num of exemplars per class
        # Herding
        self.network.eval()

        # Prepare the protoset
        x_protoset = None
        y_protoset = None

        print(nb_protos_cl)
        
        if nb_protos_cl > 0:

            for iter_dico in range(self.nb_cl):
                cl = self.dataset.order[self.compute_num_classes(iteration-1) + iter_dico].item()  # pick actual class

                pinput = self.dataset.get_dataloader_of_class(cl)

                output = []
                for img, tar in pinput:
                    img = img.to(self.device)
                    output.append(self.network.forward(img).cpu().detach())

                # Collect data in the feature space for each class
                mapped_prototypes = torch.cat(output).numpy()
                print([hash(str(i)) for i in img[:]])
                D = mapped_prototypes.T
                D = D / np.linalg.norm(D, axis=0)
                # Herding procedure : ranking of the potential exemplars
                mu = np.mean(D, axis=1)

                print(f"Shape of output {mapped_prototypes.shape}")
                # set exemplar to zero
                self.alpha_dr_herding[cl] = np.zeros(mapped_prototypes.shape[0])  # number of rows
                w_t = mu
                iter_herding = 0
                iter_herding_eff = 0
                # Herding algorithm
                while not (np.sum(self.alpha_dr_herding[cl] != 0) == min(nb_protos_cl, 500)) and iter_herding_eff < 1000:
                    tmp_t = np.dot(w_t, D)
                    ind_max = np.argmax(tmp_t)
                    iter_herding_eff += 1
                    if self.alpha_dr_herding[cl][ind_max] == 0:
                        self.alpha_dr_herding[cl][ind_max] = 1 + iter_herding
                        iter_herding += 1
                    w_t = w_t + mu - D[:, ind_max]
                print(f"Chosen exemplars {np.max(self.alpha_dr_herding[cl])}")

            x_protoset = []
            y_protoset = []

            # Storing the selected exemplars in the protoset
            for iteration2 in range(iteration + 1):

                for iter_dico in range(self.nb_cl):
                    cl = self.dataset.order[iteration2 * self.nb_cl + iter_dico].item()

                    alph = self.alpha_dr_herding[cl]  # select the herd of the current class
                    alph = (alph > 0) * (alph < nb_protos_cl + 1) * 1.  # put one in the ones to select

                    # append exeplars in the protoset
                    img = self.dataset.get_images_of_class(cl)
                    x_protoset += [img[j] for j in range(len(alph)) if alph[j] == 1]
                    y_protoset += [cl for j in range(len(alph)) if alph[j] == 1]

        print("Protoset shape:")
        print(len(x_protoset))
        return x_protoset, y_protoset

    def compute_num_classes(self, iteration):
        return self.nb_cl * (iteration + 1)

    def compute_means(self, iteration):

        class_means = np.zeros((64, 100, 2))
        nb_protos_cl = int(np.ceil(self.mem_size / self.compute_num_classes(iteration)))  # num of exemplars per class

        for iteration2 in range(iteration+1):

            for iter_dico in range(self.nb_cl):
                cl = self.dataset.order[self.compute_num_classes(iteration2-1) + iter_dico].item()  # pick actual class

                print(f"Class {cl}")
                
                # compute network resposes for images of class cl
                pinput = self.dataset.get_dataloader_of_class(cl)
                output = []
                for img, tar in pinput:
                    img = img.to(self.device)
                    output.append(self.network.forward(img).cpu().detach())
                    
                # Collect data in the feature space for each class
                mapped_prototypes = torch.cat(output).numpy()  # should be 500 x 64 in CIFAR
                print([hash(str(i)) for i in img[:]])
                
                print(f"Output shape {mapped_prototypes.shape}")
                
                D = mapped_prototypes.T  # now each column is a sample
                D = D / np.linalg.norm(D, axis=0)

                # iCaRL
                alph = self.alpha_dr_herding[cl]  # importance of each image of this class
                dict_size = len(self.alpha_dr_herding[cl])
                alph = (alph > 0) * (alph < nb_protos_cl + 1) * 1.  # 1 if in the current herd

                # print(self.alpha_dr_herding[cl])
                # Handle the case in which there are no prototypes
                s = np.sum(alph)
                if s == 0:
                    s = 1

                print(f"Exemplars {s}")
                alph = alph / s  # to make the average only for the current prototypes.
                class_means[:, cl, 0] = (np.dot(D, alph))
                # dot operation is for weighting each f(xi) with alpha
                class_means[:, cl, 0] /= np.linalg.norm(class_means[:, cl, 0])

                # Normal NCM
                alph = np.ones(dict_size) / dict_size  # to make the avg over all samples
                class_means[:, cl, 1] = (np.dot(D, alph))
                # dot operation is for weighting each f(xi) with alpha
                class_means[:, cl, 1] /= np.linalg.norm(class_means[:, cl, 1])
                
                print("")
                
        return class_means

    def test(self, iteration, class_means=None, cumulative=True):

        if class_means is None and self.mem_size > 0:
            class_means = self.compute_means(iteration)

        top1_acc_list = np.zeros(3)

        stat_hb1 = []
        stat_icarl = []
        stat_ncm = []

        data_loader = self.dataset.test_dataloader(iteration, cumulative=cumulative)

        for inputs, targets_prep in data_loader:
            inputs = inputs.to(self.device)

            # compute prediction
            outputs = self.network.forward(inputs)  # returns embeddings
            pred = self.network.predict(outputs).cpu().detach().numpy()  # return score classes as logits
            outputs = outputs.cpu().detach().numpy()

            outputs = (outputs.T / np.linalg.norm(outputs.T, axis=0)).T  # normalize output

            # Compute the accuracy over the batch
            targets_prep = targets_prep.numpy()

            if self.mem_size > 0:
                # Compute score for iCaRL
                sqd = cdist(class_means[:, :, 0].T, outputs, 'sqeuclidean')  # Squared euclidean distance
                score_icarl = (-sqd).T
                
                # Compute score for NCM
                sqd = cdist(class_means[:, :, 1].T, outputs, 'sqeuclidean')  # Squared euclidean distance
                score_ncm = (-sqd).T

                stat_icarl += ([ll in best for ll, best in zip(targets_prep, np.argsort(score_icarl, axis=1)[:, -1:])])
                stat_ncm += ([ll in best for ll, best in zip(targets_prep, np.argsort(score_ncm, axis=1)[:, -1:])])

            stat_hb1 += ([ll in best for ll, best in zip(targets_prep, np.argsort(pred, axis=1)[:, -1:])])
            # use the logits

        if self.mem_size > 0:
            top1_acc_list[0] = np.average(stat_icarl) * 100.  # ICarl
            top1_acc_list[2] = np.average(stat_ncm) * 100.  # NCM

        top1_acc_list[1] = np.average(stat_hb1) * 100.  # Hybrid 1

        return top1_acc_list

    def predict(self, inputs, method=0):
        """
        :return the predicted class for the inputs as tensor in cpu

        :param inputs: tensors to be evaluated
        :param method: 0 (def) is ICaRL, 1 is NCM, 2 is ICaRL-inv, 3 is with sigmoid
        """
        class_means = self.compute_means(self.n_classes // self.nb_cl)
        outputs = self.network.forward(inputs)  # returns embeddings
        if method == 3:
            pred = self.network.predict(outputs).cpu().detach()
        elif 0 <= method < 3:
            outputs = outputs.cpu().detach().numpy()
            outputs = (outputs.T / np.linalg.norm(outputs.T, axis=0)).T  # normalize output
            sqd = cdist(class_means[:, :, method].T, outputs, 'sqeuclidean')  # Squared euclidean distance
            score = (-sqd).T
            pred = np.argsort(score, axis=1)[:, -1:]
            pred = torch.tensor(pred)
        else:
            raise Exception("Pass method between 0 and 3 inclusive")

        return pred<jupyter_output><empty_output><jupyter_text>MAIN<jupyter_code>######### Modifiable Settings ##########

batch_size = 128  # Batch size
n = 5  # Set the depth of the architecture: n = 5 -> 32 layers (See He et al. paper)
nb_cl = 2  # Classes per group
nb_runs = 10  # Number of runs (random ordering of classes at each run)
run = 0
########################################

# get the data
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize([0.5,0.5,0.5], [0.5,0.5,0.5])]) # range between -1,1 : (x - 0.5) * 2

augmentation = transforms.RandomCrop((32,32), padding=4)

data = ICIFAR('data', order_file='data/cifar_order.npy', download=False,
              num_cl_first=nb_cl, num_cl_after=nb_cl, 
              augmentation=augmentation, transform=transform, 
              batch_size=batch_size, run_number=run, workers=8)
# define network
network = CifarResNet()
# define the method
icarl = ICarl(network, device='cuda', epochs=1) 
# run fit!
acc = icarl.fit(data, nb_cl)

with open(f"icifar.txt", "a") as f:
    f.write(str(datetime.utcnow()))
    f.write(f"\nrun:{run}\n")
    f.write(str(acc))


data.order
t = torch.randn(10,3,3)
print(t)
[hash(str(i)) for i in t[:]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11404089925356654, 'fraction_numerical': 0.016386202022878847, 'mean_word_length': 1.8991037131882202, 'pattern_counts': {'":': 2, '<': 18, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '75952', 'license': 'MIT', 'n_tokens_mistral': 7467, 'n_tokens_neox': 6862, 'n_words': 1629, 'path': 'jupyter/ICaRL_test.ipynb', 'repo_name': 'fcdl94/ICL', 'size': '22641'}