<jupyter_start><jupyter_text>Loss  Loss BCE<jupyter_code>class BCE_sigmoid(nn.Module):
    def __init__(self, size_average = False):
        super(BCE_sigmoid, self).__init__()
        self.size_average = size_average

    def forward(self, x, labels):
        N = x.size(0)  # batch size
        mask1 = labels.eq(0) 

        mask = 1 - mask1.float()

        target = labels.gt(0) 
        target = target.float()

        # This loss combines a Sigmoid layer and the BCELoss in one single class
        loss = F.binary_cross_entropy_with_logits(x, target, size_average = False)

        if self.size_average:
            loss = loss/N

        return loss

class BCE_sigmoid_negtive_bias_all(nn.Module):
    def __init__(self, size_average = False, AU_num = 12, AU_idx = [0,1,2,3,4,5,6,7,8,9,10,11], database = 0):
        super(BCE_sigmoid_negtive_bias_all, self).__init__()
        self.size_average = size_average
        self.AU_num = AU_num
        self.AU_idx = AU_idx
        self.boundary = 1

        # balance weights for different databases
        if database == 0:
            self.weight = torch.tensor([0.3, 0.2, 0.3, 0.2, 0.2, 0.2, 0.2, 0.3, 0.2, 0.5, 0.2, 0.1])
            self.weight = self.weight.cuda()
        elif database == 1:
            self.weight = [0.3, 0.3, 0.5, 0.3, 0.3, 0.5, 0.3, 0.5, 0.3, 0.3, 0.3, 0.3]

        self.balance_a = []
        for i in range(0,self.AU_num):
            self.balance_a.append(self.weight[self.AU_idx[i]])

    def forward(self, x, labels):
        N = x.size(0)

        mask1 = labels.eq(0)
        mask = 1 - mask1.float()

        # selective learning balance
        for i in range(0, self.AU_num):
            temp = labels[:,i]
            zero_num = torch.sum(temp.eq(0))
            pos_num = torch.sum(temp.eq(1))
            neg_num = torch.sum(temp.eq(-1))
            zero_num = zero_num.float()
            pos_num = pos_num.float()
            neg_num = neg_num.float()
            half_num = (N - zero_num)*self.balance_a[i]

            if (pos_num.item() <  half_num.item()):
                idx = torch.nonzero(temp.eq(-1))

                sample_num = int(neg_num.data - math.ceil(half_num.data))

                if sample_num < 1:
                    continue

                zero_idx = random.sample(idx, sample_num)
                for j in range(0, len(zero_idx)):
                    mask[int(zero_idx[j].data), i] = 0

                # postive under-representation
                if pos_num.data[0] != 0:
                    ratio = half_num/pos_num
                    if ratio.data[0] > self.boundary:
                        ratio = self.boundary

                    idx = torch.nonzero(temp.eq(1))
                    for j in range(0, len(idx)):
                        mask[int(idx[j].data), i] = ratio

        target = labels.gt(0)
        target = target.float()

        loss = F.binary_cross_entropy_with_logits(x, target, size_average = False)

        if self.size_average:
            loss = loss/N

        return loss<jupyter_output><empty_output><jupyter_text>Loss multi view final<jupyter_code>class MultiView_all_loss(nn.Module):
    def __init__(self, AU_num = 12, AU_idx = [0,1,2,3,4,5,6,7,8,9,10,11], fusion_mode = 0,
                 use_web = 0, database = 0, lambda_co_regularization = 400, lambda_multi_view = 100):

        super(MultiView_all_loss, self).__init__()

        self.lossfunc = BCE_sigmoid_negtive_bias_all(size_average=True, AU_num = AU_num, AU_idx = AU_idx, database = database)

        self.BCE = nn.BCELoss()
        self.sigmoid = nn.Sigmoid()
        self.log_sigmoid = nn.LogSigmoid()

        self.AU_num = AU_num
        self.AU_idx = AU_idx
        self.lambda_co_regularization = lambda_co_regularization
        self.lambda_multi_view = lambda_multi_view
        self.eps = 0.001

        self.fusion_mode = fusion_mode
        self.use_web = use_web

    def forward(self, gt, pre, pre1, pre2, weight1, bias1, weight2, bias2, feat1, feat2, flag):
        # flag is used for denoting whether the image is labeled 
        # flag == 1 means the image is labeled
        N = gt.size(0)

        mask = flag.eq(1)
        mask = mask.cuda()

        pre_label1 = torch.masked_select(pre1, mask) 
        pre_label1 = pre_label1.view(-1, self.AU_num)

        pre_label2 = torch.masked_select(pre2, mask)
        pre_label2 = pre_label2.view(-1, self.AU_num)

        pre_label = torch.masked_select(pre, mask)
        pre_label = pre_label.view(-1, self.AU_num)

        gt = torch.masked_select(gt, mask)
        gt = gt.view(-1, self.AU_num)

        if bool(gt.numel()):
            loss_pred = self.lossfunc(pre_label, gt)
            loss_pred1 = self.lossfunc(pre_label1, gt)
            loss_pred2 = self.lossfunc(pre_label2, gt)
        else:
            loss_pred = Variable(torch.FloatTensor([0])).cuda()
            loss_pred1 = Variable(torch.FloatTensor([0])).cuda()
            loss_pred2 = Variable(torch.FloatTensor([0])).cuda()

        if self.fusion_mode == 0:
            loss_BCE = (loss_pred1 + loss_pred2)/2
        else:
            loss_BCE = loss_pred + (loss_pred1 + loss_pred2)/2

        ############### loss multi-view ###############
        loss_multi_view = torch.FloatTensor([0])
        loss_multi_view = loss_multi_view.cuda()

        bias1 = bias1.view(self.AU_num, -1)
        feat1 = torch.cat((weight1, bias1), 1) 
        bias2 = bias2.view(self.AU_num, -1)
        feat2 = torch.cat((weight2, bias2), 1) 

        tmp = torch.norm(feat1, 2, 1)
        feat_norm1 = feat1 / tmp.view(self.AU_num, -1)
        tmp = torch.norm(feat2, 2, 1)
        feat_norm2 = feat2 / tmp.view(self.AU_num, -1)

        x = feat_norm1 * feat_norm2 
        x = torch.sum(x, 1)
        loss_weight_orth = torch.mean(torch.abs(x))
        loss_multi_view = loss_multi_view + loss_weight_orth

        loss_multi_view = loss_multi_view*self.lambda_multi_view
        ############ end loss multi-view ############

        ############### J-S divergence ###############
        loss_similar = torch.FloatTensor([0])
        loss_similar = loss_similar.cuda()

        # use_web: 0 only test images/ 1 training with unlabeled images
        if self.use_web != 0:
            p1 = self.sigmoid(pre1)
            log_p1 = self.log_sigmoid(pre1)
            p2 = self.sigmoid(pre2)
            log_p2 = self.log_sigmoid(pre2)
            p = (p1+p2)/2

            mask_idx = torch.ge(p1, -1)
            idx1 = torch.le(p1, 1 - self.eps) 
            idx2 = torch.ge(p1, self.eps)
            idx = idx1&idx2&mask_idx
            tmp_p1 = 1-p1[idx] + self.eps
            Hp1 = torch.mean(-(p1[idx]*log_p1[idx] + tmp_p1*torch.log(tmp_p1)))

            idx1 = torch.le(p2, 1 - self.eps)
            idx2 = torch.ge(p2, self.eps)
            idx = idx1&idx2&mask_idx
            tmp_p2 = 1-p2[idx] + self.eps
            Hp2 = torch.mean(-(p2[idx]*log_p2[idx] + tmp_p2*torch.log(tmp_p2)))

            idx1 = torch.le(p, 1 - self.eps)
            idx2 = torch.ge(p, self.eps)
            idx = idx1&idx2&mask_idx
            tmp_p11 = p[idx] + self.eps
            tmp_p22 = 1-p[idx] + self.eps
            H1 = torch.mean(-(tmp_p11*torch.log(tmp_p11) + (tmp_p22)*torch.log(tmp_p22)))

            H2 = (Hp1 + Hp2)/2

            loss_web = torch.abs(H1 - H2)
            loss_similar = loss_web

        loss_similar = loss_similar * self.lambda_co_regularization # Lcr
        ################# end J-S divergence #################
        # print("loss_BCE %4.2f, loss_multi_view %4.2f, loss_similar %4.2f" % (loss_BCE, loss_multi_view, loss_similar))
        loss = loss_BCE + loss_multi_view + loss_similar

        return loss, loss_pred, loss_pred1, loss_pred2, loss_multi_view, loss_similar<jupyter_output><empty_output><jupyter_text>Net  ResNet<jupyter_code>__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101',
           'resnet152']

model_urls = {
    'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth',
    'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth',
    'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth',
    'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth',
    'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth',
} # download pre-trained network

def conv3x3(in_planes, out_planes, stride=1):
    """3x3 convolution with padding"""
    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=1, bias=False)

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(BasicBlock, self).__init__()
        self.conv1 = conv3x3(inplanes, planes, stride)
        self.bn1 = nn.BatchNorm2d(planes) 
        self.relu = nn.ReLU(inplace=True)
        self.conv2 = conv3x3(planes, planes)
        self.bn2 = nn.BatchNorm2d(planes)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class Bottleneck(nn.Module): 
    expansion = 4

    def __init__(self, inplanes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
                               padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes * 4)
        self.relu = nn.ReLU(inplace=True)
        self.downsample = downsample
        self.stride = stride

    def forward(self, x):
        residual = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            residual = self.downsample(x)

        out += residual
        out = self.relu(out)

        return out

class ResNet(nn.Module):

    def __init__(self, block, layers, num_classes=1000, ave_size=7, num_output = 1):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
                               bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)
        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)

        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)

        self.avgpool = nn.AvgPool2d(ave_size, stride=1)
        self.fc = nn.Linear(512 * block.expansion, num_classes)
        self.num_output = num_output
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()

    def _make_layer(self, block, planes, blocks, stride=1):
        downsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample = nn.Sequential(
                nn.Conv2d(self.inplanes, planes * block.expansion,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(planes * block.expansion),
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return nn.Sequential(*layers)

    def forward(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        conv1 = self.maxpool(x)

        conv2 = self.layer1(conv1)
        conv3 = self.layer2(conv2)  # B*128*28*28
        conv4 = self.layer3(conv3)  # B*256*14*14
        conv5 = self.layer4(conv4)  # B*512*7*7

        x = self.avgpool(conv5)
        feat = x.view(x.size(0), -1)
        x = self.fc(feat)

        if self.num_output == 1:
            return  x, conv1
        elif self.num_output == 2:
            return x, conv2
        elif self.num_output == 3:
            return x, conv3
        elif self.num_output == 4:
            return x, conv4
        elif self.num_output == 5:
            return x, conv5
        elif self.num_output == 6:
            return x, feat
        elif self.num_output == 36:
            return x, conv3, feat
        else:
            return x

def resnet18(pretrained=False, **kwargs):
    """Constructs a ResNet-18 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet18']))
    return model

def resnet34(pretrained=False, **kwargs):
    """Constructs a ResNet-34 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet34']))
    return model

def resnet50(**kwargs):
    """Constructs a ResNet-50 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
    return model

def resnet101(pretrained=False, **kwargs):
    """Constructs a ResNet-101 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet101']))
    return model

def resnet152(pretrained=False, **kwargs):
    """Constructs a ResNet-152 model.
    Args:
        pretrained (bool): If True, returns a model pre-trained on ImageNet
    """
    model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)
    if pretrained:
        model.load_state_dict(model_zoo.load_url(model_urls['resnet152']))
    return model<jupyter_output><empty_output><jupyter_text>ResNet GCN<jupyter_code>def gen_A_cov(AU_num, AU_idx, database):
    self_AU_num = AU_num
    self_AU_idx = AU_idx

    path = '/content/drive/My Drive/dl_project/net/'

    if database == 0:
        path = path + 'relation_EmotioNet.mat'
        # parameters Wij of AU classifiers for both views
        # used as nodes of GCN
        # first pre-train the feature generators and classifiers
        # use the weights of the pre-trained classifiers as the initial input of GCN
        self_relation_all = sio.loadmat(path)['relation']
    elif database == 1:
        path = path + 'relation_BP4D.mat'
        self_relation_all = sio.loadmat(path)['relation']

    self_AU_relation = torch.zeros(self_AU_num, self_AU_num)
    for i in range(0, AU_num):
        for j in range(0, AU_num):
            self_AU_relation[i, j] = self_relation_all[self_AU_idx[i], self_AU_idx[j]]

    _adj = torch.abs(self_AU_relation)

    return _adj

def gen_adj(A):  # get adjacency matrix
    D = torch.pow(A.sum(1).float(), -0.5)
    D = torch.diag(D)
    adj = torch.matmul(torch.matmul(A, D).t(), D)
    return adj

class GraphConvolution(nn.Module):

    def __init__(self, in_features, out_features, bias=False):
        super(GraphConvolution, self).__init__()
        self.in_features = in_features
        self.out_features = out_features
        self.weight = Parameter(torch.Tensor(in_features, out_features))
        if bias:
            self.bias = Parameter(torch.Tensor(1, 1, out_features))
        else:
            self.register_parameter('bias', None)
        self.reset_parameters()

    def reset_parameters(self):
        stdv = 1. / math.sqrt(self.weight.size(1))
        self.weight.data.uniform_(-stdv, stdv)
        if self.bias is not None:
            self.bias.data.uniform_(-stdv, stdv)

    def forward(self, input, adj):
        support = torch.matmul(input, self.weight)
        support = support.cuda()
        adj = adj.cuda()
        output = torch.matmul(adj, support)

        if self.bias is not None:
            return output + self.bias
        else:
            return output

    def __repr__(self):
        return self.__class__.__name__ + ' (' \
               + str(self.in_features) + ' -> ' \
               + str(self.out_features) + ')'

class GCN_layer(nn.Module):
    def __init__(self, num_classes = 12, AU_idx = [0,1,2,3,4,5,6,7,8,9,10,11], in_channel = 513, database = 0, weight_relation = 6):
        super(GCN_layer, self).__init__()

        self.num_classes = num_classes
        self.AU_idx = AU_idx
        self.database = database
        self.weight_relation = weight_relation

        self.gc1 = GraphConvolution(in_channel, 513)
        self.gc2 = GraphConvolution(513, 513)
        self.relu = nn.LeakyReLU(0.2)

        _adj = gen_A_cov(num_classes, AU_idx, database)

        self.A = _adj

    def forward(self, x):

        x = x.t() # [12,513]
        adj = gen_adj(self.A).detach()
        x = self.gc1(x, adj)
        x = self.relu(x)
        x = self.gc2(x, adj)

        x = x.transpose(0, 1)
        return x

    def get_config_optim(self, lr, lrp):
        return [
                {'params': self.features.parameters(), 'lr': lr * lrp},
                {'params': self.gc1.parameters(), 'lr': lr},
                {'params': self.gc2.parameters(), 'lr': lr},
                ]<jupyter_output><empty_output><jupyter_text>ResNet multi view<jupyter_code>class ResNet_GCN_two_views(nn.Module):
    def __init__(self, AU_num = 12, AU_idx = [0,1,2,3,4,5,6,7,8,9,10,11], output = 1, fusion_mode = 0, database = 0):
        super(ResNet_GCN_two_views, self).__init__()                

        self.net1 = resnet34(num_classes=AU_num, num_output=6)
        self.net2 = resnet34(num_classes=AU_num, num_output=6)

        self.AU_num = AU_num
        self.AU_idx = AU_idx
        self.fusion_mode = fusion_mode
        self.output = output
        self.scale = 1

        # Different methods to fuse the features of the two views.
        # For fair comparasion, we choose to not fuse the two features.
        ### fusion_mode: 0 no fusion / 1 concate / 2 mean / 3 concate two layer
        if self.fusion_mode == 0 or self.fusion_mode == 1:
            self.fc = nn.Linear(1024, AU_num) 
        elif self.fusion_mode == 2:
            self.fc = nn.Linear(512, AU_num)
        elif self.fusion_mode == 3:
            self.fc = nn.Sequential(
                nn.Linear(1024, 512),
                nn.ELU(),
                nn.Linear(512, AU_num)
            )

        self.Is_begin_weight = True
        self.begin_weight1 = None
        self.begin_weight2 = None
        self.relation = GCN_layer(num_classes = AU_num, AU_idx = self.AU_idx, in_channel = 513, database = database);
        # obtain AU relationship by GCN

    def forward(self, data):
        N = data.size(0) # data = [batch size, 3, 224, 224]

        output1, feat1 = self.net1(data)
        output2, feat2 = self.net2(data)

        weight1 = self.net1.fc.weight
        bias1 = self.net1.fc.bias
        weight2 = self.net2.fc.weight
        bias2 = self.net2.fc.bias

        bias1 = bias1.view(self.AU_num, -1)
        weight_norm1 = torch.cat((weight1, bias1), 1)
        bias2 = bias2.view(self.AU_num, -1)
        weight_norm2 = torch.cat((weight2, bias2), 1)

        feat_norm1 = feat1
        feat_norm2 = feat2

        # We need the pre-trained weights as the initialization of GCN. 
        # These values are used for controlling the initialization.
        if self.Is_begin_weight: 
            self.begin_weight1 = weight_norm1
            self.begin_weight2 = weight_norm2
            self.Is_begin_weight = False
        else:
            weight_norm1 = self.relation(self.begin_weight1.t())
            weight_norm1 = weight_norm1.t()
            weight_norm2 = self.relation(self.begin_weight2.t())
            weight_norm2 = weight_norm2.t()

        output1 = torch.mm(feat_norm1, torch.t(weight_norm1[:, 0:512])) + weight_norm1[:, 512]
        output1 = self.scale * output1 
        output2 = torch.mm(feat_norm2, torch.t(weight_norm2[:, 0:512])) + weight_norm2[:, 512]
        output2 = self.scale * output2 

        if self.fusion_mode == 0 or self.fusion_mode == 1:
            temp = torch.cat((feat1, feat2), 1)
            output = self.fc(temp)
        elif self.fusion_mode == 2:
            temp = (feat1 + feat2) / 2
            output = self.fc(temp)
        elif self.fusion_mode == 3:
            temp = torch.cat((feat1, feat2), 1)
            output = self.fc(temp)

        if self.output == 1:
            return weight1, bias1, weight2, bias2, feat1, feat2, output1, output2, output
        else: 
            return output1, output2, output<jupyter_output><empty_output><jupyter_text>Main<jupyter_code>batch_size_num = 64
epoch_num = 30 
learning_rate = 0.001 
weight_decay = 0
test_batch_size = 128
unlabel_num = 50000

########## parameters ############
### fusion_mode: 0 no fusion / 1 concate / 2 mean / 3 concate two layer
### database: 0 CK+ / 1 BP4D
### use_web: 0 only test images/ 1 training with unlabeled images
### lambda_co_regularization:
### AU_idx: the AU idxes you want to consider
fusion_mode = 2
database = 0
use_web = 1

AU_num = 12
AU_idx = [0,1,2,3,4,5,6,7,8,9,10,11]

lambda_co_regularization = 100 # 100
lambda_multi_view = 400

########## transforms #############
## transform for training
transform_train = transforms.Compose([
    transforms.Resize(240),
    transforms.RandomCrop(224),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize((0.5355, 0.4249, 0.3801), (0.2832, 0.2578, 0.2548)),
])

## transform for testing
transform_test = transforms.Compose([
    transforms.Resize(240),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize((0.5355, 0.4249, 0.3801), (0.2832, 0.2578, 0.2548)),
])

def transfrom_img_train(img):
    img = transform_train(img)

    return img

def transfrom_img_test(img):
    img = transform_test(img)

    return img

########## lossfunction ###########
# loss function combination all losses, including the L_mv, L_cr and the BCE loss for AU classification
# Input: AU_num, AU_idx, fusion_mode, use_web and database are explained in the parameters section

# Explaination of the BCE loss: We choose the modified Selective Learning BCE loss for supervision.
# Since the positive/negative ratio is very small for some AUs, we only consider the positive under-representation situation
# and set a boundary for all AUs.
lossfunc = MultiView_all_loss(AU_num = AU_num, AU_idx = AU_idx, fusion_mode = fusion_mode,
                use_web = use_web, database = database,
                lambda_co_regularization = lambda_co_regularization, lambda_multi_view = lambda_multi_view)
trainset = ImageFolder("/content/drive/My Drive/dl_project/ck_dataset/train", transform_train)
testset = ImageFolder("/content/drive/My Drive/dl_project/ck_dataset/test", transform_test)

trainloader = DataLoader(trainset, batch_size=batch_size_num, shuffle=True)
testloader = DataLoader(testset, batch_size=test_batch_size, shuffle=True)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>net = ResNet_GCN_two_views(AU_num=AU_num, AU_idx=AU_idx, output=1, fusion_mode=fusion_mode, database=database)
net.cuda()

optimizer = optim.Adam(net.parameters(), lr=0.001) # lr=0.001
loss_count = torch.zeros(epoch_num)
test_count = torch.zeros(epoch_num)
for epoch in range(epoch_num):
  sum_loss = 0.0
  for i, data in enumerate(trainloader):
    img, labels = data
    img = img.cuda() # [batch_size, 3, 224, 224]
    ground_truth = torch.zeros((len(labels), AU_num)) # [batch_size,12]
    for i in range(len(labels)):
      ground_truth[i,labels[i]] = 1 # 1 represents the occurrency of the AU

    ground_truth = ground_truth.cuda() # [batch_size,12]

    weight1, bias1, weight2, bias2, feat1, feat2, output1, output2, output = net(img)

    flag = torch.ones((len(labels), AU_num)) # [batch_size,12]
    sup_loss, loss_pred, loss_pred1, loss_pred2, loss_multi_view, loss_similar = lossfunc(ground_truth, output, output1, output2, weight1, bias1, weight2, bias2, feat1, feat2, flag)

    optimizer.zero_grad()
    loss = sup_loss
    loss.backward()
    optimizer.step()
    sum_loss+=loss.item()

  loss_count[epoch] = sum_loss
  print("Epoch %d, loss %4.2f" % (epoch, sum_loss))
torch.save(net.state_dict(), "/content/drive/My Drive/dl_project/model/net_epoch30_batch64.weights")<jupyter_output><empty_output><jupyter_text>test<jupyter_code>net = ResNet_GCN_two_views(AU_num=AU_num, AU_idx=AU_idx, output=2, fusion_mode=fusion_mode, database=database)
model_path = '/content/drive/My Drive/dl_project/model/net_epoch30_batch64.weights'
net.load_state_dict(torch.load(model_path))
net.cuda()
count = 0
f1_score_count = np.zeros(len(testloader))
with torch.no_grad():
  for i, data in enumerate(testloader):
    img, labels = data

    img = img.cuda()
    labels = labels.cuda()

    AU_view1, AU_view2, AU_fusion = net(img)
    # AU_view1 = torch.sigmoid(AU_view1)
    # AU_view2 = torch.sigmoid(AU_view2)
    AU_fusion = torch.sigmoid(AU_fusion)
    AU_pred = torch.argmax(AU_fusion,axis=1)
    count += torch.eq(AU_pred, labels).sum().item()

    #F1 score
    # p = precision_score(y_true, y_pred, average='binary')
    # r = recall_score(y_true, y_pred, average='binary')
    f1_score_count[i] = f1_score(labels.data.cpu().numpy(), AU_pred.data.cpu().numpy(), average='micro')

accuracy = count / len(testset) * 100
print("accuracy on test set:",accuracy)
print("average F1 score over 12 AUs", np.mean(f1_score_count))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12580511507024747, 'fraction_numerical': 0.036498549851218504, 'mean_word_length': 2.4674154368551653, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '462637', 'license': 'MIT', 'n_tokens_mistral': 9690, 'n_tokens_neox': 9059, 'n_words': 1911, 'path': 'multi_label_co_regularization.ipynb', 'repo_name': 'COMP6248-Reproducability-Challenge/Multi-Label-Co-regularization', 'size': '26549'}