<jupyter_start><jupyter_text>Niu<jupyter_code>def label_to_levels(label, num_classes):
    levels = [1]*label + [0]*(num_classes - 1 - label)
    levels = torch.tensor(levels, dtype=torch.float32)
    return levels

levels = []

for label in y_train:
    levels_from_label = label_to_levels(label, num_classes=NUM_CLASSES)
    levels.append(levels_from_label)

levels = torch.stack(levels)
levels
# for simplicity, this is the ordinal network:

# def __init__()
niu = torch.nn.Linear(99, NUM_CLASSES-1)

# def forward(self, X_train):
logits = niu(X_train)
logits.shape
def loss_niu(logits, levels):
    val =  -torch.sum((torch.log(torch.sigmoid(logits))*levels + 
                       torch.log(1 - torch.sigmoid(logits))*(1-levels)),
                      dim=1)
    return torch.mean(val)



loss_niu(logits, levels)<jupyter_output><empty_output><jupyter_text>Conditional<jupyter_code>sets = []
for i in range(NUM_CLASSES-1):
    label_mask = y_train > i-1
    label_tensor = (y_train[label_mask] > i).to(torch.int64)
    sets.append((label_mask, label_tensor))
sets
# same as Niu et al.

# def __init__(self):
conditional_net = torch.nn.Linear(99, NUM_CLASSES-1)

# def forward(self, X_train):
logits = conditional_net(X_train)
logits.shape
def loss_conditional1(logits, sets):
    
    losses = []
    for task_index, s in enumerate(sets):
        train_examples = s[0]
        train_labels = s[1]
        pred = logits[train_examples, task_index]
        
        if len(s[1]) < 1:
            continue        
        
        loss = -torch.mean(torch.log(torch.sigmoid(pred))*train_labels + 
                           torch.log(1 - torch.sigmoid(pred))*(1-train_labels))
        
        losses.append(loss)
        
    return losses


losses = loss_conditional1(logits, sets)

for i, loss in enumerate(losses):
    print(f'task {i} loss: {loss}')
# More stable  implementation combining log and sigmoid

def loss_conditional2(logits, sets):
    
    losses = []
    for task_index, s in enumerate(sets):
        train_examples = s[0]
        train_labels = s[1]
        pred = logits[train_examples, task_index]
        
        if len(s[1]) < 1:
            continue
        
        loss = -torch.mean( F.logsigmoid(pred)*train_labels 
                           + (F.logsigmoid(pred) - pred)*(1-train_labels) )
        
        losses.append(loss)
        
    return losses


losses = loss_conditional2(logits, sets)

for i, loss in enumerate(losses):
    print(f'task {i} loss: {loss}')<jupyter_output>task 0 loss: 0.9681528806686401
task 1 loss: 0.8592420220375061
task 2 loss: 0.7534387707710266
task 3 loss: 0.579859733581543<jupyter_text>average over training examples:<jupyter_code>def loss_conditional_v2(logits, sets):

    num_examples = 0
    losses = 0.
    for task_index, s in enumerate(sets):
        train_examples = s[0]
        train_labels = s[1]

        if len(train_labels) < 1:
            continue

        num_examples += len(train_labels)
        pred = logits[train_examples, task_index]
        
        loss = -torch.sum( F.logsigmoid(pred)*train_labels
                                + (F.logsigmoid(pred) - pred)*(1-train_labels) )
        losses += loss
    return losses/num_examples


losses = loss_conditional_v2(logits, sets)

losses<jupyter_output><empty_output><jupyter_text>Conditional with branches<jupyter_code># Before:

conditional_net = torch.nn.Linear(99, NUM_CLASSES-1)
logits = conditional_net(X_train)
logits.shape
# def __init__():
branches = [torch.nn.Linear(99, 10) for i in range(NUM_CLASSES-1)]
output_layers = [torch.nn.Linear(10, 1) for i in range(NUM_CLASSES-1)]

# def forward(self, X_train):
logits = torch.empty(X_train.shape[0], len(output_layers))

for i in range(len(output_layers)):
    logits[:, i] = output_layers[i](torch.relu(branches[i](X_train))).squeeze()

logits.shape
losses = loss_conditional1(logits, sets)

for i, loss in enumerate(losses):
    print(f'task {i} loss: {loss}')
losses = loss_conditional2(logits, sets)

for i, loss in enumerate(losses):
    print(f'task {i} loss: {loss}')
# def __init__():
branches = [torch.nn.Linear(99, 10) for i in range(NUM_CLASSES-1)]
output_layers = [torch.nn.Linear(10, 1) for i in range(NUM_CLASSES-1)]

# def forward(self, X_train):
logits = []

for i in range(len(output_layers)):
    logits.append(output_layers[i](torch.relu(branches[i](X_train))).squeeze())

torch.stack(logits).T.shape<jupyter_output><empty_output><jupyter_text>Usage in Forward Pass<jupyter_code># same as Niu et al.

# def __init__(self):
conditional_net = torch.nn.Linear(99, NUM_CLASSES-1)

# def forward(self, X_train):
logits = conditional_net(X_train)
logits
logits.shape<jupyter_output><empty_output><jupyter_text>Conditional v1 with `cumprod`<jupyter_code>probas = torch.sigmoid(logits)
probas = torch.cumprod(probas, dim=1)

probas
def proba_to_label(probas):
    """
    Converts predicted probabilities from extended binary format
    to integer class labels

    Parameters
    ----------
    probas : torch.tensor, shape(n_examples, n_labels)
        Torch tensor consisting of probabilities returned by CORAL model.

    Examples
    ----------
    >>> # 3 training examples, 6 classes
    >>> probas = torch.tensor([[0.934, 0.861, 0.323, 0.492, 0.295],
    ...                        [0.496, 0.485, 0.267, 0.124, 0.058],
    ...                        [0.985, 0.967, 0.920, 0.819, 0.506]])
    >>> proba_to_label(probas)
    tensor([2, 0, 5])
    """
    predict_levels = probas > 0.5
    predicted_labels = torch.sum(predict_levels, dim=1)
    return predicted_labels

predicted_labels = proba_to_label(probas).float()
predicted_labels<jupyter_output><empty_output><jupyter_text>Conditional v2 like v1 but without `cumprod`<jupyter_code>probas = torch.sigmoid(logits)
probas
predicted_labels = proba_to_label(probas).float()
predicted_labels<jupyter_output><empty_output><jupyter_text>Conditional v2-argmax (Debug) calculate the probs of (y=0, y=1, y=2,…) using (1-P(y>0), P(y>0)(1-P(y>1|y>0), P(y>0)P(y>1|y>0)(1-P(y>2|y>1),…), and then using argmax to find the predicted label.<jupyter_code>probas = torch.sigmoid(logits)
probas
ones = torch.ones((probas.shape[0], 1))#.to(device)
comp_1 = torch.cat((ones, torch.cumprod(probas, dim=1)), dim=1)
comp_2 = torch.cat((1-probas, ones), dim=1)
probas_y = torch.mul(comp_1, comp_2)
probas_y

predicted_labels = torch.argmax(probas_y, dim=1)
predicted_labels<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1322800495662949, 'fraction_numerical': 0.036864931846344484, 'mean_word_length': 3.5568101623147497, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '442738', 'license': 'MIT', 'n_tokens_mistral': 2547, 'n_tokens_neox': 2359, 'n_words': 461, 'path': 'notebooks/03_conditional-method-code-example.ipynb', 'repo_name': 'Raschka-research-group/corn-ordinal-neuralnet', 'size': '6456'}