def gradient_normalizers(grads, losses, normalization_type):
    gn = {}
    if (normalization_type == 'l2'):
        for t in grads:
            gn[t] = torch.sqrt(torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum())
    elif (normalization_type == 'loss'):
        for t in grads:
            gn[t] = min(losses[t].mean(), 10.0)
    elif (normalization_type == 'loss+'):
        for t in grads:
            gn[t] = min((losses[t].mean() * torch.sqrt(torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum())), 10)
    elif ((normalization_type == 'none') or (normalization_type == 'eq')):
        for t in grads:
            gn[t] = 1.0
    else:
        raise ValueError('ERROR: Invalid Normalization Type')
    return gn
def gradient_normalizers(grads, losses, normalization_type):
    gn = {}
    if (normalization_type == 'l2'):
        for t in grads:
            gn[t] = torch.sqrt(torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum())
    elif (normalization_type == 'loss'):
        for t in grads:
            gn[t] = min(losses[t].mean(), 10.0)
    elif (normalization_type == 'loss+'):
        for t in grads:
            gn[t] = min((losses[t].mean() * torch.sqrt(torch.stack([gr.pow(2).sum().data for gr in grads[t]]).sum())), 10)
    elif ((normalization_type == 'none') or (normalization_type == 'eq')):
        for t in grads:
            gn[t] = 1.0
    else:
        raise ValueError('ERROR: Invalid Normalization Type')
    return gn