<jupyter_start><jupyter_text>use cpu because the follow computation need a lot of memory<jupyter_code>device = 'cpu'
train_features, train_labels = train_features.to(device), train_labels.to(device)
num_train_data = train_labels.shape[0]
num_class = torch.max(train_labels) + 1

torch.manual_seed(args.rng_seed)
torch.cuda.manual_seed_all(args.rng_seed)
perm = torch.randperm(num_train_data).to(device)
print(perm)<jupyter_output>tensor([36044, 49165, 37807,  ..., 42128, 15898, 31476])<jupyter_text>constrained normalized cut<jupyter_code>K = 100
K = 20
start = time.time()
def make_column_normalize(X):
    return X.div(torch.norm(X, p=2, dim=0, keepdim=True))

cosin_similarity = torch.mm(train_features, train_features.t())
dist = (1 - cosin_similarity) / 2
n = dist.size(0)

dist_sorted, idx = dist.topk(K, dim=1, largest=False, sorted=True)
k_dist = dist_sorted[:, -1:]

similarity_sorted = torch.exp(-dist_sorted * 2 / 0.2)

# build the sparse similarity matrix
from scipy import sparse
col = idx.numpy().flatten()
row = np.tile(np.linspace(0,n-1,n,dtype=np.int64).reshape(-1,1),(1,K)).flatten()
data = similarity_sorted.numpy().flatten()
similarity_sparse = sparse.bsr_matrix((data, (row, col)), shape=(n,n))

similarity_sparse = (similarity_sparse + similarity_sparse.T) / 2.
print('similarity done')

degree = torch.from_numpy(np.sum(similarity_sparse, axis=0)).float()
#degree_normed = 1. / np.sqrt(degree)
#L_sys = np.diag(degree_normed) * (np.diag(degree) - similarity_sparse) * np.diag(degree_normed)
from scipy.sparse import csgraph
L_sys = csgraph.laplacian(similarity_sparse, normed=True)
print('L_sys done')
print(time.time() - start)
num_eigenvectors = 200 # the number of precomputed spectral eigenvectors.

from scipy import sparse
L_sparse = L_sys
start = time.time()
eigenvalues, eigenvectors = linalg.eigsh(L_sparse, k=num_eigenvectors, which='SM', tol=1e-2, maxiter=30000)
eigenvalues = eigenvalues.astype(np.float32)
eigenvectors = eigenvectors.astype(np.float32)
eigenvalues, eigenvectors = torch.from_numpy(eigenvalues.real)[1:], torch.from_numpy(eigenvectors.real)[:, 1:]
eigenvalues, idx = eigenvalues.sort()
eigenvectors = eigenvectors[:, idx]
print('eigenvectors done')
print(eigenvalues)
print(time.time() - start)
fig = plt.figure(dpi=200)

for num_labeled_data in [50, 100, 250, 500, 1000, 2000, 4000, 8000]:
    # index of labeled and unlabeled
    # even split
    index_labeled = []
    index_unlabeled = []
    data_per_class = num_labeled_data // args.num_class
    for c in range(10):
        indexes_c = perm[train_labels[perm] == c]
        index_labeled.append(indexes_c[:data_per_class])
        index_unlabeled.append(indexes_c[data_per_class:])
    index_labeled = torch.cat(index_labeled)
    index_unlabeled = torch.cat(index_unlabeled)

#     index_labeled = perm[:num_labeled_data]
#     index_unlabeled = perm[num_labeled_data:]
    
    # prior
    unary_prior = torch.zeros([num_train_data, num_class]).float()
    unary_prior[index_labeled, :] = -1
    unary_prior[index_labeled, train_labels[index_labeled]] = 1
    AQ = unary_prior.abs()
    pd = degree.view(-1, 1) * (AQ + unary_prior) / 2
    nd = degree.view(-1, 1) * (AQ - unary_prior) / 2
    np_ratio = pd.sum(dim=0) / nd.sum(dim=0)
    unary_prior_norm = (pd / np_ratio).sqrt() - (nd * np_ratio).sqrt()
    unary_prior_norm = make_column_normalize(unary_prior_norm)
    
    # logits and prediction
    alpha = 0
    lambda_reverse = (1 / (eigenvalues - alpha)).view(1, -1)
    logits = torch.mm(lambda_reverse * eigenvectors, torch.mm(eigenvectors.t(), unary_prior_norm))
    logits = make_column_normalize(logits) * math.sqrt(logits.shape[0])   
    logits = logits - logits.max(1, keepdim=True)[0]
    _, predict = logits.max(dim=1)
    
    temperature_nc = 1
    
    # pseudo weights
    logits_sorted = logits.sort(dim=1, descending=True)[0]
    subtract = logits_sorted[:, 0] - logits_sorted[:, 1]
    pseudo_weights = 1 - torch.exp(- subtract / temperature_nc)

    exp = (logits * temperature_nc).exp()
    probs = exp / exp.sum(1, keepdim=True)
    probs_sorted, predict_all = probs.sort(1, True)
    assert torch.all(predict == predict_all[:, 0])

    idx = pseudo_weights[index_unlabeled].sort(dim=0, descending=True)[1]
    pseudo_indexes = index_unlabeled[idx]
    pseudo_labels = predict[index_unlabeled][idx]
    pseudo_probs = probs[index_unlabeled][idx]
    pseudo_weights = pseudo_weights[index_unlabeled][idx]
    assert torch.all(pseudo_labels == pseudo_probs.max(1)[1])

    save_dict = {
        'pseudo_indexes': pseudo_indexes,
        'pseudo_labels': pseudo_labels,
        'pseudo_probs': pseudo_probs,
        'pseudo_weights': pseudo_weights,
        'labeled_indexes': index_labeled,
        'unlabeled_indexes': index_unlabeled,
    }
    torch.save(save_dict, os.path.join(args.save_path, f'{num_labeled_data}.pth.tar'))

    # for plot
    correct = pseudo_labels == train_labels[pseudo_indexes]

    entropy = - (pseudo_probs * torch.log(pseudo_probs + 1e-7)).sum(dim=1)
    confidence = (- entropy * 1).exp()
    confidence /= confidence.max()

    arange = 1 + np.arange(confidence.shape[0])
    xs = arange / confidence.shape[0]
    correct_tmp = correct[confidence.sort(descending=True)[1]]
    accuracies = np.cumsum(correct_tmp.numpy()) / arange
    plt.plot(xs, accuracies, label='num_labeled_data={}'.format(num_labeled_data))

    acc = correct.float().mean()

    print('num_labeled={:4} T_nc={}, prec={:.2f}, AUC={:.2f}'.format(
        num_labeled_data, temperature_nc, acc * 100, accuracies.mean() * 100))
        
plt.xlabel('accumulated unlabeled data ratio')
plt.ylabel('unlabeled top1 accuracy')
plt.xticks(np.arange(0, 1.01, 0.1))
plt.grid()
plt.title('num_eigenvectors={}'.format(num_eigenvectors))
legend = plt.legend(loc='upper left', bbox_to_anchor=(1, 1))
plt.show()<jupyter_output>tensor([24681, 42151, 48978, 41040, 36909,  8628, 24936, 35926, 15934,  8801,
        36293, 28026,  3814, 34981, 21135, 16904, 20152,  3486, 11894, 29780,
        23932, 33744, 41766, 42979, 49518, 11341,  6091, 48161, 36335, 29858,
        36044, 25569, 46340,  8832, 38677, 37807,  4480, 18517,  8409, 15769,
        49165, 43846, 13449, 34615, 38862, 17849, 49031,  4115, 29909, 34515])
num_labeled=  50 T_nc=1, prec=58.29, AUC=75.12
tensor([24681, 42151, 48978, 41040, 36909, 10197, 38770, 40649, 43279, 27934,
         8628, 24936, 35926, 15934,  8801, 27814, 37390, 12841, 22270, 10107,
        36293, 28026,  3814, 34981, 21135, 46451, 48404,  8366, 32477, 48870,
        16904, 20152,  3486, 11894, 29780, 47462, 34297,  1271, 28738, 12147,
        23932, 33744, 41766, 42979, 49518, 18912, 32018, 17323,  1974, 27259,
        11341,  6091, 48161, 36335, 29858, 17315, 22879, 43307, 23185, 10318,
        36044, 25569, 46340,  8832, 38677, 36881, 11751,  9278,  1836,  8557,
        37807,  [...]<jupyter_text>matrix visulization<jupyter_code>num_to_show = 2000
index_to_show = torch.randperm(50000)[:num_to_show]
# dist_sorted, idx = dist.topk(K, dim=1, largest=False, sorted=True)
# index_to_show = torch.Tensor(list(set(idx[:, 1].numpy()))).long()[:num_to_show]

index_to_show = torch.Tensor(sorted(index_to_show.numpy(), key=lambda index: train_labels[index].item())).long()

tmp = cosin_similarity[index_to_show, :][:, index_to_show].clone()
tmp[torch.arange(num_to_show), torch.arange(num_to_show)] = 0
plt.matshow(tmp)
plt.axis('off')
# plt.savefig('cosine_similarity.pdf')
plt.show()
tmp1 = eigenvectors * torch.sqrt(lambda_reverse)
eigenvectors_normed = tmp1 / torch.norm(tmp1, dim=1).reshape(-1, 1)
plt.matshow((torch.mm(eigenvectors_normed, eigenvectors_normed.t()))[index_to_show, :][:, index_to_show])
plt.axis('off')
# plt.savefig('nc.pdf')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14499291145766208, 'fraction_numerical': 0.09988400567083387, 'mean_word_length': 4.619116582186821, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '98548', 'license': 'MIT', 'n_tokens_mistral': 3592, 'n_tokens_neox': 2934, 'n_words': 545, 'path': 'notebooks/nc.ipynb', 'repo_name': 'Bhaskers-Blu-Org2/metric-transfer.pytorch', 'size': '7759'}