<jupyter_start><jupyter_text>Load Dataset<jupyter_code>task_name = 'sst-2'
data_dir = '/home/mila/c/caomeng/scratch/glue_data/{}'.format(task_name.upper())
processor = processors[task_name]()
output_mode = output_modes[task_name]
label_list = processor.get_labels()
num_labels = len(label_list)
examples = []
train_examples = processor.get_train_examples(data_dir)
dev_examples = processor.get_dev_examples(data_dir)
print('- train examples: {}'.format(len(train_examples)))
print('- dev examples: {}'.format(len(dev_examples)))
examples.extend(train_examples)
examples.extend(dev_examples)

if task_name == 'mnli':
    processor = processors['mnli-mm']()
    dev_mismatched_examples = processor.get_dev_examples(data_dir)
    print('- dev mismatched examples: {}'.format(len(dev_mismatched_examples)))
    examples.extend(dev_mismatched_examples)
len(examples)<jupyter_output><empty_output><jupyter_text>Convert Examples to Features<jupyter_code>teacher_model_path = '/home/mila/c/caomeng/scratch/huggingface/bert-base-uncased'
train_guids = []

for e in examples:
    train_guids.append(e.guid)
tokenizer = BertTokenizer.from_pretrained(teacher_model_path, local_files_only=True)
max_seq_length = 128
train_features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer, output_mode)
print(len(train_features))
print(train_features[0])
train_data, all_label_ids = get_tensor_data(output_mode, train_features)
train_data.tensors[0].shape
print(all_label_ids.shape)
print(all_label_ids[:10])
train_dataloader = DataLoader(train_data, sampler=SequentialSampler(train_data), batch_size=32)<jupyter_output><empty_output><jupyter_text>Load Teacher Model<jupyter_code>teacher_model = BertModel.from_pretrained(teacher_model_path, local_files_only=True)
teacher_model.to('cuda')
print('model loaded.')
cls_features = []
example_ids = []

for batch_ in tqdm(train_dataloader, desc="Evaluating"):
    batch_ = tuple(t.to('cuda') for t in batch_)
    with torch.no_grad():
        input_ids, input_mask, segment_ids, label_ids, seq_lengths = batch_
        pooled = teacher_model(input_ids, segment_ids, input_mask, return_dict=True).pooler_output

        cls_features.append(pooled)
        example_ids.append(input_ids)
cls_feature_vectors = torch.cat(cls_features, dim=0)
cls_feature_vectors.shape<jupyter_output><empty_output><jupyter_text>K-means<jupyter_code>%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs
cls_feature_numpy = cls_feature_vectors.cpu().numpy()
all_label_ids = all_label_ids.cpu().numpy()
assert cls_feature_numpy.shape[0] == all_label_ids.shape[0]
CLUSTER_NUMBER = 4
kmeans = KMeans(n_clusters=CLUSTER_NUMBER, random_state=0).fit(cls_feature_numpy)
cluster_centers = kmeans.cluster_centers_
cluster_labels = kmeans.labels_
print(cluster_centers.shape)
print(cluster_labels.shape)
cluster_labels[:10]<jupyter_output><empty_output><jupyter_text>Save Clustering IDs<jupyter_code>assert len(train_guids) == cluster_labels.shape[0]
cluster_map = {}

for guid, l in zip(train_guids, cluster_labels):
    cluster_map[guid] = int(l)
# import json

# with open('clusters/cluster_sst2_k{}.json'.format(CLUSTER_NUMBER), 'w') as fp:
#     json.dump(cluster_map, fp)<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code>import random

from sklearn.decomposition import PCA
def get_cmap(n, name='hsv'):
    '''Returns a function that maps each index in 0, 1, ..., n-1 to a distinct 
    RGB color; the keyword argument name must be a standard mpl colormap name.'''
    return plt.cm.get_cmap(name, n)
vis_data = cls_feature_numpy[:10000, :]
vis_labels = cluster_labels[:10000]
pca = PCA(n_components=2)
pca_results = pca.fit_transform(vis_data)
pca_results.shape
plt.figure(1)

cmap = get_cmap(CLUSTER_NUMBER)

for k in range(CLUSTER_NUMBER):
    cluster_data = vis_labels == k
    plt.scatter(pca_results[cluster_data, 0], pca_results[cluster_data, 1],
                color=cmap(k), marker='.', s=10)

# plt.scatter(cluster_centers[:, 0], cluster_centers[:, 1], c='black', s=50)
plt.title("K-Means (K={})".format(CLUSTER_NUMBER))
plt.xticks([])
plt.yticks([])
plt.savefig('kmeans.png')
plt.show()
vis_labels = all_label_ids[:10000]
plt.figure(1)

cmap = ['#4EACC5', '#FF9C34', '#4E9A06']

for k in range(3):
    cluster_data = vis_labels == k
    plt.scatter(pca_results[cluster_data, 0], pca_results[cluster_data, 1],
                color=cmap[k], marker='.', s=10)

# plt.scatter(cluster_centers[:, 0], cluster_centers[:, 1], c='black', s=50)
plt.title("K-Means (True Label)")
plt.xticks([])
plt.yticks([])
plt.savefig('ture_label.png')
plt.show()
# merge json
# train_json = json.load(open('clusters/cluster_mnli_k3_train.json'))
# dev_m_json = json.load(open('clusters/cluster_mnli_k3_dev_matched.json'))
# dev_mm_json = json.load(open('clusters/cluster_mnli_k3_dev_mismatched.json'))

# print(len(train_json))
# print(len(dev_m_json))
# print(len(dev_mm_json))
# with open('clusters/cluster_mnli_k{}.json'.format(CLUSTER_NUMBER), 'w') as fp:
#     json.dump({**train_json, **dev_m_json, **dev_mm_json}, fp)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13461909353905496, 'fraction_numerical': 0.013307618129218901, 'mean_word_length': 7.141287284144427, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '545597', 'license': 'Apache-2.0', 'n_tokens_mistral': 1996, 'n_tokens_neox': 1916, 'n_words': 303, 'path': 'cluster.ipynb', 'repo_name': 'mcao516/SSKD-TinyBERT', 'size': '5185'}