<jupyter_start><jupyter_text>Step 1: Load LAD Dataset Option 1 Load Dataset from Scratch<jupyter_code>file_root = '/media/hxd/82231ee6-d2b3-4b78-b3b4-69033720d8a8/MyDatasets/LAD'
data_root = file_root + '/LAD_annotations/'
img_root = file_root + '/LAD_images/'
# load attributes list
attributes_list_path = data_root + 'attribute_list.txt'
fsplit = open(attributes_list_path, 'r', encoding='UTF-8')
lines_attribute = fsplit.readlines()
fsplit.close()
list_attribute = list()
list_attribute_value = list()
for each in lines_attribute:
    tokens = each.split(', ')
    list_attribute.append(tokens[0])
    list_attribute_value.append(tokens[1])
# load label list
label_list_path = data_root + 'label_list.txt'
fsplit = open(label_list_path, 'r', encoding='UTF-8')
lines_label = fsplit.readlines()
fsplit.close()
list_label = dict()
list_label_value = list()
for each in lines_label:
    tokens = each.split(', ')
    list_label[tokens[0]]=tokens[1]
    list_label_value.append(tokens[1])
from torchvision.transforms import Compose, Resize, CenterCrop, ToTensor, Normalize
from PIL import Image

preprocess = Compose([
    Resize((224, 224), interpolation=Image.BICUBIC),
    CenterCrop((224, 224)),
    ToTensor()
])

image_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073]).cuda()
image_std = torch.tensor([0.26862954, 0.26130258, 0.27577711]).cuda()
# load all the labels, attributes, images data from the LAD dataset
attributes_per_class_path = data_root + 'attributes.txt'
fattr = open(attributes_per_class_path, 'r', encoding='UTF-8')
lines_attr = fattr.readlines()
fattr.close()
images = list()
attr = list()
labels = list()
for each in lines_attr:
    tokens = each.split(', ')
    labels.append(list_label[tokens[0]])
    img_path = tokens[1]
    image = preprocess(Image.open(os.path.join(img_root, img_path)).convert("RGB"))
    images.append(image)
    attr_r = list(map(int, tokens[2].split()[1:-1]))
    attr.append([val for i,val in enumerate(list_attribute_value) if attr_r[i] == 1])
# Dump processed image and text to local
with open('../checkpoints/data_img_raw.pkl', 'wb') as file:
    pickle.dump(images, file)
with open('../checkpoints/data_txt_raw.pkl', 'wb') as file:
    pickle.dump({'label': labels, 'att': attr}, file)<jupyter_output><empty_output><jupyter_text>Option 2 Load LAD Dataset from Saved Files<jupyter_code>with open('../checkpoints/data_img_raw.pkl', 'rb') as file:
    images = pickle.load(file)
with open('../checkpoints/data_txt_raw.pkl', 'rb') as file:
    b = pickle.load(file)
    
labels = b['label']
attr = b['att']<jupyter_output><empty_output><jupyter_text>Step 2: Obtain the Image and Text Features Option 1 Load CLIP to obtain features<jupyter_code># normalize images
image_input = torch.tensor(np.stack(images)).cuda()
image_input -= image_mean[:, None, None]
image_input /= image_std[:, None, None]
# Convert labels to tokens
tokenizer_label = SimpleTokenizer()
text_tokens = [tokenizer_label.encode(desc) for desc in labels]

sot_token = tokenizer_label.encoder['<|startoftext|>']
eot_token = tokenizer_label.encoder['<|endoftext|>']

text_inputs_label = torch.zeros(len(text_tokens), model.context_length, dtype=torch.long)
for i, tokens in enumerate(text_tokens):
    tokens = [sot_token] + tokens + [eot_token]
    text_inputs_label[i, :len(tokens)] = torch.tensor(tokens)
text_inputs_label = text_inputs_label.cuda()
# Convert attributes to tokens
tokenizer_att = SimpleTokenizer()
text_tokens = [[tokenizer_att.encode(desc) for desc in att] for att in attr]

sot_token = tokenizer_att.encoder['<|startoftext|>']
eot_token = tokenizer_att.encoder['<|endoftext|>']
text_inputs_att = list()

for j, tokens_img in enumerate(text_tokens):
    text_input = torch.zeros(len(tokens_img), model.context_length, dtype=torch.long)
    for i, tokens in enumerate(tokens_img):
        tokens = [sot_token] + tokens + [eot_token]
        text_input[i, :len(tokens)] = torch.tensor(tokens)
    text_inputs_att.append(text_input.cuda())

# Load CLIP model
with torch.no_grad():
    image_features = model.encode_image(image_input).float()
with torch.no_grad():
    label_fea = model.encode_text(text_inputs_label.cuda()).float()
with torch.no_grad():
    text_feature = list()
    for txt in text_inputs_att:
        if len(txt) == 0:
            text_feature.append(torch.empty(0, 512).cuda())
        else:
            text_feature.append(model.encode_text(txt).float())
image_features /= image_features.norm(dim=-1, keepdim=True)

label_fea /= label_fea.norm(dim=-1, keepdim=True)

text_feature = torch.stack([torch.mean(item,0) for item in text_feature])
text_feature /= text_feature.norm(dim=-1, keepdim=True)
# Save image and text features
with open('../checkpoints/data_txt_feature.pkl', 'wb') as file:
    pickle.dump({'label': label_fea, 'att': text_feature}, file)
with open('../checkpoints/data_img_feature.pkl', 'wb') as file:
    pickle.dump(image_features, file)<jupyter_output><empty_output><jupyter_text>Option 2 Load saved image and text features<jupyter_code>with open('../checkpoints/data_txt_feature.pkl', 'rb') as file:
    b = pickle.load(file)

label_fea = b['label']
text_feature = b['att']
with open('../checkpoints/data_img_feature.pkl', 'rb') as file:
    image_features = pickle.load(file)<jupyter_output><empty_output><jupyter_text>Construct the dataloader for classification<jupyter_code>from torch.utils.data import Dataset
from sklearn import preprocessing

class Dataset(Dataset):

    def __init__(self, image_features, text_feature, labels, data_indx):
        self.image_features = image_features
        self.text_feature = text_feature
        self.labels = labels
        self.data_indx = data_indx
#         self.imgs = image_input
#         self.attr = attr

    def __len__(self):
        return len(self.image_features)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        sample = {'image': self.image_features[idx], 
                  'attribute': self.text_feature[idx], 
                  'label': self.labels[idx],
                  'data_indx': self.data_indx[idx]
#                   'imgs': self.imgs[idx],
#                   'attr': self.attr[idx]
                 }

        return sample
   

le = preprocessing.LabelEncoder()
le.fit(labels)
class_list = list(le.classes_)
labels_list = torch.tensor(le.transform(labels)).cuda()

attr_ = [';'.join(attr[0]) for item in attr]
data_indx = list(range(4600))
# dataset = Dataset(image_features, text_feature, labels_list, torch.tensor(np.stack(images)).cuda(), attr_)
dataset = Dataset(image_features, text_feature, labels_list, data_indx)
train_set, test_set = torch.utils.data.random_split(dataset,[4600-500,500])
trainloader = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)
testloader = torch.utils.data.DataLoader(test_set, batch_size=64, shuffle=True)
import torch.nn as nn
from torch.utils.data import DataLoader
# defining the model architecture
class Net(nn.Module):   
  def __init__(self):
      super(Net, self).__init__()

      self.linear_layers = nn.Sequential(
          nn.Linear(1024, 512),
          nn.Linear(512, 230)
      )

  # Defining the forward pass    
  def forward(self, x, t):
      con = torch.cat((x, t), 1)
      out = self.linear_layers(con)
      return out
model = Net().cuda()
error = nn.CrossEntropyLoss().cuda()
learning_rate = 0.01
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
scheduler = StepLR(optimizer, step_size=10, gamma=0.1)
num_epochs = 30
# Lists for visualization of loss and accuracy 
epoch_list = []
train_accuracy_list = []
train_loss_list = []
valid_accuracy_list = []
valid_loss_list = []
PATH = "../checkpoints/cnn.pth"

for epoch in range(num_epochs):
    correct = 0
    running_loss = 0
    model.train()
    for data in trainloader:
        # Transfering images and labels to GPU if available
#         image_batch, text_batch, label_batch, im_batch, att_batch = data['image'], data['attribute'], data['label'], data['imgs'], data['attr']
        image_batch, text_batch, label_batch, idx_batch = data['image'], data['attribute'], data['label'], data['data_indx']
        # Forward pass 
        outputs = model(image_batch, text_batch)
        #CrossEntropyLoss expects floating point inputs and long labels.
        loss = error(outputs, label_batch)
        # Initializing a gradient as 0 so there is no mixing of gradient among the batches
        optimizer.zero_grad()
        #Propagating the error backward
        loss.backward()
        # Optimizing the parameters
        optimizer.step()
    
        predictions = torch.max(outputs, 1)[1].cuda()
        correct += (predictions == label_batch).sum()
        running_loss += loss.item()

    train_loss_list.append(float(running_loss) / float(len(trainloader.dataset)))
    train_accuracy_list.append(float(correct) / float(len(trainloader.dataset)))
    
    # test on validation set
    correct = 0
    running_loss = 0
    with torch.no_grad():
        for data in testloader:
            image_batch, text_batch, label_batch, idx_batch = data['image'], data['attribute'], data['label'], data['data_indx']

            outputs = model(image_batch, text_batch)

            predictions = torch.max(outputs, 1)[1].cuda()
            correct += (predictions == label_batch).sum()
            running_loss += loss.item()


                               
    valid_loss_list.append(float(running_loss) / float(len(testloader.dataset)))
    valid_accuracy_list.append(float(correct) / float(len(testloader.dataset)))
                               
    print("Epoch: {}, train_loss: {}, train_accuracy: {}%, test_loss: {}, test_accuracy: {}%".format(epoch, 
                                                      train_loss_list[-1], 
                                                      train_accuracy_list[-1], 
                                                      valid_loss_list[-1], 
                                                      valid_accuracy_list[-1]))
            
    
                          
    epoch_list.append(epoch)     
    scheduler.step()
    
    if (epoch % 10) == 0:
        torch.save({
            'model_state_dict': model.state_dict(),
            'optimizer_state_dict': optimizer.state_dict()
        }, PATH)
m = nn.Softmax()
data = next(iter(testloader))
image_batch, text_batch, label_batch, idx_batch = data['image'], data['attribute'], data['label'], data['data_indx']
outputs = model(image_batch, text_batch)

for id in range(64):
    plt.imshow(images[idx_batch[id]].cpu().detach().permute(1, 2, 0))
    plt.show()
    print(m(outputs[id]).cpu().topk(3, dim=-1))
    top3 = m(outputs[id]).cpu().topk(3, dim=-1).indices
    print([class_list[i] for i in top3])
    print(attr[idx_batch[id]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13276705752008125, 'fraction_numerical': 0.017172929554057796, 'mean_word_length': 3.619189765458422, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '8406', 'license': 'MIT', 'n_tokens_mistral': 3643, 'n_tokens_neox': 3503, 'n_words': 734, 'path': 'LAD_CLIP.ipynb', 'repo_name': 'xiaodanhu/AttDiscovery', 'size': '10831'}