<jupyter_start><jupyter_text>Image Path training :" ../input/rakuten-france-multimodal-product-classification/images/images/image_test" test: "../input/rakuten-france-multimodal-product-classification/images/images/image_train"<jupyter_code>x_train = pd.read_csv("/content/X_train_update.csv")
y_train = pd.read_csv("/content/Y_train_CVw08PX.csv")
x_test = pd.read_csv("/content/X_test_update.csv")<jupyter_output><empty_output><jupyter_text><jupyter_code>x_train.columns
x_train.head()
x_train.rename(columns={"Unnamed: 0": "Id"}, inplace=True)
y_train.rename(columns={"Unnamed: 0": "Id"}, inplace=True)
x_test.rename(columns={"Unnamed: 0": "Id"}, inplace=True)
x_train.describe()
y_train.describe()
h=y_train.prdtypecode.value_counts()
len(h)
import seaborn as sns
sns.countplot(y_train.prdtypecode)
len(x_train)
x_train.sample(5)
y_train.sample(5)
print(x_test.sample(5))<jupyter_output>Id  ...     imageid
13436  98352  ...  1256824315
11674  96590  ...  1310548748
12422  97338  ...  1027825514
7280   92196  ...  1273174792
1756   86672  ...   971432380

[5 rows x 5 columns]<jupyter_text>classifcation multinomiale non balance<jupyter_code>!pip install sentencepiece
import torch
from transformers import CamembertTokenizer

model_name = "camembert-base"
tokenizer = CamembertTokenizer.from_pretrained(model_name)
import matplotlib.pyplot as plt
reviews_len = [len(tokenizer.encode(review, max_length=512))
                          for review in x_train.designation.values]
print("Average length: {:.1f}".format(np.mean(reviews_len)))
print("Max length: {}".format(max(reviews_len)))

plt.figure(figsize=(10,5))
ax = sns.distplot(reviews_len, bins=150, kde=False, hist_kws=dict(alpha=0.8))
ax.set(xlabel='Number of tokens')

# Finalize the plot
sns.despine(bottom=True)
plt.tight_layout(h_pad=2)

# Saving plot
fig = ax.get_figure()
fig.savefig('./number_of_tokens.png', dpi=200)
import warnings
warnings.filterwarnings('ignore')
from sklearn.pipeline import FeatureUnion, Pipeline, make_pipeline
from sklearn.base import BaseEstimator, TransformerMixin
import spacy
import nltk
nltk.download('stopwords')
!python -m spacy download en_core_web_md
import en_core_web_md
nlp = en_core_web_md.load()
class simple_Text_cleaner(BaseEstimator, TransformerMixin):
  # add another additional parameter, just for fun, while we are at it
    def __init__(self, stopwords,columns,tokenizer=WordPunctTokenizer()): 
        self.columns = columns
        self.stopwords = stopwords
        self.tokenizer=tokenizer
    def rm_stopwords(self,tokens):
        tokens=self.tokenizer.tokenize(tokens)
        return [ tk for tk in tokens if  tk not in self.stopwords ]
    def text_clean_up(self,s=""):
        import re
        user_pattern       = '@[^\s]+'
        s=re.sub(user_pattern, "", s)
        remove = '"#$%&()*+/:;<=>@[\\]^_`{|}~”“'
        pattern = r"[{}]".format(remove)
        s=re.sub(pattern,' ', s) 
        sequencePattern   = r"(.)\1\1+"
        seqReplacePattern = r"\1\1"
        s=re.sub(sequencePattern, seqReplacePattern, s)
        s=re.sub("<[^>]*>",' ', s)
        s=re.sub("[\r\n]+",' ', s)
        s=re.sub("http\S+",' ', s)
        s=re.sub("\$[^>]*\$",' ', s)
        s=re.sub("\d+",' ', s)
        s=re.sub("\s\s+",' ', s)
        s.strip()
        return s
    def fit(self,X,y=None):
        return self
    def transform(self,X,y=None):
        X_=X.copy()
        X_["image_path"]=(X_["imageid"].apply(lambda x: str(int(x)))+"_product_"+X_["productid"].apply(lambda x: str(int(x)))+".jpg").apply(lambda x: "image_"+x)
        for i in self.columns:
            X_[i]=X_[i].apply(lambda x: " ".join(self.rm_stopwords(self.text_clean_up(x))))
        return X_
stop_words = stopwords.words('french')
stop_words.extend(stopwords.words('english'))
cleaner=simple_Text_cleaner(stopwords=stop_words,columns=["designation"])
new_data=cleaner.transform(x_train)<jupyter_output><empty_output><jupyter_text>image_imageid_product_productid.jpg.<jupyter_code>new_reviews_len = [len(tokenizer.encode(review, max_length=512))
                          for review in new_data.designation.values]
print("Average length: {:.1f}".format(np.mean(new_reviews_len)))
print("Max length: {}".format(max(new_reviews_len)))

plt.figure(figsize=(10,5))
ax = sns.distplot(new_reviews_len, bins=150, kde=False, hist_kws=dict(alpha=0.8))
ax.set(xlabel='Number of tokens')

# Finalize the plot
sns.despine(bottom=True)
plt.tight_layout(h_pad=2)

# Saving plot
fig = ax.get_figure()
fig.savefig('./number_of_tokens_clean.png', dpi=200)<jupyter_output>Average length: 19.7
Max length: 114<jupyter_text>On va travailler sur un max_length de 60<jupyter_code>device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
cpu=torch.device("cpu")
!pip install omegaconf
!pip install hydra-core
from transformers import CamembertModel, CamembertTokenizer

# You can replace "camembert-base" with any other model from the table, e.g. "camembert/camembert-large".
tokenizer = CamembertTokenizer.from_pretrained("camembert-base")
camembert = CamembertModel.from_pretrained("camembert-base")

camembert.eval()  # disable dropout (or leave in train mode to finetune)

import torch
# Tokenize in sub-words with SentencePiece
tokenized_sentence = tokenizer.tokenize("J'aime le camembert !")
# ['▁J', "'", 'aime', '▁le', '▁ca', 'member', 't', '▁!'] 
print(tokenized_sentence)
# 1-hot encode and add special starting and end tokens 
encoded_sentence = tokenizer.encode(tokenized_sentence)
# [5, 121, 11, 660, 16, 730, 25543, 110, 83, 6] 
# NB: Can be done in one step : tokenize.encode("J'aime le camembert !")
print(encoded_sentence)
# Feed tokens to Camembert as a torch tensor (batch dim 1)BERT
encoded_sentence = torch.tensor(encoded_sentence).unsqueeze(0)
embeddings= camembert(encoded_sentence)["last_hidden_state"]
#print(embeddings.size())
# embeddings.detach()
# embeddings.size torch.Size([1, 10, 768])
# tensor([[[-0.0254,  0.0235,  0.1027,  ..., -0.1459, -0.0205, -0.0116],
#         [ 0.0606, -0.1811, -0.0418,  ..., -0.1815,  0.0880, -0.0766],
#         [-0.1561, -0.1127,  0.2687,  ..., -0.0648,  0.0249,  0.0446],
#         ...,



from transformers import CamembertConfig
# (Need to reload the model with new config)
config = CamembertConfig.from_pretrained("camembert-base", output_hidden_states=True)
camembert = CamembertModel.from_pretrained("camembert-base", config=config)
camembert.eval()  # disable dropout (or leave in train mode to finetune)

all_layer_embeddings = camembert(encoded_sentence)[2]
#  all_layer_embeddings list of len(all_layer_embeddings) == 13 (input embedding layer + 12 self attention layers)
all_layer_embeddings[5]
print(len(all_layer_embeddings))
# layer 5 contextual embedding : size torch.Size([1, 10, 768])
#tensor([[[-0.0032,  0.0075,  0.0040,  ..., -0.0025, -0.0178, -0.0210],
#         [-0.0996, -0.1474,  0.1057,  ..., -0.0278,  0.1690, -0.2982],
#         [ 0.0557,
all_layer_embeddings[-1].size()

def encode_reviews(tokenizer, reviews, max_length):
    token_ids = np.zeros(shape=(len(reviews), max_length),
                         dtype=np.int32)
    for i, review in enumerate(reviews):
        encoded = tokenizer.encode(review, max_length=max_length)
        token_ids[i, 0:len(encoded)] = encoded
    attention_mask = (token_ids != 0).astype(np.int32)
    return {"input_ids": token_ids, "attention_mask": attention_mask}

class CamembertPreprocessor(BaseEstimator, TransformerMixin):
    def __init__(self, tokenizer, max_seq_length,column):
        self.tokenizer = tokenizer
        self.max_seq_length = max_seq_length
        self.column=column
    def fit(self, X=None):
        return self 
    
    def transform(self, X, y):
        # 1. Tokenize
        X_encoded=X.copy()
        X_encoded[self.column] = encode_reviews(self.tokenizer, X[columns], self.max_seq_length)
        return X_encoded     
    
    def fit_transform(self, X, y):        
        return self.transform(X, y)
MAX_SEQ_LEN = 60 

short_reviews = sum(np.array(reviews_len) <= MAX_SEQ_LEN)
long_reviews = sum(np.array(reviews_len) > MAX_SEQ_LEN)

print("{} reviews with LEN > {} ({:.2f} % of total data)".format(
    long_reviews,
    MAX_SEQ_LEN,
    100 * long_reviews / len(reviews_len)
))<jupyter_output>1129 reviews with LEN > 60 (1.33 % of total data)<jupyter_text>Simple sentence embedding strategy: mean of second to last hidden layer<jupyter_code>import torch.nn as nn 
import transformers as tt
from transformers import CamembertModel, CamembertTokenizer
class Second_to_last_SentenceEmbedding(nn.Module):
    def __init__(self):
        #super(, self).__init__()
        super().__init__()
        self.config = CamembertConfig.from_pretrained("camembert-base", output_hidden_states=True)
        self.camembert =CamembertModel.from_pretrained("camembert-base", config=self.config)
        self.fc1=nn.Linear(768,100)
        self.relu=nn.ReLU()
        self.fc2=nn.Linear(100,27)
        for p in self.camembert.parameters():
            p.requires_grad_(False)
    def forward(self, input,attention_mask=None):
        hidden = self.camembert(input,attention_mask=attention_mask)[2]
        token_vecs=hidden[-2]
        x=torch.mean(token_vecs,dim=1)
      
        return torch.mean(token_vecs,dim=1)
model=Second_to_last_SentenceEmbedding().to(device)
model.eval()
x_train.shape
model(encoded_sentence.to(device)).size()
tokenizer.encode_plus(
                        "<NAME> !",                      # Sentence to encode.
                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                        max_length = 64,           # Pad & truncate all sentences.
                        pad_to_max_length = True,
                        return_attention_mask = True,   # Construct attn. masks.
                        return_tensors = 'pt',     # Return pytorch tensors.
                   )<jupyter_output><empty_output><jupyter_text>- Loading Image First we will build a custom Dataset uitilitaries and a Data Loader<jupyter_code>from skimage import io, transform
import numpy as np
import matplotlib.pyplot as plt
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils

class MultimodalDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self,tokenizer, input_csv_file, root_dir,max_length=60 , transform=None,Y_csv_file=None):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.input_file = pd.read_csv(input_csv_file)
        self.input_file.rename(columns={"Unnamed: 0": "Id"}, inplace=True)
        self.root_dir = root_dir
        self.transform = transform
        self.max_length=max_length
        self.input_file["image_path"]=(self.input_file["imageid"].apply(lambda x: str(int(x)))+"_product_"+self.input_file["productid"].apply(lambda x: str(int(x)))+".jpg").apply(lambda x: "image_"+x)
        if Y_csv_file !=None:
            self.output = pd.read_csv(Y_csv_file)
            self.output.rename(columns={"Unnamed: 0": "Id"}, inplace=True)
            self.classes = list(set(self.output["prdtypecode"].values))
        else:
            self.ouptut=None
    def __len__(self):
        return len(self.input_file)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        
        img_name = os.path.join(self.root_dir,
                                self.input_file["image_path"].iloc[idx])
        image = io.imread(img_name)
        text = self.input_file["designation"].iloc[idx]
        encoded_text = tokenizer.encode_plus(
                        text,                      # Sentence to encode.
                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                        max_length = 64,           # Pad & truncate all sentences.
                        pad_to_max_length = True,
                        return_attention_mask = True,   # Construct attn. masks.
                        return_tensors = 'pt',     # Return pytorch tensors.
                   )
        sample = {'image': image, 'input_ids':encoded_text['input_ids'] ,'attention_mask':encoded_text['attention_mask']}

        if self.transform:
            sample['image'] = self.transform(sample['image'])
        if self.output is None:
            return sample
        sample["label"]=torch.tensor(self.classes.index(self.output["prdtypecode"].loc[self.input_file["Id"].iloc[idx]]))
        return sample
from __future__ import print_function, division
import os
import torch
import pandas as pd
from skimage import io, transform
import numpy as np
def show_landmarks(image):
    """Show image with landmarks"""
    plt.imshow(image)
    plt.pause(0.001)  # pause a bit so that plots are updated

plt.figure()
show_landmarks(io.imread(os.path.join('/content/images/images/image_train', new_data["image_path"].iloc[25])))
plt.show()
?np.random.random_integers
face_dataset = MultimodalDataset(tokenizer=tokenizer, input_csv_file="/content/X_train_update.csv", root_dir="/content/images/images/image_train",max_length=60 , transform=None,Y_csv_file="/content/Y_train_CVw08PX.csv")

fig = plt.figure()
l=np.random.random_integers(0,84916,6)
for i in range(len(l)):
    sample = face_dataset[l[i]]

    print(i, sample['image'].shape)

    ax = plt.subplot(1, 6, i + 1)
    plt.tight_layout()
    ax.set_title('Sample #{}'.format(face_dataset.classes[sample["label"]]))
    ax.axis('off')
    show_landmarks(sample["image"])
    plt.show()
4%3
height=[]
width=[]
for i in range(84916):
  image=io.imread(os.path.join('/content/images/images/image_train', new_data["image_path"].iloc[i]))
  height.append(image.shape[0])
  width.append(image.shape[1])<jupyter_output><empty_output><jupyter_text>Training Dataset Image Transformation (Data Augmentation)<jupyter_code>class Microscope:
    """
    Cutting out the edges around the center circle of the image
    Imitating a picture, taken through the microscope

    Args:
        p (float): probability of applying an augmentation
    """

    def __init__(self, p: float = 0.5):
        self.p = p

    def __call__(self, img):
        """
        Args:
            img (PIL Image): Image to apply transformation to.

        Returns:
            PIL Image: Image with transformation.
        """
        if random.random() < self.p:
            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder
                        (img.shape[0]//2, img.shape[1]//2), # center point of circle
                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius
                        (0, 0, 0), # color
                        -1)

            mask = circle - 255
            img = np.multiply(img, mask)
        
        return img

    def __repr__(self):
        return f'{self.__class__.__name__}(p={self.p})'
train_transform = transforms.Compose([
    Microscope(p=0.5),
    transforms.ToPILImage(),
    transforms.RandomResizedCrop(size=384, scale=(0.8, 1.0)),
    transforms.RandomHorizontalFlip(),
    transforms.RandomVerticalFlip(),
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
])
test_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225])
])

data_transforms = {
    'train': transforms.Compose([
        Microscope(p=0.5),
        transforms.ToPILImage(),
        transforms.RandomResizedCrop(size=384, scale=(0.8, 1.0)),
        transforms.RandomHorizontalFlip(),
        transforms.RandomVerticalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'test': transforms.Compose([
        transforms.ToPILImage(),
        transforms.Resize(512),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}<jupyter_output><empty_output><jupyter_text>Building the model we will use<jupyter_code>import torch.nn as nn 
import transformers as tt
from transformers import CamembertModel, CamembertTokenizer
from torchvision import models,transforms
class Feature_Extraction(nn.Module):
    # deuxieme dimension de l'output de chaque tenseurs a la sortie d'un layer de resnet
    resnet_caracteristics=[64,64,64,64,256,512,1024,2048,2048]
    pooling_target=[(4,8),(4,8),(4,8),(4,8),(2,4),(2,2),(1,2),(1,1),(1,1)]
    embedding_strategy={"Second_to_last_average":768,"Start_token_embedding":768,"last_four_embedding_average":3072}
    def __init__(self,resnet_layers=-1,to_tune=False,sentence_embedding="Second_to_last_average"):
        #super(, self).__init__()
        super().__init__()
        self.config = CamembertConfig.from_pretrained("camembert-base", output_hidden_states=True)
        self.camembert =CamembertModel.from_pretrained("camembert-base", config=self.config)
        self.Resnet = models.resnet50(pretrained=True)
        self.Resnet= nn.Sequential(*list(self.Resnet.children())[:resnet_layers])
        self.pooling=nn.AdaptiveMaxPool2d(self.pooling_target[resnet_layers-1])
        self.flat=nn.Flatten()
        self.strategy=sentence_embedding
        if sentence_embedding not in self.embedding_strategy:
            self.strategy="Second_to_last_average"    
        if self.strategy =="Start_token_embedding":
            to_tune=True
        if to_tune == False:
            for p in self.camembert.parameters():
                p.requires_grad_(False)
            for p in self.Resnet.parameters():
                p.requires_grad_(False)
        self.output_size=2048+ self.embedding_strategy[self.strategy]
    def sentence_embedding(self,hiddens):
        # we get the output of the second to last hidden layers and average it over all token
        if self.strategy == "Second_to_last_average":
            return torch.mean(hiddens[-2],dim=1) 
        # we use the first token embedding  from the ouptut last hidden layers 
        # fine_tune should be true 
        elif self.strategy == "Start_token_embedding":
            return hiddens[-1].permute(1,0,2)[0]
        # we use the last four hidden layer average and concatenate them
        elif self.strategy == "last_four_embedding_average":
            x=torch.cat((hiddens[-4],hiddens[-3],hiddens[-2],hiddens[-1]),dim=2)
            return torch.mean(x,dim=1)
    def forward(self, input,image,attention_mask=None):
        hiddens = self.camembert(input,attention_mask=attention_mask)[2]
        embeddings=self.sentence_embedding(hiddens)
        x=self.Resnet(image)
        h=self.pooling(x)
        return torch.cat((embeddings, self.flat(h)),dim=1)
Resnet = models.resnet50(pretrained=True)
list(Resnet.children())
?nn.Flatten
image=io.imread(os.path.join('/content/images/images/image_train', new_data["image_path"].iloc[25]))
print(image.shape)
image=data_transforms["train"](image)
l=[]
l2=[]
h=nn.Flatten()
for i in range(1,10):
  print(i)
  resnet=nn.Sequential(*list(Resnet.children())[:i])
  resnet.to(device)
  resnet.eval()
  output=resnet(image.to(device).unsqueeze(0))
  l.append(output.size())  
  l2.append(h(output).size())
l,l2
len(l),len(l[:-1])
768*4
torch.cat((all_layer_embeddings[-2],all_layer_embeddings[-1]),dim=2).size()
?nn.AdaptiveMaxPool2d
import random
import cv2
face_dataset = MultimodalDataset(tokenizer=tokenizer, input_csv_file="/content/X_train_update.csv", root_dir="/content/images/images/image_train",max_length=60 , transform=data_transforms["train"],Y_csv_file="/content/Y_train_CVw08PX.csv")
x=face_dataset[25]
x
x["image"].size()
extractor=Feature_Extraction(resnet_layers=6,to_tune=False,sentence_embedding="Second_to_last_average").to(device)
extractor.eval()
embed,features=extractor(input=x["input_ids"].to(device),image=(x["image"].to(device)).unsqueeze(0),attention_mask=x["attention_mask"].to(device))
extractor.strategy
embed.size()#1,768
class Multimodal_Dense_model(nn.Module):
    def __init__(self,dropout=0.2,resnet_layers=-1,to_tune=False,sentence_embedding="Second_to_last_average"):
        #super(, self).__init__()
        super().__init__()
        self.feature_extractor=Feature_Extraction(resnet_layers=resnet_layers,to_tune=to_tune,sentence_embedding=sentence_embedding)
        self.input_size=self.feature_extractor.output_size
        self.dense_layer=nn.Sequential(OrderedDict([
          ('dense1', nn.Linear(in_features=self.input_size,out_features=768),
          ('relu1', nn.ReLU()),
          ('dropout1', nn.Dropout(p=0.2)),
          ('dense2', nn.Linear(in_features=768,out_features=256),
          ('relu2', nn.ReLU()),
          ('dropout2', nn.Dropout(p=0.2)),
          ('dense3', nn.Linear(in_features=256,out_features=64),
          ('relu3', nn.ReLU()),
          ('dropout3', nn.Dropout(p=0.2))
        ]))
        self.fc=nn.Linear(64,27)
    def forward(self, input,image,attention_mask=None):
        feature=self.feature_extractor(input=input,image=image,attention_mask=attention_mask)
        h=self.dense_layer(feature)
        return self.fc(h)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13962580284836637, 'fraction_numerical': 0.034859908777808805, 'mean_word_length': 3.5494389159432562, 'pattern_counts': {'":': 14, '<': 39, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '89850', 'license': 'MIT', 'n_tokens_mistral': 7846, 'n_tokens_neox': 7286, 'n_words': 1306, 'path': 'rakuten_method(2).ipynb', 'repo_name': 'YoungDevil-glitch/Rakuten-Paris-Product-Classification', 'size': '21499'}