<jupyter_start><jupyter_text><jupyter_code>import torch
from tqdm.notebook import tqdm
!pip install transformers
from transformers import BertTokenizer
from torch.utils.data import TensorDataset

from transformers import BertForSequenceClassification
import numpy as np
import pandas as pd
from pandas import Series,DataFrame
import os
import json
import sys
from datetime import datetime
import tensorflow as tf
df = pd.read_csv('1001.csv',sep=";",encoding="latin-1")
df
possible_labels = df.sub_industry_code.unique()

label_dict = {}
for index, possible_label in enumerate(possible_labels):
    label_dict[possible_label] = index
label_dict
df['label'] = df.sub_industry_code.replace(label_dict)
df
import seaborn as sns
import matplotlib.pyplot as plt
plt.figure(figsize=(50, 6))
ax =sns.countplot(df.sub_industry_code)

ax.set_xticklabels(ax.get_xticklabels(), rotation=40, ha="right")
plt.show()
plt.xlabel('review score')
df = df[df.sub_industry_code != '1026-5']
#['1007-5','1024-5','1026-5','1023-2','1014-3']
from sklearn.model_selection import train_test_split

X_train, X_val, y_train, y_val = train_test_split(df.index.values, 
                                                  df.label.values, 
                                                  test_size=0.15, 
                                                  random_state=42, 
                                                  stratify=df.label.values)
df['data_type'] = ['not_set']*df.shape[0]
df.loc[X_train, 'data_type'] = 'train'
df.loc[X_val, 'data_type'] = 'val'
df.groupby(['label', 'data_type']).count()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', 
                                          do_lower_case=True)
df['desc_clean'].isnull().sum()
df['desc_clean'].replace('', np.nan, inplace=True)
df.dropna(subset=['desc_clean'], inplace=True)
token_lens = []

for txt in df.desc_clean:
  tokens = tokenizer.encode(txt, max_length=512)
  token_lens.append(len(tokens))
import seaborn as sns
import matplotlib.pyplot as plt
sns.distplot(token_lens)
plt.xlim([0, 256]);
plt.xlabel('Token count');
encoded_data_train = tokenizer.batch_encode_plus(
    df[df.data_type=='train'].desc_clean.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    pad_to_max_length=True, 
    max_length=256, 
    return_tensors='pt'
)

encoded_data_val = tokenizer.batch_encode_plus(
    df[df.data_type=='val'].desc_clean.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    pad_to_max_length=True, 
    max_length=256, 
    return_tensors='pt'
)


input_ids_train = encoded_data_train['input_ids']
attention_masks_train = encoded_data_train['attention_mask']
labels_train = torch.tensor(df[df.data_type=='train'].label.values)

input_ids_val = encoded_data_val['input_ids']
attention_masks_val = encoded_data_val['attention_mask']
labels_val = torch.tensor(df[df.data_type=='val'].label.values)
dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)
dataset_val = TensorDataset(input_ids_val, attention_masks_val, labels_val)
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                      num_labels=len(label_dict),
                                                      output_attentions=False,
                                                      output_hidden_states=False)
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler

batch_size = 32

dataloader_train = DataLoader(dataset_train, 
                              sampler=RandomSampler(dataset_train), 
                              batch_size=batch_size)

dataloader_validation = DataLoader(dataset_val, 
                                   sampler=SequentialSampler(dataset_val), 
                                   batch_size=batch_size)

from transformers import AdamW, get_linear_schedule_with_warmup

optimizer = AdamW(model.parameters(),
                  lr=1e-5, 
                  eps=1e-8)
epochs = 8

scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps=0,
                                            num_training_steps=len(dataloader_train)*epochs)
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

def f1_score_func(preds, labels):
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return f1_score(labels_flat, preds_flat, average='weighted')

def cr(preds, labels):
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return classification_report(labels_flat, preds_flat)

def c_m(preds, labels):
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return confusion_matrix(labels_flat, preds_flat)

def accuracy_per_class(preds, labels):
    label_dict_inverse = {v: k for k, v in label_dict.items()}
    
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()

    for label in np.unique(labels_flat):
        y_preds = preds_flat[labels_flat==label]
        y_true = labels_flat[labels_flat==label]
        print(f'Class: {label_dict_inverse[label]}'+ ';'+f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\n')
        #print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\n')



import random

seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)

print(device)
def evaluate(dataloader_val):

    model.eval()
    
    loss_val_total = 0
    predictions, true_vals = [], []
    
    for batch in dataloader_val:
        
        batch = tuple(b.to(device) for b in batch)
        
        inputs = {'input_ids':      batch[0],
                  'attention_mask': batch[1],
                  'labels':         batch[2],
                 }

        with torch.no_grad():        
            outputs = model(**inputs)
            
        loss = outputs[0]
        logits = outputs[1]
        loss_val_total += loss.item()

        logits = logits.detach().cpu().numpy()
        label_ids = inputs['labels'].cpu().numpy()
        predictions.append(logits)
        true_vals.append(label_ids)
    
    loss_val_avg = loss_val_total/len(dataloader_val) 
    
    predictions = np.concatenate(predictions, axis=0)
    true_vals = np.concatenate(true_vals, axis=0)
            
    return loss_val_avg, predictions, true_vals
import torch
torch.cuda.empty_cache()
for epoch in tqdm(range(1, epochs+1)):
    
    model.train()
    
    loss_train_total = 0


    progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)
    for batch in progress_bar:

        model.zero_grad()
        
        batch = tuple(b.to(device) for b in batch)
        
        inputs = {'input_ids':      batch[0],
                  'attention_mask': batch[1],
                  'labels':         batch[2],
                 }       

        outputs = model(**inputs)
        
        loss = outputs[0]
        loss_train_total += loss.item()
        loss.backward()

        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()
        scheduler.step()
        
        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})
         
        
    torch.save(model.state_dict(), f'finetuned_BERT_epoch_{epoch}.model')
        
    tqdm.write(f'\nEpoch {epoch}')
    
    loss_train_avg = loss_train_total/len(dataloader_train)            
    tqdm.write(f'Training loss: {loss_train_avg}')
    
    val_loss, predictions, true_vals = evaluate(dataloader_validation)
    val_f1 = f1_score_func(predictions, true_vals)
    tqdm.write(f'Validation loss: {val_loss}')
    tqdm.write(f'F1 Score (Weighted): {val_f1}')
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                      num_labels=len(label_dict),
                                                      output_attentions=False,
                                                      output_hidden_states=False)

model.to(device)
model.load_state_dict(torch.load('finetuned_BERT_epoch_6.model', map_location=torch.device('cuda')))<jupyter_output><empty_output><jupyter_text>_, predictions, true_vals = evaluate(dataloader_validation)<jupyter_code>accuracy_per_class(predictions, true_vals)

'1001-1': 13,
 '1001-10': 2,
 '1001-11': 9,
 '1001-12': 6,
 '1001-13': 10,
 '1001-14': 0,
 '1001-2': 3,
 '1001-3': 12,
 '1001-4': 1,
 '1001-5': 11,
 '1001-6': 8,
 '1001-7': 7,
 '1001-8': 5,
 '1001-9': 4
print(cr(predictions, true_vals))
'1001-1': 13,
 '1001-10': 2,
 '1001-11': 9,
 '1001-12': 6,
 '1001-13': 10,
 '1001-14': 0,
 '1001-2': 3,
 '1001-3': 12,
 '1001-4': 1,
 '1001-5': 11,
 '1001-6': 8,
 '1001-7': 7,
 '1001-8': 5,
 '1001-9': 4
## Plot confusion matrix
def c_m(preds, labels):
    preds_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return confusion_matrix(labels_flat, preds_flat)

classes = np.unique(true_vals)
from sklearn import metrics
cm = c_m(predictions, true_vals)
fig, ax = plt.subplots(figsize=(50, 50))
#plt.figure(figsize=(50, 6))

sns.heatmap(cm, annot=True, fmt='d', ax=ax, cmap=plt.cm.Blues, 
            cbar=False)
ax.set(xlabel="predictions", ylabel="true_vals", xticklabels=classes, 
       yticklabels=classes, title="Confusion matrix")
plt.yticks(rotation=0)
# Get some data:
#ata_leadBook = data_leadBook.sample(20)
texts = ['graphic design print web']
real = ['1001-11']
#true_labels = sample['label'].to_list()
# Tokenise the texts and run the model:
inputs = tokenizer(texts, padding='max_length', return_tensors='pt')
model.eval()
torch.device("cuda" if torch.cuda.is_available() else "cpu"); 
inputs.to(device)

with torch.no_grad():outputs = model(**inputs)
# Get the predicted labels
pred_labels = np.argmax(outputs[0].cpu().detach().numpy(), axis=1).tolist()
pred_labels



label_dict1 ={13: '1001-1',
 2: '1001-10',
 9:'1001-11'}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1262002743484225, 'fraction_numerical': 0.03203997648442093, 'mean_word_length': 2.9964761158966327, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '182923', 'license': 'MIT', 'n_tokens_mistral': 3650, 'n_tokens_neox': 3380, 'n_words': 588, 'path': 'L_Sub_industry_Classification.ipynb', 'repo_name': 'VentesWorks/pythonKakfaWrapper', 'size': '10206'}