Upload krfinbert_esg (1).py

krfinbert_esg (1).py

@@ -0,0 +1,353 @@
+# -*- coding: utf-8 -*-
+"""krfinbert_esg.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1_cVBwxsa7LcHzjzCcS4l1ds0wxNPQrjm
+"""
+
+from google.colab import drive
+drive.mount('/content/drive')
+
+import pandas as pd
+import numpy as np
+
+import warnings
+warnings.filterwarnings('ignore') # to avoid warnings
+
+import random
+import pandas as pd
+from tqdm import tqdm
+import seaborn as sns
+import matplotlib.pyplot as plt
+
+"""
+Sklearn Libraries
+"""
+from sklearn.metrics import f1_score
+from sklearn.model_selection import train_test_split
+
+"""
+Transformer Libraries
+"""
+!pip install transformers
+from transformers import BertTokenizer,  AutoModelForSequenceClassification, AdamW, get_linear_schedule_with_warmup
+
+"""
+Pytorch Libraries
+"""
+import torch
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
+
+esg_data = pd.read_csv("/content/drive/MyDrive/kpmg_personal/concat.csv", 
+                             encoding='utf-8')
+
+esg_data
+
+plt.figure(figsize = (15,8))
+
+sns.set(style='darkgrid')
+    
+# Increase information on the figure
+sns.set(font_scale=1.3)
+sns.countplot(x='category', data = esg_data)
+plt.title('ESG Category Distribution')
+plt.xlabel('E,S,G,N')
+plt.ylabel('Number of Contents')
+
+def show_random_contents(total_number, df):
+    
+    # Get the random number of reviews
+    n_contents = df.sample(total_number)
+    
+    # Print each one of the reviews
+    for val in list(n_contents.index):
+        print("Contents #°{}".format(val))
+        print(" - Category: {}".format(df.iloc[val]["category"]))
+        print(" - Contents: {}".format(df.iloc[val]["contents"]))
+        print("")
+        
+# Show 5 random headlines
+show_random_contents(5, esg_data)
+
+def encode_categories_values(df):
+    
+    possible_categories = df.category.unique()
+    category_dict = {}
+    
+    for index, possible_category in enumerate(possible_categories):
+        category_dict[possible_category] = index
+    
+    # Encode all the sentiment values
+    df['label'] = df.category.replace(category_dict)
+    
+    return df, category_dict
+ 
+# Perform the encoding task on the data set
+esg_data, category_dict = encode_categories_values(esg_data)
+
+X_train,X_val, y_train, y_val = train_test_split(esg_data.index.values, 
+                                                  esg_data.label.values, 
+                                                  test_size = 0.15, 
+                                                  random_state = 2022, 
+                                                  stratify = esg_data.label.values)
+
+esg_data.loc[X_train, 'data_type'] = 'train'
+esg_data.loc[X_val, 'data_type'] = 'val'
+
+# Vizualiez the number of sentiment occurence on each type of data
+esg_data.groupby(['category', 'label', 'data_type']).count()
+
+# Get the FinBERT Tokenizer
+finbert_tokenizer = BertTokenizer.from_pretrained('snunlp/KR-FinBert-SC', 
+                                          do_lower_case=True)
+
+def get_contents_len(df):
+    
+    contents_sequence_lengths = []
+    
+    print("Encoding in progress...")
+    for content in tqdm(df.contents):
+        encoded_content = finbert_tokenizer.encode(content, 
+                                         add_special_tokens = True)
+        
+        # record the length of the encoded review
+        contents_sequence_lengths.append(len(encoded_content))
+    print("End of Task.")
+    
+    return contents_sequence_lengths
+
+def show_contents_distribution(sequence_lengths, figsize = (15,8)):
+    
+    # Get the percentage of reviews with length > 512
+    len_512_plus = [rev_len for rev_len in sequence_lengths if rev_len > 512]
+    percent = (len(len_512_plus)/len(sequence_lengths))*100
+    
+    print("Maximum Sequence Length is {}".format(max(sequence_lengths)))
+    
+    # Configure the plot size
+    plt.figure(figsize = figsize)
+
+    sns.set(style='darkgrid')
+    
+    # Increase information on the figure
+    sns.set(font_scale=1.3)
+    
+    # Plot the result
+    sns.distplot(sequence_lengths, kde = False, rug = False)
+    plt.title('Contents Lengths Distribution')
+    plt.xlabel('Contents Length')
+    plt.ylabel('Number of Contents')
+
+show_contents_distribution(get_contents_len(esg_data))
+
+# Encode the Training and Validation Data
+encoded_data_train = finbert_tokenizer.batch_encode_plus(
+    esg_data[esg_data.data_type=='train'].contents.values, 
+    return_tensors='pt',
+    add_special_tokens=True, 
+    return_attention_mask=True, 
+    pad_to_max_length=True, 
+    max_length=200 # the maximum lenght observed in the headlines
+)
+
+encoded_data_val = finbert_tokenizer.batch_encode_plus(
+    esg_data[esg_data.data_type=='val'].contents.values, 
+    return_tensors='pt',
+    add_special_tokens=True, 
+    return_attention_mask=True, 
+    pad_to_max_length=True, 
+    max_length=200 # the maximum length observed in the headlines
+)
+
+
+input_ids_train = encoded_data_train['input_ids']
+attention_masks_train = encoded_data_train['attention_mask']
+labels_train = torch.tensor(esg_data[esg_data.data_type=='train'].label.values)
+
+input_ids_val = encoded_data_val['input_ids']
+attention_masks_val = encoded_data_val['attention_mask']
+sentiments_val = torch.tensor(esg_data[esg_data.data_type=='val'].label.values)
+
+
+dataset_train = TensorDataset(input_ids_train, attention_masks_train, labels_train)
+dataset_val = TensorDataset(input_ids_val, attention_masks_val, sentiments_val)
+
+model = AutoModelForSequenceClassification.from_pretrained("snunlp/KR-FinBert-SC",
+                                                          num_labels=len(category_dict),
+                                                          output_attentions=False,
+                                                          output_hidden_states=False,
+                                                           ignore_mismatched_sizes=True)
+
+batch_size = 5
+
+dataloader_train = DataLoader(dataset_train, 
+                              sampler=RandomSampler(dataset_train), 
+                              batch_size=batch_size)
+
+dataloader_validation = DataLoader(dataset_val, 
+                                   sampler=SequentialSampler(dataset_val), 
+                                   batch_size=batch_size)
+
+optimizer = AdamW(model.parameters(),
+                  lr=1e-5, 
+                  eps=1e-8)
+
+epochs = 5
+
+scheduler = get_linear_schedule_with_warmup(optimizer, 
+                                            num_warmup_steps=0,
+                                            num_training_steps=len(dataloader_train)*epochs)
+
+def f1_score_func(preds, labels):
+    preds_flat = np.argmax(preds, axis=1).flatten()
+    labels_flat = labels.flatten()
+    return f1_score(labels_flat, preds_flat, average='weighted')
+
+def accuracy_per_class(preds, labels):
+    label_dict_inverse = {v: k for k, v in category_dict.items()}
+    
+    preds_flat = np.argmax(preds, axis=1).flatten()
+    labels_flat = labels.flatten()
+
+    for label in np.unique(labels_flat):
+        y_preds = preds_flat[labels_flat==label]
+        y_true = labels_flat[labels_flat==label]
+        print(f'Class: {label_dict_inverse[label]}')
+        print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\n')
+
+seed_val = 2022
+random.seed(seed_val)
+np.random.seed(seed_val)
+torch.manual_seed(seed_val)
+torch.cuda.manual_seed_all(seed_val)
+
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+model.to(device)
+
+
+def evaluate(dataloader_val):
+
+    model.eval()
+    
+    loss_val_total = 0
+    predictions, true_vals = [], []
+    
+    for batch in dataloader_val:
+        
+        batch = tuple(b.to(device) for b in batch)
+        
+        inputs = {'input_ids':      batch[0],
+                  'attention_mask': batch[1],
+                  'labels':         batch[2],
+                 }
+
+        with torch.no_grad():        
+            outputs = model(**inputs)
+            
+        loss = outputs[0]
+        logits = outputs[1]
+        loss_val_total += loss.item()
+
+        logits = logits.detach().cpu().numpy()
+        label_ids = inputs['labels'].cpu().numpy()
+        predictions.append(logits)
+        true_vals.append(label_ids)
+    
+    loss_val_avg = loss_val_total/len(dataloader_val) 
+    
+    predictions = np.concatenate(predictions, axis=0)
+    true_vals = np.concatenate(true_vals, axis=0)
+            
+    return loss_val_avg, predictions, true_vals
+
+
+for epoch in tqdm(range(1, epochs+1)):
+    
+    model.train()
+    
+    loss_train_total = 0
+
+    progress_bar = tqdm(dataloader_train, desc='Epoch {:1d}'.format(epoch), leave=False, disable=False)
+    for batch in progress_bar:
+
+        model.zero_grad()
+        
+        batch = tuple(b.to(device) for b in batch)
+        
+        inputs = {'input_ids':      batch[0],
+                  'attention_mask': batch[1],
+                  'labels':         batch[2],
+                 }       
+
+        outputs = model(**inputs)
+        
+        loss = outputs[0]
+        loss_train_total += loss.item()
+        loss.backward()
+
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+
+        optimizer.step()
+        scheduler.step()
+        
+        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})
+         
+    torch.save(model.state_dict(), f'finetuned_finBERT_epoch_{epoch}.model')
+        
+    tqdm.write(f'\nEpoch {epoch}')
+    
+    loss_train_avg = loss_train_total/len(dataloader_train)            
+    tqdm.write(f'Training loss: {loss_train_avg}')
+    
+    val_loss, predictions, true_vals = evaluate(dataloader_validation)
+    val_f1 = f1_score_func(predictions, true_vals)
+    tqdm.write(f'Validation loss: {val_loss}')
+    tqdm.write(f'F1 Score (Weighted): {val_f1}')
+
+model = AutoModelForSequenceClassification.from_pretrained("snunlp/KR-FinBert-SC",
+                                                          num_labels=len(category_dict),
+                                                          output_attentions=False,
+                                                          output_hidden_states=False,
+                                                           ignore_mismatched_sizes=True)
+
+model.to(device)
+
+model.load_state_dict(torch.load('finetuned_finBERT_epoch_4.model', 
+                                 map_location=torch.device('cpu')))
+
+_, predictions, true_vals = evaluate(dataloader_validation)
+
+accuracy_per_class(predictions, true_vals)
+
+# max_length = 200
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+