<jupyter_start><jupyter_text>For Reference: https://github.com/sumitc29/NLP/tree/master/BERT%20Text%20classification%20model<jupyter_code>"""importing all packages"""
import numpy as np 
import pandas as pd
import re
import gc
import os
import fileinput
import string
import tensorflow as tf
import zipfile
import datetime
import sys
from tqdm  import tqdm
tqdm.pandas()
from nltk.tokenize import wordpunct_tokenize
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score, roc_auc_score

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation
from tensorflow.keras.layers import Bidirectional,Activation, Dense, Dropout, Embedding, Flatten, Conv1D, MaxPooling1D, LSTM, Input
from sklearn.metrics import classification_report

from bert import modeling
from bert import optimization
from  bert import run_classifier

import tokenization
import time
%time
# import tensorflow as tf

# print(tf.__version__)

# !pip uninstall tensorflow==2.2.0

# !pip install tensorflow==1.15.0

# !pip install bert-tensorflow
"""loading pretrained BERT classifier alongwith other files"""
!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
!wget https://raw.githubusercontent.com/google-research/bert/master/modeling.py 
!wget https://raw.githubusercontent.com/google-research/bert/master/optimization.py 
!wget https://raw.githubusercontent.com/google-research/bert/master/run_classifier.py 
!wget https://raw.githubusercontent.com/google-research/bert/master/tokenization.py 
  """Extracting zip files into model folder"""
import zipfile
folder = '/content/drive/MyDrive/Sentiment Analysis_Preskale/model_folder'
with zipfile.ZipFile("uncased_L-12_H-768_A-12.zip","r") as zip_ref:
    zip_ref.extractall(folder)
import pathlib
pathlib.Path(folder).parent.absolute()
    
"""creating additional folder in model folder for convenience"""
BERT_MODEL = 'uncased_L-12_H-768_A-12'
BERT_PRETRAINED_DIR = f'{folder}/uncased_L-12_H-768_A-12'
OUTPUT_DIR = f'{folder}/outputs'
print(f'>> Model output directory: {OUTPUT_DIR}')
print(f'>>  BERT pretrained directory: {BERT_PRETRAINED_DIR}')
data=pd.read_csv("/content/drive/My Drive/Sentiment Analysis_Preskale/EDA FOR REVIEWS DATASET/Cleaned_reviews.csv",index_col="Unnamed: 0")
data.head()
data=data.loc[:,["content","Polarity","clean_text"]]
data.columns
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 15746 entries, 0 to 15745
Data columns (total 3 columns):
 #   Column      Non-Null Count  Dtype 
---  ------      --------------  ----- 
 0   content     15746 non-null  object
 1   Polarity    15746 non-null  object
 2   clean_text  15737 non-null  object
dtypes: object(3)
memory usage: 492.1+ KB<jupyter_text>Some columns which i perceived are not useful for the modeling purpose and the target columns "Polarity" is essential to be encoded<jupyter_code>data.isnull().sum()
data.dropna(inplace=True)
data.reset_index(inplace=True,drop=True)
data.head()<jupyter_output><empty_output><jupyter_text>Label Encoding the 'Polarity' column<jupyter_code># cond=[df.Polarity=="Positive",df.Polarity=="Neutral",df.Polarity=="Negative"]
# choice=[2,1,0]
# data.Polarity=np.select(cond,choice)
# data.Polarity.head()<jupyter_output><empty_output><jupyter_text>OR<jupyter_code>from sklearn.preprocessing import LabelEncoder
le=LabelEncoder()
data["Polarity"]=le.fit_transform(data["Polarity"])
data.head()
data.Polarity.value_counts().rename({2:'Positive',0:'Negative',1:'Neutral'})
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(data["clean_text"].values, data["Polarity"].values, test_size=0.2, random_state=42,stratify= data.Polarity.values)
print(f"X_train.shape: {X_train.shape}")
print(f"y_train.shape: {y_train.shape}")
print(f"X_test.shape: {X_test.shape}")
print(f"y_test.shape: {y_test.shape}")
# data['data_type']= ['not_set']*data.shape[0]

# data.loc[X_train, 'data_type']= 'train'
# data.loc[X_test, 'data_type'] = 'val'


# data.groupby(['content','Polarity','data_type']).count()
def create_examples(lines, set_type, labels=None):
#Generate data for the BERT model
    guid = f'{set_type}'
    examples = []
    if guid == 'train':
        for line, label in zip(lines, labels):
            text_a = line
            label = str(label)
            """Creating single training/test example for simple sequence classification."""
            examples.append(run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
    else:
        for line in lines:
            text_a = line
            label = '0'
            examples.append(run_classifier.InputExample(guid=guid, text_a=text_a, text_b=None, label=label))
    return examples
# Model Hyper Parameters
TRAIN_BATCH_SIZE = 32
EVAL_BATCH_SIZE = 8
LEARNING_RATE = 1e-5
NUM_TRAIN_EPOCHS = 3.0
WARMUP_PROPORTION = 0.1
MAX_SEQ_LENGTH = 50
# Model configs
SAVE_CHECKPOINTS_STEPS = 100000 #if you wish to finetune a model on a larger dataset, use larger interval
# each checpoint weights about 1,5gb
ITERATIONS_PER_LOOP = 100000
NUM_TPU_CORES = 8
VOCAB_FILE = os.path.join(BERT_PRETRAINED_DIR, 'vocab.txt')
CONFIG_FILE = os.path.join(BERT_PRETRAINED_DIR, 'bert_config.json')
INIT_CHECKPOINT = os.path.join(BERT_PRETRAINED_DIR, 'bert_model.ckpt')
DO_LOWER_CASE = BERT_MODEL.startswith('uncased')

label_list = [str(num) for num in range(data['Polarity'].nunique())]
tokenizer = tokenization.FullTokenizer(vocab_file=VOCAB_FILE, do_lower_case=DO_LOWER_CASE)
"""Creating input in the form of Text sequence classification"""
train_examples = create_examples(X_train, 'train', labels=y_train)
tpu_cluster_resolver = None #Since training will happen on GPU, we won't need a cluster resolver
#TPUEstimator also supports training on CPU and GPU. You don't need to define a separate tf.estimator.Estimat
"""Making all running configurations"""
run_config = tf.contrib.tpu.RunConfig(
    cluster=tpu_cluster_resolver,
    model_dir=OUTPUT_DIR,
    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS,
    tpu_config=tf.contrib.tpu.TPUConfig(
        iterations_per_loop=ITERATIONS_PER_LOOP,
        num_shards=NUM_TPU_CORES,
        per_host_input_for_training=tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2))

"""this much of total iterations will be called"""
num_train_steps = int(len(train_examples) / TRAIN_BATCH_SIZE * NUM_TRAIN_EPOCHS)

"""Why to use this"""
num_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)
model_fn = run_classifier.model_fn_builder(
    bert_config=modeling.BertConfig.from_json_file(CONFIG_FILE),
    num_labels=len(label_list),
    init_checkpoint=INIT_CHECKPOINT,
    learning_rate=LEARNING_RATE,
    num_train_steps=num_train_steps,
    num_warmup_steps=num_warmup_steps,
    use_tpu=False, #If False training will fall on CPU or GPU, depending on what is available  
    use_one_hot_embeddings=True)
estimator = tf.contrib.tpu.TPUEstimator(
    use_tpu=False, #If False training will fall on CPU or GPU, depending on what is available 
    model_fn=model_fn,
    config=run_config,
    train_batch_size=TRAIN_BATCH_SIZE,
    eval_batch_size=EVAL_BATCH_SIZE)
'''Since the model is pretrained why do we need to train it again'''
print('Please wait...')
"""preparing training data """
train_features = run_classifier.convert_examples_to_features(train_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
print('>> Started training at {} '.format(datetime.datetime.now()))
print('  Num examples = {}'.format(len(train_examples)))
print('  Batch size = {}'.format(TRAIN_BATCH_SIZE))
tf.logging.info("  Num steps = %d", num_train_steps)
"""Here run classifier the the classifier from the bert which will get trained and will be responsible for prediction as well"""
train_input_fn = run_classifier.input_fn_builder(
    features=train_features,
    seq_length=MAX_SEQ_LENGTH,
    is_training=True,
    drop_remainder=True)
estimator.train(input_fn=train_input_fn, max_steps=num_train_steps)
print('>> Finished training at {}'.format(datetime.datetime.now()))

def input_fn_builder(features, seq_length, is_training, drop_remainder):
  """Creates an `input_fn` closure to be passed to TPUEstimator."""
  """This is used to make the proper format of the prediction variable"""

  all_input_ids = []
  all_input_mask = []
  all_segment_ids = []
  all_label_ids = []

  for feature in features:
    all_input_ids.append(feature.input_ids)
    all_input_mask.append(feature.input_mask)
    all_segment_ids.append(feature.segment_ids)
    all_label_ids.append(feature.label_id)

  def input_fn(params):
    """The actual input function."""
    print(params)
    batch_size = 500

    num_examples = len(features)

    d = tf.data.Dataset.from_tensor_slices({
        "input_ids":
            tf.constant(
                all_input_ids, shape=[num_examples, seq_length],
                dtype=tf.int32),
        "input_mask":
            tf.constant(
                all_input_mask,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
        "segment_ids":
            tf.constant(
                all_segment_ids,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
        "label_ids":
            tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32),
    })

    if is_training:
      d = d.repeat()
      d = d.shuffle(buffer_size=100)

    d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)
    return d

  return input_fn

predict_examples = create_examples(X_test, 'test')
predict_features = run_classifier.convert_examples_to_features(predict_examples, label_list, MAX_SEQ_LENGTH, tokenizer)
"""predictions and acuracy"""
predict_input_fn = input_fn_builder(
    features=predict_features,
    seq_length=MAX_SEQ_LENGTH,
    is_training=False,
    drop_remainder=False)

result = estimator.predict(input_fn=predict_input_fn)
result
preds = []
for prediction in result:
      preds.append(np.argmax(prediction['probabilities']))
from sklearn.metrics import accuracy_score
print("Accuracy of BERT is:",accuracy_score(y_test,preds))
print(classification_report(y_test,preds))
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
#plotting confusion matrix
import matplotlib.pyplot as plt
import itertools
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='BERT_TF Confusion matrix',
                          cmap=plt.cm.YlOrRd):
    """
    See full source and example: 
    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html
    
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title,fontsize=17)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=0)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="right",
                 color="White" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()
cm= confusion_matrix(y_test,preds)
plot_confusion_matrix(cm, ['Negative','Neutral','Positive'])

import pickle as pk
file="/content/drive/MyDrive/Sentiment Analysis_Preskale/estimator.pkl"
estimator = pk.dump(estimator,open(file, 'w'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1085682164940175, 'fraction_numerical': 0.012366498032602586, 'mean_word_length': 4.2, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 26, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '314874', 'license': 'Apache-2.0', 'n_tokens_mistral': 4349, 'n_tokens_neox': 4145, 'n_words': 856, 'path': 'BERT_Pre_Trained_MULTI_CLASS.ipynb', 'repo_name': 'MOTURUPRAVEENBHARGAV/bert_models', 'size': '12453'}