<jupyter_start><jupyter_text>Importing Data From Kaggle<jupyter_code>from google.colab import files
files.upload()
! mkdir -p ~/.kaggle
!cp kaggle.json ~/.kaggle/
!pip install kaggle
!chmod 600 /root/.kaggle/kaggle.json
!kaggle datasets list
!kaggle competitions download -c Sentiment-Analysis-of-tweets
!unzip /content/train.txt.zip<jupyter_output>Archive:  /content/train.txt.zip
  inflating: train.txt<jupyter_text>Installing Requirements<jupyter_code>!nvidia-smi
!pip install tensorflow-gpu
!pip install --upgrade grpcio
!pip install tqdm
!pip install bert-for-tf2
!pip install sentencepiece<jupyter_output>Collecting sentencepiece
[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)

[K     |▎                               | 10kB 29.3MB/s eta 0:00:01
[K     |▋                               | 20kB 2.1MB/s eta 0:00:01
[K     |█                               | 30kB 2.8MB/s eta 0:00:01
[K     |█▏                              | 40kB 3.1MB/s eta 0:00:01
[K     |█▌                              | 51kB 2.5MB/s eta 0:00:01
[K     |█▉                              | 61kB 2.8MB/s eta 0:00:01
[K     |██▏                             | 71kB 3.1MB/s eta 0:00:01
[K     |██▍                             | 81kB 3.4MB/s eta 0:00:01
[K     |██▊                             | 92kB 3.6MB/s eta 0:00:01
[K     |███                             | 102kB 3.4MB/s eta 0:00:01
[K     |███▍                            | 112kB 3.4MB/s eta 0:00:01
[K     |███▋                     [...]<jupyter_text>Importing Libraries<jupyter_code>import os
from tqdm import tqdm

import pandas as pd
import numpy as np

import tensorflow as tf
from tensorflow import keras

import bert
from bert import BertModelLayer
from bert.loader import StockBertConfig, map_stock_config_to_params, load_stock_weights
from bert.tokenization.bert_tokenization import FullTokenizer

import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
from matplotlib import rc

from sklearn.metrics import confusion_matrix, classification_report

%matplotlib inline
%config InlineBackend.figure_format='retina'

sns.set(style='whitegrid', palette='muted', font_scale=1.2)



rcParams['figure.figsize'] = 12, 8

RANDOM_SEED = 42

np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)<jupyter_output>/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm<jupyter_text>Data Visulisation<jupyter_code>train = pd.read_csv("/content/train.txt")
test = pd.read_csv("/content/test_samples.txt")
train.head()
train.shape
train.describe()
train.info()
plt.figure(figsize = (10, 7))
sns.countplot(data = train, x = 'sentiment')
train.sentiment.value_counts()
test['sentiment'] = "positive"<jupyter_output><empty_output><jupyter_text>importing BERT pretrained model<jupyter_code>!wget https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip
!unzip uncased_L-12_H-768_A-12.zip
os.makedirs("model", exist_ok=True)
!mv uncased_L-12_H-768_A-12/ model
bert_model_name="uncased_L-12_H-768_A-12"

bert_ckpt_dir = os.path.join("model/", bert_model_name)
bert_ckpt_file = os.path.join(bert_ckpt_dir, "bert_model.ckpt")
bert_config_file = os.path.join(bert_ckpt_dir, "bert_config.json")<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>class SentimentAnalysisData:
  DATA_COLUMN = "tweet_text"
  LABEL_COLUMN = "sentiment"

  def __init__(self, train, test, tokenizer: FullTokenizer, classes, max_seq_len=192):
    self.tokenizer = tokenizer
    self.max_seq_len = 0
    self.classes = classes
    
    ((self.train_x, self.train_y), (self.test_x, self.test_y)) = map(self._prepare, [train, test])

    print("max seq_len", self.max_seq_len)
    self.max_seq_len = min(self.max_seq_len, max_seq_len)
    self.train_x, self.test_x = map(self._pad, [self.train_x, self.test_x])

  def _prepare(self, df):
    x, y = [], []
    
    for _, row in tqdm(df.iterrows()):
      text, label = row[SentimentAnalysisData.DATA_COLUMN], row[SentimentAnalysisData.LABEL_COLUMN]
      tokens = self.tokenizer.tokenize(text)
      tokens = ["[CLS]"] + tokens + ["[SEP]"]
      token_ids = self.tokenizer.convert_tokens_to_ids(tokens)
      self.max_seq_len = max(self.max_seq_len, len(token_ids))
      x.append(token_ids)
      y.append(self.classes.index(label))

    return np.array(x), np.array(y)

  def _pad(self, ids):
    x = []
    for input_ids in ids:
      input_ids = input_ids[:min(len(input_ids), self.max_seq_len - 2)]
      input_ids = input_ids + [0] * (self.max_seq_len - len(input_ids))
      x.append(np.array(input_ids))
    return np.array(x)<jupyter_output><empty_output><jupyter_text>Creating Bert model<jupyter_code>tokenizer = FullTokenizer(vocab_file=os.path.join(bert_ckpt_dir, "vocab.txt"))
def create_model(max_seq_len, bert_ckpt_file):

  with tf.io.gfile.GFile(bert_config_file, "r") as reader:
      bc = StockBertConfig.from_json_string(reader.read())
      bert_params = map_stock_config_to_params(bc)
      bert_params.adapter_size = None
      bert = BertModelLayer.from_params(bert_params, name="bert")
        
  input_ids = keras.layers.Input(shape=(max_seq_len, ), dtype='int32', name="input_ids")
  bert_output = bert(input_ids)
  
  print("bert shape", bert_output.shape)
  #cls_out = keras.layers.SpatialDropout1D(0.3)(bert_output)
  #cls_out = keras.layers.Bidirectional(keras.layers.LSTM(64, return_sequences=True))(bert_output)
  #cls_out = keras.layers.SpatialDropout1D(0.5)(cls_out)
  #cls_out = keras.layers.Bidirectional(keras.layers.LSTM(64, return_sequences=True))(cls_out)
  #cls_out = keras.layers.concatenate([keras.layers.GlobalMaxPooling1D()(cls_out),keras.layers.GlobalAveragePooling1D()(cls_out),])
  cls_out = keras.layers.Lambda(lambda seq: seq[:, 0, :])(bert_output)

  

  cls_out = keras.layers.Dropout(0.5)(cls_out)
  #cls_out = keras.layers.Flatten()(cls_out)
  logits = keras.layers.Dense(units=768, activation="tanh")(cls_out)
  logits = keras.layers.Dropout(0.5)(logits)
  #logits = keras.layers.Dense(units=512, activation="tanh")(cls_out)
  #logits = keras.layers.Dropout(0.5)(logits)
  logits = keras.layers.Dense(units=len(classes), activation="softmax")(logits)

  model = keras.Model(inputs=input_ids, outputs=logits)
  model.build(input_shape=(None, max_seq_len))

  load_stock_weights(bert, bert_ckpt_file)
        
  return model
classes = train.sentiment.unique().tolist()

data = SentimentAnalysisData(train, test, tokenizer, classes, max_seq_len=128)
classes
data.train_x.shape
data.train_x[0]
data.train_y[0]
data.max_seq_len
model = create_model(data.max_seq_len, bert_ckpt_file)
model.summary()<jupyter_output>Model: "model_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_ids (InputLayer)       [(None, 128)]             0         
_________________________________________________________________
bert (BertModelLayer)        (None, 128, 768)          108890112 
_________________________________________________________________
lambda_1 (Lambda)            (None, 768)               0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 768)               0         
_________________________________________________________________
dense_2 (Dense)              (None, 768)               590592    
_________________________________________________________________
dropout_3 (Dropout)          (None, 768)               0         
___________________________________________________________[...]<jupyter_text>Training model<jupyter_code>model.compile(
  optimizer=keras.optimizers.Adam(1e-5),
  loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
  metrics=[keras.metrics.SparseCategoricalAccuracy(name="acc")]
)
history = model.fit(
  x=data.train_x, 
  y=data.train_y,
  validation_split=0.1,
  batch_size=16,
  shuffle=True,
  epochs=epochs,
)<jupyter_output>Epoch 1/3
1208/1208 [==============================] - 550s 455ms/step - loss: 0.9507 - acc: 0.5787 - val_loss: 0.8597 - val_acc: 0.6865
Epoch 2/3
1208/1208 [==============================] - 547s 453ms/step - loss: 0.8509 - acc: 0.6902 - val_loss: 0.8064 - val_acc: 0.7317
Epoch 3/3
1208/1208 [==============================] - 547s 453ms/step - loss: 0.8058 - acc: 0.7398 - val_loss: 0.8307 - val_acc: 0.7108<jupyter_text>model evluation<jupyter_code>ax = plt.figure().gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))

ax.plot(history.history['loss'])
ax.plot(history.history['val_loss'])
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['train', 'test'])
plt.title('Loss over training epochs')
plt.show();
ax = plt.figure().gca()
ax.xaxis.set_major_locator(MaxNLocator(integer=True))

ax.plot(history.history['acc'])
ax.plot(history.history['val_acc'])
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['train', 'test'])
plt.title('Accuracy over training epochs')
plt.show();<jupyter_output><empty_output><jupyter_text>prediction<jupyter_code>y_pred = model.predict(data.test_x).argmax(axis=1)
y_pred.shape
predictions = []
for i in range(len(y_pred)):
  predictions.append(classes[y_pred[i]])

predictions = np.array(predictions)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>submissions = pd.DataFrame({"tweet_id": test['tweet_id'],
    "sentiment": predictions})
submissions.to_csv("my_submission.csv", index=False, header=True)
submissions.info()
submissions['sentiment'].describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14213454729387962, 'fraction_numerical': 0.041375821952453214, 'mean_word_length': 4.211386399578282, 'pattern_counts': {'":': 2, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '80281', 'license': 'MIT', 'n_tokens_mistral': 3889, 'n_tokens_neox': 3533, 'n_words': 560, 'path': 'BERT_SENTIMENT_ANALYSIS_TENSORFLOW.ipynb', 'repo_name': 'shreenath2001/Sentiment_Analysis_of_Twitter', 'size': '9885'}