<jupyter_start><jupyter_text>Initialization Check GPU<jupyter_code>!nvidia-smi<jupyter_output>Mon Aug 24 03:58:23 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.57       Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |
| N/A   52C    P8    31W / 149W |      0MiB / 11441MiB |      0%      Default |
|                               |                      |                 ERR! |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Install Tensorflow-GPU<jupyter_code>!pip install tensorflow-gpu
!pip install tensorflow_text<jupyter_output>Collecting tensorflow-gpu
[?25l  Downloading https://files.pythonhosted.org/packages/0f/11/763f55d3d15efd778ef24453f126e6c33635680e5a2bb346da3fab5997cb/tensorflow_gpu-2.3.0-cp36-cp36m-manylinux2010_x86_64.whl (320.4MB)
[K     |████████████████████████████████| 320.4MB 52kB/s 
[?25hRequirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu) (3.3.0)
Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu) (2.10.0)
Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu) (0.3.3)
Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu) (1.1.2)
Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu) (3.12.4)
Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorfl[...]<jupyter_text>Import Packages<jupyter_code># DATA PROCESSING
import numpy as np
import pandas as pd
import gzip # To utilize previously encoded data

# DEEP LEARNING
import tensorflow as tf 
from tensorflow import keras
from sklearn.model_selection import train_test_split
import tensorflow_hub as hub
import tensorflow_text

# PREPROCESSING
import re
import inflect
import string, unicodedata
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize

# DATA DISPLAY
import matplotlib.pyplot as plt
from tqdm import tqdm #For showing progress bars<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code># READ ORIGINAL DATA
df = pd.read_csv('training_data.csv', usecols=['Text', 'Sentiment'])
print(df.shape)
# SET RANDOM SEED FOR REPRODUCIBLE TRAIN/DEV/TEST SETS
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)

#tf.random.set_seed(RANDOM_SEED)
# SPLIT DATA
train, test, y_train, y_test =\
  train_test_split(
    df.Text, 
    df.Sentiment, 
    test_size=.1, 
    random_state=RANDOM_SEED)
# DOWNLOAD NATURAL LANGUAGE PREPROCESSING TOOLKITS
nltk.download('punkt') # for stemming 
nltk.download('stopwords') # for stopwords
porter = PorterStemmer() # create porter stemming algorithm object
# CHECK SHAPES AND DATA
print(train.shape)
print(test.shape)
train.head()<jupyter_output>(900000,)
(100000,)<jupyter_text>Encoding  Extract from previously encoded data Add training and test datasets to your drive by right clicking and selecting 'make a shortcut to your drive'. Afterwards, the output will prompt for an authentication password, which you can receive by allowing the file stream to access the drive in which the datasets are stored.<jupyter_code># from google.colab import drive
# drive.mount("/content/drive")
f=gzip.GzipFile('/content/drive/My Drive/train.npy.gz','r')
X_train = np.load(f)
X_train = np.array(X_train)
f2=gzip.GzipFile('/content/drive/My Drive/test.npy.gz','r')
X_test = np.load(f2)
X_test = np.array(X_test)<jupyter_output><empty_output><jupyter_text>Re-encode data (~3 to 4 hours)<jupyter_code># LOAD UNIVERAL SERIAL ENCODER
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3")

## OLD ENCODING ALGORITHM ##
# splitlist = np.array_split(trainlist,50)
# # GET EMBEDDINGS
# embedlist=[]
# j=0
# for i in splitlist:
#   s = embed(i)
#   for k in s:
#     embedlist.append(k)
#     j+=1
#     print(j)<jupyter_output><empty_output><jupyter_text>Encoding Training and Test Sets<jupyter_code>X_train = np.zeros((train.shape[0],512))
i=0
for r in tq.tqdm(trains):
  X_train[i] = tf.reshape(use(r),[-1]).numpy()
  i+=1
X_test = np.zeros((test.shape[0],512))
i=0
for r in tq.tqdm(tests):
  X_test[i] = tf.reshape(use(r),[-1]).numpy()
  i+=1<jupyter_output>100%|██████████| 100000/100000 [20:38<00:00, 80.74it/s]<jupyter_text>Check Encodings Check if shapes and values are correct: Expected (n,512) 32-bit float vectors for feature train/test arrays and (n,) for labels<jupyter_code>print(X_train.shape,X_test.shape,y_train.shape)
print(X_test)<jupyter_output>(900000, 512) (100000, 512) (900000,)
[[-0.04484706 -0.0484597  -0.02886226 ... -0.01150637  0.01021791
  -0.07051698]
 [ 0.01337283  0.0073399   0.03012324 ... -0.04162316  0.03671977
  -0.03627334]
 [ 0.06179673 -0.02251715  0.02605925 ...  0.0276723  -0.04356809
   0.05065024]
 ...
 [-0.00890851  0.01316289  0.01039192 ... -0.01220796 -0.0712463
   0.10207573]
 [-0.05828437 -0.00819871  0.02271808 ... -0.03066864 -0.02567255
   0.00929535]
 [-0.0683042   0.04698984  0.02167922 ... -0.05713771  0.00492626
   0.04706876]]<jupyter_text>Save Encodings and Compress<jupyter_code>f = gzip.GzipFile("train.npy.gz", "w")
np.save(file=f, arr=X_train)
f.close()
f2 = gzip.GzipFile("test.npy.gz","w")
np.save(file=f2, arr=X_test)
f2.close()

## Non-compressed saving creates 12GB in filesize
#np.savetxt('test.csv',X_train,delimiter=',')
#np.savetxt('train.csv',X_test,delimiter=',')<jupyter_output><empty_output><jupyter_text>Dimensionality Reshaping for Alternate Models<jupyter_code># from sklearn.preprocessing import OneHotEncoder
# y_train = OneHotEncoder(sparse=False).fit_transform(y_train.to_numpy().reshape(-1,1))
# y_test = OneHotEncoder(sparse=False).fit_transform(y_test.to_numpy().reshape(-1,1))

# maxpolxtr = X_train.reshape(-1, 1, 512)
# maxpolxte  = X_test.reshape(-1, 1, 512)
# maxpolytr = np.array(y_train).reshape(-1, 1, 1)
# maxpolyte = np.array(y_test).reshape(-1, 1, 1)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>model = keras.Sequential()

model.add(
  keras.layers.Dense(
    units=128,
    input_shape=(X_train.shape[1], ),
    activation='relu'
  )
)
model.add(
  keras.layers.Dropout(rate=0.5)
)

model.add(
  keras.layers.Dense(
    units=64,
    kernel_regularizer='l2',
    activation='relu'
  )
)
model.add(
  keras.layers.Dropout(rate=0.5)
)

# model.add(
#   keras.layers.Dense(
#     units=64,
#     kernel_regularizer=keras.regularizers.l2(0.001),
#     activation='relu'
#   )
# )
# model.add(
#   keras.layers.Dropout(rate=0.5)
# )

model.add(keras.layers.Dense(1, activation='sigmoid'))
model.compile(
    loss='binary_crossentropy', 
    optimizer=keras.optimizers.Adam(0.001),
    metrics=['accuracy']
)


# model = keras.Sequential([
#     keras.layers.GlobalMaxPooling1D(),
#     keras.layers.Dense(256,activation='relu'),
#     keras.layers.Dropout(0.3),
#     keras.layers.Dense(128,activation='relu'),
#     keras.layers.Dropout(0.5),
#     keras.layers.Dense(64,activation='relu'),
#     keras.layers.Dense(1, activation='sigmoid')
# ])
# model.compile(loss="binary_crossentropy", optimizer=keras.optimizers,Adam(0.01), metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>CNN<jupyter_code># model = tf.keras.Sequential([
#     tf.keras.layers.Embedding(vocab_size, 16, input_length=max_len),
#     tf.keras.layers.SeparableConv1D(16, 5, activation='relu', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same'),
#     tf.keras.layers.GlobalMaxPooling1D(),
#     tf.keras.layers.Dense(128, activation='relu'),
#     tf.keras.layers.Dropout(0.5),
#     tf.keras.layers.Dense(1, activation='sigmoid')
# ])

# model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(0.001), metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Bidirectional LSTM<jupyter_code># model = tf.keras.Sequential([
#     tf.keras.layers.Embedding(vocab_size, 16, input_length=max_len),
#     tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16, return_sequences=True)),
#     tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(16)),
#     tf.keras.layers.Dense(1, activation='sigmoid')
# ])
# model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(0.001), metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Early StoppingDetermines which epoch model starts to overfit and lose validation accuracy<jupyter_code>class ResetStatesCallback(tf.keras.callbacks.Callback):
    def on_epoch_begin(self, epoch, logs):
        self.model.reset_states()
reset_states = ResetStatesCallback()
early_stopping = tf.keras.callbacks.EarlyStopping(patience=50)
model_checkpoint = tf.keras.callbacks.ModelCheckpoint("maincheckpoint.h5", save_best_only=True)<jupyter_output><empty_output><jupyter_text>Train*   8-10 Epochs*   Default learning rate (0.001)*   16 Batch Size<jupyter_code>history = model.fit(
    X_train,y_train, 
    epochs=10, 
    batch_size=16, 
    validation_data=(X_test,y_test), 
    verbose=1, 
    shuffle=True
)
model.summary()<jupyter_output>Model: "sequential_11"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_24 (Dense)             (None, 128)               65664     
_________________________________________________________________
dropout_17 (Dropout)         (None, 128)               0         
_________________________________________________________________
dense_25 (Dense)             (None, 64)                8256      
_________________________________________________________________
dropout_18 (Dropout)         (None, 64)                0         
_________________________________________________________________
dense_26 (Dense)             (None, 1)                 65        
=================================================================
Total params: 73,985
Trainable params: 73,985
Non-trainable params: 0
_________________________________________________[...]<jupyter_text>Plot Model Metrics<jupyter_code>plt.plot(history.history['loss'], label='train loss')
plt.plot(history.history['val_loss'], label='val loss')
plt.xlabel("epoch")
plt.ylabel("accuracy")
plt.legend()
plt.plot(history.history['accuracy'], label='train accuracy')
plt.plot(history.history['val_accuracy'], label='val accuracy')
plt.xlabel("epoch")
plt.ylabel("accuracy")
plt.legend()<jupyter_output><empty_output><jupyter_text>Evaluation on Test Data<jupyter_code>model.evaluate(X_test, y_test)
# Load the saved encoded dataset
f3=gzip.GzipFile('/content/drive/My Drive/judgement.npy.gz','r')
X_test = np.load(f)
X_test = np.array(X_test)
f4=gzip.GzipFile('/content/drive/My Drive/judgement2.npy.gz','r') # We split up our data to quickly encode it on multiple computers
X_test2 = np.load(f)
X_test2 = np.array(X_test2)
# Concatentate Arrays Row-Wise
X_test = np.concatenate((X_test,X_test2))
# Load the original dataset
testingdata = pd.read_csv('contestant_judgment.csv')
# Predict sentiment from model
pred=model.predict(X_test)
# Write into csv
testingdata['Sentiment']=pred
testingdata.to_csv('predictions.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1702074481379655, 'fraction_numerical': 0.06348412896775806, 'mean_word_length': 4.001666666666667, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 68, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '26533', 'license': 'MIT', 'n_tokens_mistral': 4757, 'n_tokens_neox': 4237, 'n_words': 718, 'path': 'Main.ipynb', 'repo_name': 'AndrewZL/SentimentAnalysis-IgnitionHacks', 'size': '12003'}