<jupyter_start><jupyter_text>Glove EmbeddingEmbeding_D = 50 Seq2Seq attention  Download Dataset<jupyter_code>!curl --output dataset.zip https://archive.ics.uci.edu/ml/machine-learning-databases/00462/drugsCom_raw.zip
!unzip dataset.zip
# !rm dataset.zip
import pandas as pd
import html
from nltk.tokenize import word_tokenize,RegexpTokenizer
from nltk.corpus import stopwords 
import nltk
nltk.download('stopwords')
nltk.download('punkt')
import gensim
import numpy as np
from keras.utils.np_utils import to_categorical
from sklearn.model_selection import train_test_split
from keras.preprocessing.sequence import pad_sequences
import h5py<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.<jupyter_text>Loading DataTokenization,removing stopwords<jupyter_code># Loading Data

def load_dataset(filename,classes=0):
    df = pd.read_csv(filename, sep='\t')
    df.drop(df.columns[0], axis=1, inplace=True)
    df.drop(['drugName', 'condition', 'date', 'usefulCount'], axis=1, inplace=True)
    df['review'] = df['review'].apply(lambda x: x[1:-1])
    df['review'] = df['review'].apply(html.unescape)
    df['rating'] = df['rating'].astype('int64')
    df['label'] = 0

    print(df.groupby('rating').size())


    label_range = [0,4,6,10]
    if(classes == 2):
        label_range = [0,5,10]
    for i,r in enumerate(label_range):
        if(i == 0):
            continue
        df.label[(df['rating'] > label_range[i-1]) & (df['rating'] <=  label_range[i])] = i-1
    return df

# df = load_dataset("drugsComTrain_raw.tsv")
# df_test = load_dataset("drugsComTest_raw.tsv")
sequence_length = 100
stop_words = set(stopwords.words('english'))  
tokenizer = RegexpTokenizer(r'\w+')
def func(row):
    l = tokenizer.tokenize(row['review'])
    s = [i.lower() for i in l if i not in stop_words]
    return s
# df['processed'] = df.apply (lambda row: func(row), axis=1)<jupyter_output><empty_output><jupyter_text>Embedings  Glove embedings  Download Glove embeddings<jupyter_code>!wget "http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip"
!unzip glove.6B.zip
!rm glove.6B.zip<jupyter_output>--2019-11-24 16:17:36--  http://downloads.cs.stanford.edu/nlp/data/glove.6B.zip
Resolving downloads.cs.stanford.edu (downloads.cs.stanford.edu)... 192.168.127.12
Connecting to downloads.cs.stanford.edu (downloads.cs.stanford.edu)|192.168.127.12|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 862182613 (822M) [application/zip]
Saving to: ‘glove.6B.zip’

glove.6B.zip        100%[===================>] 822.24M  1.85MB/s    in 6m 30s  

2019-11-24 16:24:06 (2.11 MB/s) - ‘glove.6B.zip’ saved [862182613/862182613]

Archive:  glove.6B.zip
  inflating: glove.6B.50d.txt        
  inflating: glove.6B.100d.txt       
  inflating: glove.6B.200d.txt       
  inflating: glove.6B.300d.txt<jupyter_text>Util functions<jupyter_code>def embed(processed, w2v_dict, default_vec):
    vectors = []
    for w in processed:
        if(w in w2v_dict):
            vectors.append(w2v_dict[w])
        else:
            vectors.append(default_vec)
    vectors = np.array(vectors)
    return vectors

def embed_using_glove(df, embed_dim=50):
    # Load Glove embeddings.
    GLOVE_FILE = 'glove.6B.' + str(embed_dim) + 'd.txt'
    # Get number of vectors and hidden dimensions
    with open(GLOVE_FILE, 'r') as f:
        for i, line in enumerate(f):
            pass
    n_vec = i + 1
    hidden_dim = len(line.split(' ')) - 1
    # Find the average of all embeddings to be assigned to tokens not in the embeddings.
    # Create a word to vector dictionary for embedding.
    avg_vec = np.zeros((hidden_dim), dtype=np.float32)
    w2v_dict = {}
    count = 0
    with open(GLOVE_FILE, 'r') as f:
        for line in f:
            w2v_dict[line.split(' ')[:1][0]] = np.array([float(n) for n in line.split(' ')[1:]], dtype=np.float32)
            avg_vec += np.array([float(n) for n in line.split(' ')[1:]], dtype=np.float32)
            count+=1
    avg_vec/=count

    df['vectors'] = df.apply(lambda row: embed(row['processed'], w2v_dict, avg_vec), axis=1)
    return avg_vec<jupyter_output><empty_output><jupyter_text>Convert tokens in to Glove embeddings and save them to file.<jupyter_code># embed_using_glove(df_train, 50)
#df_train.to_pickle("train_glove_50.pckl")
# embed_using_glove(df_test, 50)
# df_test.to_pickle("test_glove_50.pckl")<jupyter_output><empty_output><jupyter_text>Word2Vec embeddings  Download the Google word2vec embeddings<jupyter_code>!curl --output word2vec.bin.gz https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz
!gunzip word2vec.bin.gz<jupyter_output>% Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 1570M  100 1570M    0     0  44.6M      0  0:00:35  0:00:35 --:--:-- 45.8M<jupyter_text>Load the word2vec embeddings into memory<jupyter_code><jupyter_output><empty_output><jupyter_text>Util funcitons<jupyter_code>def embed_w2v(processed, w2v_dict, default_vec):
    vectors = []
    for w in processed:
        if(w in w2v_dict):
            vectors.append(w2v_dict[w])
        else:
            vectors.append(default_vec)
    vectors = np.array(vectors)
    return vectors<jupyter_output><empty_output><jupyter_text>Get equal-class versions of train and test dataset<jupyter_code># df_test_w2v['label'].value_counts()
# df_test_w2v
# df_train_w2v = pd.concat([df_train[df_train['label']==0].sample(n=4829).reset_index(drop=True),df_train[df_train['label']==1].sample(n=4829).reset_index(drop=True),df_train[df_train['label']==2].sample(n=4829).reset_index(drop=True)],ignore_index=True)
# df_test_w2v = pd.concat([df_test[df_test['label']==0].sample(n=4829).reset_index(drop=True),df_test[df_test['label']==1].sample(n=4829).reset_index(drop=True),df_test[df_test['label']==2].sample(n=4829).reset_index(drop=True)],ignore_index=True)<jupyter_output><empty_output><jupyter_text>Convert the tokens into Word2Vec embeddings<jupyter_code># w2v_avg = np.average(w2v_model.vectors, axis=0)
# df_train_w2v['vectors'] = df_train_w2v.apply(lambda row: embed_w2v(row['processed'], w2v_model, w2v_avg), axis=1)
# df_test_w2v['vectors'] = df_test_w2v.apply(lambda row: embed_w2v(row['processed'], w2v_model, w2v_avg), axis=1)<jupyter_output><empty_output><jupyter_text>Save the Embeddings to file<jupyter_code># df_train_w2v.to_pickle("train_w2v.pckl")
# df_test_w2v.to_pickle("test_w2v.pckl")
def embeding_using_w2v(df):
    from gensim.models import KeyedVectors
    w2v_model = KeyedVectors.load_word2vec_format('word2vec.bin', binary=True)
    w2v_avg = np.average(w2v_model.vectors, axis=0)

    df['vectors'] = df.apply(lambda row: embed_w2v(row['processed'], w2v_model, w2v_avg), axis=1)
    df['vectors'] = df.apply(lambda row: embed_w2v(row['processed'], w2v_model, w2v_avg), axis=1)    
    return w2v_avg<jupyter_output><empty_output><jupyter_text>Elmo embeddings  Install allennlp<jupyter_code>!pip install allennlp<jupyter_output>Collecting allennlp
[?25l  Downloading https://files.pythonhosted.org/packages/bb/bb/041115d8bad1447080e5d1e30097c95e4b66e36074277afce8620a61cee3/allennlp-0.9.0-py3-none-any.whl (7.6MB)
[K     |████████████████████████████████| 7.6MB 4.7MB/s 
[?25hRequirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from allennlp) (3.2.5)
Collecting jsonpickle
  Downloading https://files.pythonhosted.org/packages/07/07/c157520a3ebd166c8c24c6ae0ecae7c3968eb4653ff0e5af369bb82f004d/jsonpickle-1.2-py2.py3-none-any.whl
Requirement already satisfied: editdistance in /usr/local/lib/python3.6/dist-packages (from allennlp) (0.5.3)
Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.3.2)
Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.6/dist-packages (from allennlp) (2018.9)
Requirement already satisfied: flask>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from allennlp) (1.1.1)
Collecting word2number>=1.1
  Down[...]<jupyter_text>Imports<jupyter_code>from allennlp.commands.elmo import ElmoEmbedder
import sys<jupyter_output><empty_output><jupyter_text>Util functions<jupyter_code>avg_elmo_embedding = ['1']
def create_elmo_embeddings(elmo, documents, max_sentences = 1000):
    num_sentences = min(max_sentences, len(documents)) if max_sentences > 0 else len(documents)
    print("\n\n:: Lookup of " + str(num_sentences) + " ELMo representations. This takes a while ::")
    embeddings = []
    tokens = documents['processed'].to_numpy()
    
    documentIdx = 0
    for elmo_embedding in elmo.embed_sentences(tokens):  
        document = documents.iloc[documentIdx]
        # Average the 3 layers returned from ELMo
        avg_elmo_embedding = np.average(elmo_embedding, axis=0)             
        embeddings.append(avg_elmo_embedding)        
        
        # Some progress info
        documentIdx += 1
        percent = 100.0 * documentIdx / num_sentences
        line = '[{0}{1}]'.format('=' * int(percent / 2), ' ' * (50 - int(percent / 2)))
        status = '\r{0:3.0f}%{1} {2:3d}/{3:3d} sentences'
        sys.stdout.write(status.format(percent, line, documentIdx, num_sentences))
        
        if max_sentences > 0 and documentIdx >= max_sentences:
            break
            
    return embeddings<jupyter_output><empty_output><jupyter_text>Get smaller versions of train and test dataset<jupyter_code># df_train_elmo = df_train.sample(n=int(df_train.shape[0]/20)).reset_index(drop=True)
# df_test_elmo = df_test.sample(n=int(df_test.shape[0]/20)).reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Get ELMo embeddings<jupyter_code># elmo = ElmoEmbedder(cuda_device=0) #Set cuda_device to the ID of your GPU if you have one
# df_train_elmo['elmo_embeddings'] = create_elmo_embeddings(elmo, df_train_elmo, -1)
# df_test_elmo['elmo_embeddings'] = create_elmo_embeddings(elmo, df_test_elmo, -1)
def embeding_using_elmo(df):
    elmo = ElmoEmbedder(cuda_device=0) #Set cuda_device to the ID of your GPU if you have one
    df['vectors'] = create_elmo_embeddings(elmo, df, -1)
    return np.full_like(range(1024),0.0)<jupyter_output><empty_output><jupyter_text>PaddingSequence length = 100<jupyter_code>sequence_length = 100
num_classes = 3
def v2w(vec,avg_vec,w2v_dict):
    if((vec==avg_vec).all()):
        return 'UNK'
    if(np.all(vec == 0)):
        return 'None'
    for k in w2v_dict:
        if((w2v_dict[k]==vec).all()):
            return k
    return 'OOV'<jupyter_output><empty_output><jupyter_text>GetData<jupyter_code>def getData(normalize = False,k = -1,k_test = 1000,classes=3,embed = 'glove'):
    df_train = load_dataset("drugsComTrain_raw.tsv",classes)
    df_test = load_dataset("drugsComTest_raw.tsv",classes)
    
    print(df_train.groupby('label').size())
    print(df_test.groupby('label').size())


    if(normalize):
        mn = df_train.groupby('label').size().min()
        if(classes == 3):
            df_train = pd.concat([df_train[df_train['label']==0].sample(n=mn).reset_index(drop=True),df_train[df_train['label']==1].sample(n=mn).reset_index(drop=True),df_train[df_train['label']==2].sample(n=mn).reset_index(drop=True)],ignore_index=True)
        else:
            df_train = pd.concat([df_train[df_train['label']==0].sample(n=mn).reset_index(drop=True),df_train[df_train['label']==1].sample(n=mn).reset_index(drop=True)],ignore_index=True)
        mn = df_test.groupby('label').size().min()
        if(classes == 3):
            df_test = pd.concat([df_test[df_test['label']==0].sample(n=mn).reset_index(drop=True),df_test[df_test['label']==1].sample(n=mn).reset_index(drop=True),df_test[df_test['label']==2].sample(n=mn).reset_index(drop=True)],ignore_index=True)
        else:
            df_test = pd.concat([df_test[df_test['label']==0].sample(n=mn).reset_index(drop=True),df_test[df_test['label']==1].sample(n=mn).reset_index(drop=True)],ignore_index=True)

    if(k!=-1):
        df_train = df_train.sample(n=k).reset_index(drop=True)

    if(k_test!=-1):
        df_test = df_test.sample(n=k_test).reset_index(drop=True)

    df_train['processed'] = df_train.apply (lambda row: func(row), axis=1)
    df_test['processed'] = df_test.apply (lambda row: func(row), axis=1)

    if(embed == 'glove'):
        w2v_avg = embed_using_glove(df_train, 50)
        w2v_avg = embed_using_glove(df_test, 50)

    if(embed == 'w2v'):
        w2v_avg = embeding_using_w2v(df_train)
        w2v_avg = embeding_using_w2v(df_test)

    if(embed == 'elmo'):
        w2v_avg = embeding_using_elmo(df_train)
        w2v_avg = embeding_using_elmo(df_test)

    # df_train['vectors'] = df_train.apply (lambda row: foo(row['processed']), axis=1)
    # df_test['vectors'] = df_test.apply (lambda row: foo(row['processed']), axis=1)

    train_data = pad_sequences(df_train['vectors'], maxlen=sequence_length,dtype='float32')
    train_labels = pd.get_dummies(df_train['label']).to_numpy().reshape(-1,1,classes)

    test_data = pad_sequences(df_test['vectors'], maxlen=sequence_length,dtype='float32')
    test_labels = pd.get_dummies(df_test['label']).to_numpy().reshape(-1,1,classes)

    _GO=np.array([[range(classes)]]*train_labels.shape[0])
    _GO_test=np.array([[range(classes)]]*test_labels.shape[0])
    _GO.shape

    print(df_train.groupby('label').size())
    print(df_test.groupby('label').size())

    xTrain = {'ex':train_data,'dx':_GO}
    xTest = {'ex':test_data,'dx':_GO_test}
    print(train_data.shape,test_data.shape)

    return xTrain,xTest,train_labels,test_labels<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from tensorflow.keras.layers import Input,GRU,Attention,Bidirectional,Concatenate,RepeatVector,dot,Activation,concatenate,Dense,TimeDistributed
from tensorflow.keras.models import Model
def seq2seq(embed,classes=3):

    if(embed == 'glove'):
        EMBEDDING_D = 50
    if(embed == 'w2v'):
        EMBEDDING_D = 300
    if(embed == 'elmo'):
        EMBEDDING_D = 1024

    encoder_inputs = Input(batch_shape=(None, sequence_length, EMBEDDING_D), name='ex')
    decoder_inputs = Input(batch_shape=(None, 1, classes), name='dx')

    # Encoder network
    encoder_gru = Bidirectional(GRU(64, return_sequences=True, return_state=True, name='encoder_gru'),merge_mode='concat')
    encoder_out, state_h,state_c = encoder_gru(encoder_inputs)
    encoder_state = [state_h,state_c]
    print(encoder_out.shape)
    print(state_h.shape)
    print(state_c.shape)


    # Decoder network
    decoder_gru = Bidirectional(GRU(64, return_sequences=True, return_state=True, name='decoder_gru'),merge_mode='concat')
    decoder_out, _,_ = decoder_gru(decoder_inputs, initial_state=encoder_state)

    print(decoder_out.shape)

    #Attention Layer
    attn_out = dot([decoder_out, encoder_out], axes=[2, 2])

    print(attn_out.shape)

    attention = Activation('softmax')(attn_out)

    context = dot([attention, encoder_out], axes=[2,1])

    decoder_concat_input = concatenate([context, decoder_out])

    # decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_out, attn_out])

    dense = Dense(classes, activation='softmax', name='softmax_layer')
    dense_time = TimeDistributed(dense, name='time_distributed_layer')
    decoder_pred = dense_time(decoder_concat_input)

    full_model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=decoder_pred)
    full_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
    attention_model = Model(inputs=[encoder_inputs, decoder_inputs], outputs=attention)
    attention_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

    return full_model,attention_model<jupyter_output><empty_output><jupyter_text>Training Glove<jupyter_code>xTrain,xVal,yTrain,yVal = getData(normalize=True,k=10000)
full_model,attention_model = seq2seq(embed='glove')
training = full_model.fit(x=xTrain, y = yTrain, epochs=15, batch_size=128,validation_split=0.1)
from sklearn.metrics import classification_report
import numpy as np

Y_test = np.argmax(yVal, axis=2).reshape((-1)) # Convert one-hot to index
y_pred = full_model.predict(xVal)
y_pred = y_pred.argmax(axis=2).reshape((-1))
print(classification_report(Y_test, y_pred))
from sklearn import metrics
cm=metrics.confusion_matrix(Y_test,y_pred)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
from sklearn import metrics
cm=metrics.confusion_matrix(Y_test,y_pred)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
y_pred = attention_model.predict(xVal)
y_pred_class = full_model.predict(xVal)

req = np.argmax(y_pred,axis = 2).reshape((-1))
classes = np.argmax(y_pred_class,axis=2).reshape(-1)
imp_vecs = xVal
pos_words = []
neg_words = []
neu_words = []
for i,vecs in enumerate(imp_vecs):
    if(classes[i] == 0):
        neg_words.append({'word':v2w(vecs)})
    if(classes[i] == 1):
        neu_words.append({'word':v2w(vecs)})
    if(classes[i] == 2):
        pos_words.append({'word':v2w(vecs)})

neg_words = pd.DataFrame(neg_words)
neu_words = pd.DataFrame(neu_words)
pos_words = pd.DataFrame(pos_words)

print(neg_words['word'].value_counts())
print(neu_words['word'].value_counts())
print(pos_words['word'].value_counts())<jupyter_output><empty_output><jupyter_text>Word2Vec<jupyter_code>xTrain,xVal,yTrain,yVal = getData(normalize=True,k=10000,embed='w2v')

full_model,attention_model = seq2seq(embed='w2v')
training = full_model.fit(x=xTrain, y = yTrain, epochs=15, batch_size=128,validation_split=0.1)
from sklearn.metrics import classification_report
import numpy as np

Y_test = np.argmax(yVal, axis=2).reshape((-1)) # Convert one-hot to index
print(Y_test.shape)
y_pred = full_model.predict(xVal)
print(y_pred.shape)
y_pred = y_pred.argmax(axis=2).reshape((-1))
print(y_pred.shape)
print(classification_report(Y_test, y_pred))
from sklearn import metrics
cm=metrics.confusion_matrix(Y_test,y_pred)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
# y_pred = attention_model.predict(xVal)
# y_pred_class = full_model.predict(xVal)

# req = np.argmax(y_pred,axis = 2).reshape((-1))
# classes = np.argmax(y_pred_class,axis=2).reshape(-1)
# imp_vecs = xVal
# pos_words = []
# neg_words = []
# neu_words = []
# for i,vecs in enumerate(imp_vecs):
#     if(classes[i] == 0):
#         neg_words.append({'word':v2w(vecs,avg_vec,)})
#     if(classes[i] == 1):
#         neu_words.append({'word':v2w(vecs)})
#     if(classes[i] == 2):
#         pos_words.append({'word':v2w(vecs)})

# neg_words = pd.DataFrame(neg_words)
# neu_words = pd.DataFrame(neu_words)
# pos_words = pd.DataFrame(pos_words)

# print(neg_words['word'].value_counts())
# print(neu_words['word'].value_counts())
# print(pos_words['word'].value_counts())<jupyter_output><empty_output><jupyter_text>Elmo<jupyter_code>xTrain,xVal,yTrain,yVal = getData(normalize=True,k=5000,embed='elmo')
full_model,attention_model = seq2seq(embed='elmo')
training = full_model.fit(x=xTrain, y = yTrain, epochs=15, batch_size=128,validation_split=0.1)
from sklearn.metrics import classification_report
import numpy as np

Y_test = np.argmax(yVal, axis=2).reshape((-1)) # Convert one-hot to index
y_pred = full_model.predict(xVal)
y_pred = y_pred.argmax(axis=2).reshape((-1))
print(classification_report(Y_test, y_pred))
from sklearn import metrics
cm=metrics.confusion_matrix(Y_test,y_pred)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
# y_pred = attention_model.predict(xVal)
# y_pred_class = full_model.predict(xVal)

# req = np.argmax(y_pred,axis = 2).reshape((-1))
# classes = np.argmax(y_pred_class,axis=2).reshape(-1)
# imp_vecs = xVal
# pos_words = []
# neg_words = []
# neu_words = []
# for i,vecs in enumerate(imp_vecs):
#     if(classes[i] == 0):
#         neg_words.append({'word':v2w(vecs)})
#     if(classes[i] == 1):
#         neu_words.append({'word':v2w(vecs)})
#     if(classes[i] == 2):
#         pos_words.append({'word':v2w(vecs)})

# neg_words = pd.DataFrame(neg_words)
# neu_words = pd.DataFrame(neu_words)
# pos_words = pd.DataFrame(pos_words)

# print(neg_words['word'].value_counts())
# print(neu_words['word'].value_counts())
# print(pos_words['word'].value_counts())
xTrain,xVal,yTrain,yVal = getData(normalize=True,k=5000,embed='elmo',classes=2)
full_model,attention_model = seq2seq(embed='elmo',classes=2)
training = full_model.fit(x=xTrain, y = yTrain, epochs=15, batch_size=128,validation_split=0.1)
from sklearn.metrics import classification_report
import numpy as np

Y_test = np.argmax(yVal, axis=2).reshape((-1)) # Convert one-hot to index
y_pred = full_model.predict(xVal)
y_pred = y_pred.argmax(axis=2).reshape((-1))
print(classification_report(Y_test, y_pred))
from sklearn import metrics
cm=metrics.confusion_matrix(Y_test,y_pred)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','positive'],
                  columns = ['negative','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
t<jupyter_output><empty_output><jupyter_text>Training General<jupyter_code>xTrain,xVal,yTrain,yVal = getData(k=5000)
full_model,attention_model = seq2seq()
training=full_model.fit(x=xTrain, y=yTrain,
  epochs=10,
  validation_split=0.1,
  shuffle=True
)
type(training)
full_model.evaluate(xVal,
  y=yVal)
y_pred = attention_model.predict(xVal)
y_pred_class = full_model.predict(xVal)
req = np.argmax(y_pred,axis = 2).reshape((-1))
classes = np.argmax(y_pred_class,axis=2).reshape(-1)
imp_vecs = xVal
def v2w(vec):
    if((vec==avg_vec).all()):
        return 'UNK'
    if(np.prod(vec) == 0):
        return 'None'
    for k in w2v_dict:
        if((w2v_dict[k]==vec).all()):
            return k
    return 'OOV'
pos_words = []
neg_words = []
neu_words = []
for i,vecs in enumerate(imp_vecs):
    if(classes[i] == 0):
        neg_words.append({'word':v2w(vecs)})
    if(classes[i] == 1):
        neu_words.append({'word':v2w(vecs)})
    if(classes[i] == 2):
        pos_words.append({'word':v2w(vecs)})
neg_words = pd.DataFrame(neg_words)
neu_words = pd.DataFrame(neu_words)
pos_words = pd.DataFrame(pos_words)
print(neg_words['word'].value_counts())
print(neu_words['word'].value_counts())
print(pos_words['word'].value_counts())
y_pred_class = full_model.predict({
    'ex':xVal,
    'dx':_GO
  })

y_test = np.argmax(yVal,axis=2)
y_pred_class = np.argmax(y_pred_class,axis=2)
y_test=y_test.reshape((-1))
y_pred_class=y_pred_class.reshape((-1))
from sklearn import metrics

cm=metrics.confusion_matrix(y_test[:k],y_pred_class)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)

import tensorflow as tf
import keras

config = tf.ConfigProto( device_count = {'GPU': 1} ) 
sess = tf.Session(config=config) 
keras.backend.set_session(sess)

# from tensorflow.python.client import device_lib

# print(device_lib.list_local_devices())<jupyter_output><empty_output><jupyter_text>Normalized Data<jupyter_code># sequence_length = 100
# num_classes = 3
# data = df['vectors'].to_numpy() #,np.array(df['r_label'])
# data = pad_sequences(df['vectors'], maxlen=sequence_length,dtype='float32')
# # labels = to_categorical(labels,num_classes=3)
# # for l in labels

# labels_e = {'_GO':np.array([0,0,0]),'neg':np.array([1,0,0]),'nut':np.array([0,1,0]),'pos':np.array([0,0,1])}
# def foo2(x):
#   if(x < 5):
#     return [1,0,0]
#   elif(x<7):
#     return [0,1,0]
#   return [0,0,1]

# labels = df.rating.apply(foo2)
# labels = np.array(list(labels))
xTrain,xVal,yTrain,yVal = getData(k=10000)
full_model,attention_model = seq2seq()
xVal.shape
training=full_model.fit(x=xTrain,
  y=yTrain,
  epochs=10,
  validation_split=0.1,
  shuffle=True
)
full_model.evaluate(xVal,
  y=yVal)
y_pred = attention_model.predict(xVal)
y_pred_class = full_model.predict(xVal)
req = np.argmax(y_pred,axis = 2).reshape((-1))
classes = np.argmax(y_pred_class,axis=2).reshape(-1)
imp_vecs = xVal
def v2w(vec):
    if((vec==avg_vec).all()):
        return 'UNK'
    if(np.prod(vec) == 0):
        return 'None'
    for k in w2v_dict:
        if((w2v_dict[k]==vec).all()):
            return k
    return 'OOV'
pos_words = []
neg_words = []
neu_words = []
for i,vecs in enumerate(imp_vecs):
    if(classes[i] == 0):
        neg_words.append({'word':v2w(vecs)})
    if(classes[i] == 1):
        neu_words.append({'word':v2w(vecs)})
    if(classes[i] == 2):
        pos_words.append({'word':v2w(vecs)})
neg_words = pd.DataFrame(neg_words)
neu_words = pd.DataFrame(neu_words)
pos_words = pd.DataFrame(pos_words)
print(neg_words['word'].value_counts())
print(neu_words['word'].value_counts())
print(pos_words['word'].value_counts())
y_pred_class = full_model.predict(xVal)

y_pred_class = np.argmax(y_pred_class,axis=2)
y_pred_class=y_pred_class.reshape((-1))
y_test = np.argmax(yVal,axis=2)
y_test=y_test.reshape((-1))
from sklearn import metrics

cm=metrics.confusion_matrix(y_test[:k],y_pred_class)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','neutral','positive'],
                  columns = ['negative','neutral','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)<jupyter_output><empty_output><jupyter_text>Binary Classification<jupyter_code>xTrain,xVal,yTrain,yVal,_GO,_GO_test = getData(classes=2,k=1000)
full_model,attention_model = seq2seq(classes=2)
training=full_model.fit(x={
    'ex':xTrain,
    'dx':_GO
  },
  y=yTrain,
  epochs=10,
  validation_split=0.1,
  shuffle=True
)
full_model.evaluate({
    'ex':xVal,
    'dx':_GO_test
  },
  y=yVal)
y_pred = attention_model.predict({
    'ex':xVal,
    'dx':_GO_test
  })
y_pred_class = full_model.predict({
    'ex':xVal,
    'dx':_GO_test
  })
req = np.argmax(y_pred,axis = 2).reshape((-1))
classes = np.argmax(y_pred_class,axis=2).reshape(-1)
imp_vecs = xVal
def v2w(vec):
    if((vec==avg_vec).all()):
        return 'UNK'
    if(vec.sum() == 0):
        return 'None'
    for k in w2v_dict:
        if((w2v_dict[k]==vec).all()):
            return k
    return 'OOV'
pos_words = []
neg_words = []
neu_words = []
for i,vecs in enumerate(imp_vecs):
    if(classes[i] == 0):
        neg_words.append({'word':v2w(vecs)})
    if(classes[i] == 1):
        pos_words.append({'word':v2w(vecs)})
neg_words = pd.DataFrame(neg_words)
pos_words = pd.DataFrame(pos_words)
print(neg_words['word'].value_counts())
print(pos_words['word'].value_counts())
y_pred_class = full_model.predict({
    'ex':xVal,
    'dx':_GO_test
  })

y_pred_class = np.argmax(y_pred_class,axis=2)
y_pred_class=y_pred_class.reshape((-1))
y_test = np.argmax(yVal,axis=2)
y_test=y_test.reshape((-1))
from sklearn import metrics

cm=metrics.confusion_matrix(y_test,y_pred_class)

import seaborn as sn
import pandas as pd
import matplotlib.pyplot as plt

df_cm = pd.DataFrame(cm, index = ['negative','positive'],
                  columns = ['negative','postive'])
plt.figure(figsize = (5,5))

ax = sn.heatmap(df_cm, annot=True)

bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15577626764244643, 'fraction_numerical': 0.029099146192716502, 'mean_word_length': 4.989563765393446, 'pattern_counts': {'":': 0, '<': 100, '<?xml version=': 0, '>': 105, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '434600', 'license': 'MIT', 'n_tokens_mistral': 11611, 'n_tokens_neox': 10987, 'n_words': 1704, 'path': 'Seq2Seq_with_Attention.ipynb', 'repo_name': 'Rajesh-Khanna/NLP-Drug-review-classification', 'size': '28691'}