<jupyter_start><jupyter_text>Using pretrained GloVe Embedding<jupyter_code># Based on
# https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/6.1-using-word-embeddings.ipynb
# https://machinelearningmastery.com/develop-word-embeddings-python-gensim/
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
%pylab inline
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
print(tf.__version__)<jupyter_output>1.8.0<jupyter_text>Download data:* original imdb database from http://ai.stanford.edu/~amaas/data/sentiment/* pre-computed GloVe embeddings from on Wikipedia data (6B) https://nlp.stanford.edu/projects/glove/<jupyter_code>import os

imdb_dir = 'C:/Users/olive/Development/data/aclImdb'
train_dir = os.path.join(imdb_dir, 'train')

labels = []
texts = []

for label_type in ['neg', 'pos']:
    dir_name = os.path.join(train_dir, label_type)
    for fname in os.listdir(dir_name):
        if fname[-4:] == '.txt':
            f = open(os.path.join(dir_name, fname), encoding='UTF-8')
            texts.append(f.read())
            f.close()
            if label_type == 'neg':
                labels.append(0)
            else:
                labels.append(1)
len(texts)
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import numpy as np

maxlen = 500  # We will cut reviews after 100 words
training_samples = 15000  # We will be training on 200 samples
validation_samples = 10000  # We will be validating on 10000 samples
max_words = 10000  # We will only consider the top 10,000 words in the dataset

tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

data = pad_sequences(sequences, maxlen=maxlen)

labels = np.asarray(labels)
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

# But first, shuffle the data, since we started from data
# where sample are ordered (all negative first, then all positive).
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(data, labels, test_size=0.2, random_state=42, stratify=labels)
x_train.shape<jupyter_output><empty_output><jupyter_text>With pre-defined and fixed embeddings, we can not be better than just guessing<jupyter_code>glove_dir = 'C:/Users/olive/Development/data/glove.6B'

embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), encoding='UTF-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

print('Found %s word vectors.' % len(embeddings_index))
embedding_dim = 100

embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if i < max_words:
        if embedding_vector is not None:
            # Words not found in embedding index will be all-zeros.
            embedding_matrix[i] = embedding_vector
from keras.models import Sequential
from keras.layers import Embedding, Flatten, Dense

model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=maxlen))
model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.layers[0].set_weights([embedding_matrix])
model.layers[0].trainable = False
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['acc'])
model.summary()
batch_size=1000
model.fit(x_train, y_train,
                    epochs=10,
                    batch_size=batch_size,
                    validation_split=0.2)<jupyter_output>Train on 16000 samples, validate on 4000 samples
Epoch 1/10
16000/16000 [==============================] - 1s 48us/step - loss: 0.9644 - acc: 0.5029 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 2/10
16000/16000 [==============================] - 0s 24us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 3/10
16000/16000 [==============================] - 0s 23us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 4/10
16000/16000 [==============================] - 0s 23us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 5/10
16000/16000 [==============================] - 0s 23us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 6/10
16000/16000 [==============================] - 0s 22us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc: 0.4945
Epoch 7/10
16000/16000 [==============================] - 0s 23us/step - loss: 0.6931 - acc: 0.5014 - val_loss: 0.6932 - val_acc:[...]<jupyter_text>Embedding trainable, but still pre-set<jupyter_code>model.layers[0].trainable = True
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['acc'])
model.summary()
batch_size=1000
model.fit(x_train, y_train,
                    epochs=20,
                    batch_size=batch_size,
                    validation_split=0.2)<jupyter_output>Train on 16000 samples, validate on 4000 samples
Epoch 1/100
16000/16000 [==============================] - 1s 42us/step - loss: 0.6872 - acc: 0.5368 - val_loss: 0.6789 - val_acc: 0.5817
Epoch 2/100
16000/16000 [==============================] - 1s 38us/step - loss: 0.6610 - acc: 0.6124 - val_loss: 0.6483 - val_acc: 0.6625
Epoch 3/100
16000/16000 [==============================] - 1s 38us/step - loss: 0.6072 - acc: 0.7096 - val_loss: 0.6066 - val_acc: 0.7432
Epoch 4/100
16000/16000 [==============================] - 1s 39us/step - loss: 0.5435 - acc: 0.7864 - val_loss: 0.5704 - val_acc: 0.7510
Epoch 5/100
16000/16000 [==============================] - 1s 39us/step - loss: 0.4832 - acc: 0.8464 - val_loss: 0.5604 - val_acc: 0.7502
Epoch 6/100
16000/16000 [==============================] - 1s 38us/step - loss: 0.4364 - acc: 0.8869 - val_loss: 0.5564 - val_acc: 0.7625
Epoch 7/100
16000/16000 [==============================] - 1s 39us/step - loss: 0.4021 - acc: 0.9178 - val_loss: 0.5374 - v[...]<jupyter_text>Embeddings trained from scratch<jupyter_code>from keras.models import Sequential
from keras.layers import Embedding, Flatten, Dense

model = Sequential()
model.add(Embedding(max_words, embedding_dim, input_length=maxlen))
model.add(Flatten())
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['acc'])
model.summary()
batch_size=1000
model.fit(x_train, y_train,
                    epochs=10,
                    batch_size=batch_size,
                    validation_split=0.2)
train_loss, train_accuracy = model.evaluate(x_train, y_train, batch_size=batch_size)
train_accuracy
test_loss, test_accuracy = model.evaluate(x_test, y_test, batch_size=batch_size)
test_accuracy<jupyter_output>5000/5000 [==============================] - 0s 16us/step
{'fraction_non_alphanumeric': 0.181477367397945, 'fraction_numerical': 0.08678144959733407, 'mean_word_length': 3.9573296627666896, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '559695', 'license': 'MIT', 'n_tokens_mistral': 2936, 'n_tokens_neox': 2459, 'n_words': 548, 'path': 'notebooks/nlp/1b-glove-embedding.ipynb', 'repo_name': 'subodhchhabra/ai', 'size': '7202'}