<jupyter_start><jupyter_text>DATA PREPARATION<jupyter_code>#word embeddings with Keras Tokenizer
tokenizer = Tokenizer()

total_reviews = np.concatenate((X_train, X_test))
tokenizer.fit_on_texts(total_reviews)

#pad sequences
#review_lengths = [len(s.split()) for s in total_reviews]
#pad_length = int(np.mean(review_lengths))
pad_length = 100

#define vocabulary size
vocab_size = len(tokenizer.word_index)+1

X_train_tokens = tokenizer.texts_to_sequences(X_train)
X_test_tokens = tokenizer.texts_to_sequences(X_test)

X_train_pad = pad_sequences(X_train_tokens, maxlen=pad_length, padding='post')
X_test_pad = pad_sequences(X_test_tokens, maxlen=pad_length, padding='post')


#embedding layer
embedding_dim =100 #dimension of vector
Embedding(vocab_size, embedding_dim, input_length=pad_length)<jupyter_output><empty_output><jupyter_text>BUILDING MODEL<jupyter_code>model = Sequential([
    Embedding(vocab_size, embedding_dim, input_length=pad_length),
    GRU(units=32, dropout=0.2, recurrent_dropout=0.2),
    Dense(1, activation='sigmoid')
])

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()<jupyter_output>WARNING:tensorflow:From C:\Users\ajankowski\AppData\Local\Continuum\anaconda3\lib\site-packages\tensorflow\python\framework\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From C:\Users\ajankowski\AppData\Local\Continuum\anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_2 (Embedding)      (None, 100, 100)          22912700  
__________________________________[...]<jupyter_text>TRAINING MODEL<jupyter_code>model.fit(X_train_pad, y_train, batch_size=64, epochs=5, verbose=2, validation_data=(X_test_pad, y_test))
#testing on some examples
sample_1 = 'Dieser Film ist fantastisch! Er gefällt mir wirklich, weil er so aufregend ist.'
sample_2 = 'Sehr schöner Film.'
sample_3 = 'Was für ein schlechter Film das war. Ich hätte etwas viel Besseres erwartet.'
sample_4 = 'Dieser Film ist wirklich scheiße. Keine Aktion und die Handlung ist flach wie ein Tisch.'
sample_5 = 'Sehr interessantes Kino. Tolles Spiel der Schauspieler.'

test_samples = [sample_1, sample_2, sample_3, sample_4, sample_5]
test_samples_tokens = tokenizer.texts_to_sequences(test_samples)
test_samples_tokens_pad = pad_sequences(test_samples_tokens, maxlen=pad_length)

#predict
model.predict(x=test_samples_tokens_pad)<jupyter_output><empty_output><jupyter_text>Train word2vec Embedding<jupyter_code>#The first step is to prepare the text corpus for learning the embedding by creating word tokens, removing punctuation, 
#removing stop words etc. The word2vec algorithm processes documents sentence by sentence.
tokenizer = RegexpTokenizer(r'\w+')

review_lines = list()
lines = df_all['fullTextHtml'].values.tolist()


for line in lines:
    tokens = tokenizer.tokenize(line)
    #convert to lower case
    tokens = [w.lower() for w in tokens]
    #remove tokens that are not alphabetic
    tokens = [t for t in tokens if t.isalpha]
    #filter out stop words
    stop_words = set(stopwords.words('german'))
    tokens = [t for t in tokens if t not in stop_words]
    review_lines.append(tokens)
    
len(review_lines)
#train word2vec model
model = gensim.models.Word2Vec(sentences=review_lines, size=embedding_dim, window=5, workers=4, min_count=1)

#vocab size
words = list(model.wv.vocab)
print ('Vocabulary size is: {}'.format(len(words)))<jupyter_output>Vocabulary size is: 205903<jupyter_text>Test Word2Vec Model<jupyter_code>model.wv.most_similar('langweilig')
model.wv.most_similar_cosmul(positive=['frau', 'könig'], negative=['mann'])
#save model
model.wv.save_word2vec_format('w2v_lovelybooks_model', binary=False)<jupyter_output>C:\Users\ajankowski\AppData\Local\Continuum\anaconda3\lib\site-packages\smart_open\smart_open_lib.py:398: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function
  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL<jupyter_text>Use Pre-trained Embedding<jupyter_code>#extract the word embedding from file
embeddings_index ={}
f = open(os.path.join('','w2v_lovelybooks_model'), encoding='utf-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:])
    embeddings_index[word] = coefs
f.close()<jupyter_output><empty_output><jupyter_text>convert word embedding into tokenized vector<jupyter_code>#vectorize the text samples into a 2D integer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(review_lines)
sequences = tokenizer.texts_to_sequences(review_lines)

#pad sequences
word_index = tokenizer.word_index
print('Found {} unique tokens.'.format(len(word_index)))
review_pad = pad_sequences(sequences, maxlen=pad_length)
sentiment = df_all['sentiment'].values
print('Shape of review tensor: ', review_pad.shape)
print('Shape of sentiment tensor: ', sentiment.shape)
#Now we will map embeddings from the loaded word2vec model for each word to the tokenizer.word_index vocabulary 
#and create a matrix with  word vectors.
num_words = len(word_index) + 1
embedding_matrix = np.zeros((num_words, embedding_dim))

for word, i in word_index.items():
    if i > num_words: 
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        #words not found in embedding index will be all zeros
        embedding_matrix[i] = embedding_vector
print(num_words)
#We are now ready with the trained embedding vector to be used directly in the embedding layer. 
#In the below code, the only change from previous model is using the embedding_matrix as input to the Embedding layer 
#and setting trainable = False, since the embedding is already learned.
embedding_layer = Embedding(num_words, 
                            embedding_dim, 
                            embeddings_initializer=Constant(embedding_matrix),
                            input_length=pad_length,
                            trainable=False)

model = Sequential([
    embedding_layer,
    GRU(units=32, dropout=0.2, recurrent_dropout=0.2),
    Dense(1, activation='sigmoid')
])

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_3 (Embedding)      (None, 100, 100)          20590400  
_________________________________________________________________
gru_2 (GRU)                  (None, 32)                12768     
_________________________________________________________________
dense_2 (Dense)              (None, 1)                 33        
=================================================================
Total params: 20,603,201
Trainable params: 12,801
Non-trainable params: 20,590,400
_________________________________________________________________<jupyter_text>Training sentiment model<jupyter_code>#split the data into training set and validation set
validation_split_ratio = 0.25
indices = np.arange(review_pad.shape[0])
np.random.shuffle(indices)
review_pad = review_pad[indices]
sentiment = sentiment[indices]
num_validation_samples = int(validation_split_ratio*review_pad.shape[0])

X_train_pad = review_pad[:-num_validation_samples]
y_train = sentiment[:-num_validation_samples]
X_test_pad = review_pad[-num_validation_samples:]
y_test = sentiment[-num_validation_samples:]

print('Shape of X_train_pad tensor: ', X_train_pad.shape)
print('Shape of y_train tensor: ', y_train.shape)
print('Shape of X_test_pad tensor: ', X_test_pad.shape)
print('Shape of y_test tensor: ', y_test.shape)
np.unique(y_train, return_counts=True)
np.unique(y_test, return_counts=True)
#training the model
model.fit(X_train_pad, y_train, batch_size=128, epochs=10, verbose=2, validation_data=(X_test_pad, y_test))<jupyter_output>Train on 26748 samples, validate on 8916 samples
Epoch 1/10
 - 24s - loss: 0.5582 - acc: 0.6960 - val_loss: 0.3316 - val_acc: 0.8574
Epoch 2/10
 - 17s - loss: 0.3633 - acc: 0.8415 - val_loss: 0.2718 - val_acc: 0.8868
Epoch 3/10
 - 20s - loss: 0.3062 - acc: 0.8690 - val_loss: 0.2505 - val_acc: 0.8956
Epoch 4/10
 - 18s - loss: 0.2808 - acc: 0.8816 - val_loss: 0.2364 - val_acc: 0.9025
Epoch 5/10
 - 18s - loss: 0.2595 - acc: 0.8902 - val_loss: 0.2165 - val_acc: 0.9107
Epoch 6/10
 - 24s - loss: 0.2507 - acc: 0.8974 - val_loss: 0.2076 - val_acc: 0.9155
Epoch 7/10
 - 23s - loss: 0.2376 - acc: 0.9022 - val_loss: 0.2094 - val_acc: 0.9124
Epoch 8/10
 - 22s - loss: 0.2354 - acc: 0.9022 - val_loss: 0.2024 - val_acc: 0.9154
Epoch 9/10
 - 20s - loss: 0.2298 - acc: 0.9049 - val_loss: 0.1986 - val_acc: 0.9188
Epoch 10/10
 - 19s - loss: 0.2215 - acc: 0.9063 - val_loss: 0.1958 - val_acc: 0.9180
{'fraction_non_alphanumeric': 0.11900010503098414, 'fraction_numerical': 0.042642579560970485, 'mean_word_length': 5.038046924540266, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '335391', 'license': 'MIT', 'n_tokens_mistral': 3390, 'n_tokens_neox': 3078, 'n_words': 752, 'path': 'Lovely Books/GSA Lovelybooks.de Keras + Word2Vec.ipynb', 'repo_name': 'andrzej-malina/German-Sentiment-Analysis---AirBNB-example', 'size': '9521'}