<jupyter_start><jupyter_text><jupyter_code>import tensorflow as tf

print(tf.__version__)
string = "中医针灸"
bytes = tf.strings.unicode_split(string, 'UTF-8')
print(bytes)
from tensorflow import keras
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

sentences = [
    'I love my dog',
    'I love my cat',
    'You love my dog!',
    'Do you think my dog is amazing?'
]

tokenizer = Tokenizer(num_words = 100, oov_token="<OOV>")
tokenizer.fit_on_texts(sentences)
word_index = tokenizer.word_index
print(word_index)
# OOV - Out of vocabulary
sequences = tokenizer.texts_to_sequences(sentences)

padded = pad_sequences(sequences, maxlen=5) # max length of the sequence=5
print("\nWord Index = " , word_index)
print("\nSequences = " , sequences)
print("\nPadded Sequences:")
print(padded)
# Try with words that the tokenizer wasn't fit to
test_data = [
    'i really love my dog',
    'my dog loves my manatee'
]

print("\nWord Index = " , word_index)
test_seq = tokenizer.texts_to_sequences(test_data)
print("\nTest Sequence = ", test_seq)

padded = pad_sequences(test_seq, maxlen=10)
print("\nPadded Test Sequence: ")
print(padded)
padded_post = pad_sequences(test_seq, padding='post', truncating='pre',maxlen=10)
print("\nPadded Test Sequence: ")
print(padded_post)
padded_post = pad_sequences(test_seq, padding='pre', truncating='pre',maxlen=10)
print("\nPadded Test Sequence: ")
print(padded_post)
string = "中医针灸"
bytes = tf.strings.unicode_split(string, 'UTF-8')
print(bytes)
import json
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
vocab_size = 10000
embedding_dim = 16
max_length = 100
trunc_type='post'
padding_type='post'
oov_tok = "<OOV>"
training_size = 20000<jupyter_output><empty_output><jupyter_text>Fetch the sarcasm (text sentiment) dataset<jupyter_code>!wget --no-check-certificate \
    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/sarcasm.json \
    -O ./sarcasm.json
with open("./sarcasm.json", 'r') as f:
    datastore = json.load(f)

sentences = []
labels = []

for item in datastore:
    sentences.append(item['headline'])
    labels.append(item['is_sarcastic'])<jupyter_output><empty_output><jupyter_text>split the dataset<jupyter_code>from sklearn.model_selection import train_test_split
training_sentences, testing_sentences, training_labels, testing_labels = train_test_split(sentences, labels, test_size=0.15, random_state=100)
print(f'training set: {len(training_sentences)}')
print(f'test set: {len(testing_sentences)}')
tokenizer = Tokenizer(num_words=vocab_size, oov_token=oov_tok)
tokenizer.fit_on_texts(training_sentences)
word_index = tokenizer.word_index
list(word_index)[:10]
training_sequences = tokenizer.texts_to_sequences(training_sentences)
training_padded = pad_sequences(training_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)

testing_sequences = tokenizer.texts_to_sequences(testing_sentences)
testing_padded = pad_sequences(testing_sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)
import numpy as np
training_padded = np.array(training_padded)
training_labels = np.array(training_labels)
testing_padded = np.array(testing_padded)
testing_labels = np.array(testing_labels)<jupyter_output><empty_output><jupyter_text>Define the NN model<jupyter_code>from tensorflow.keras.layers import Embedding, GlobalAveragePooling1D, Dense, Input
from tensorflow.keras import Sequential


model = Sequential()
model.add(Embedding(vocab_size, embedding_dim, input_length=max_length, name='embed_0'))
model.add(GlobalAveragePooling1D(name='avg_pool_1'))
model.add(Dense(24, activation='relu'))
model.add(Dense(1, activation='sigmoid', name='output'))

model.compile(loss='binary_crossentropy',optimizer='adam',metrics=['accuracy'])
from tensorflow.keras.utils import plot_model

plot_model(model, to_file='model.png')
model.summary()
num_epochs = 30
history = model.fit(training_padded, training_labels, epochs=num_epochs, validation_data=(testing_padded, testing_labels), verbose=2)<jupyter_output>Epoch 1/30
710/710 - 4s - loss: 0.6388 - accuracy: 0.6261 - val_loss: 0.5061 - val_accuracy: 0.7846
Epoch 2/30
710/710 - 4s - loss: 0.3932 - accuracy: 0.8468 - val_loss: 0.3705 - val_accuracy: 0.8405
Epoch 3/30
710/710 - 4s - loss: 0.2989 - accuracy: 0.8823 - val_loss: 0.3404 - val_accuracy: 0.8508
Epoch 4/30
710/710 - 4s - loss: 0.2519 - accuracy: 0.9015 - val_loss: 0.3291 - val_accuracy: 0.8595
Epoch 5/30
710/710 - 4s - loss: 0.2197 - accuracy: 0.9155 - val_loss: 0.3293 - val_accuracy: 0.8617
Epoch 6/30
710/710 - 4s - loss: 0.1966 - accuracy: 0.9267 - val_loss: 0.3336 - val_accuracy: 0.8610
Epoch 7/30
710/710 - 4s - loss: 0.1762 - accuracy: 0.9349 - val_loss: 0.3568 - val_accuracy: 0.8550
Epoch 8/30
710/710 - 4s - loss: 0.1593 - accuracy: 0.9419 - val_loss: 0.3782 - val_accuracy: 0.8495
Epoch 9/30
710/710 - 4s - loss: 0.1454 - accuracy: 0.9475 - val_loss: 0.3906 - val_accuracy: 0.8518
Epoch 10/30
710/710 - 4s - loss: 0.1336 - accuracy: 0.9520 - val_loss: 0.3869 - val_accuracy: 0.8565[...]<jupyter_text>Visualize the training process<jupyter_code>import matplotlib.pyplot as plt

def plot_graphs(history, string):
  plt.plot(history.history[string])
  plt.plot(history.history['val_'+string])
  plt.xlabel("Epochs")
  plt.ylabel(string)
  plt.legend([string, 'val_'+string])
  plt.show()


def summarize_diagnostics(history):
  plt.figure(figsize=(10, 10))
  # plot loss
  plt.subplot(1, 2, 1)
  plt.plot(history.history['loss'], color='blue', label='Training')
  plt.plot(history.history['val_loss'], color='orange', label='Test')
  plt.legend(loc='lower right')
  plt.title('Cross Entropy Loss')
  # plot accuracy
  plt.subplot(1, 2, 2)
  plt.plot(history.history['accuracy'], color='blue', label='Training')
  plt.plot(history.history['val_accuracy'], color='orange', label='Test')
  plt.legend(loc='lower right')
  plt.title('Classification Accuracy')
  plt.show()
  
loss, acc = model.evaluate(testing_padded, testing_labels, verbose=0)
print("loss: {:6.3f}, accuracy: {:6.3f}%".format(loss, acc*100))

plot_graphs(history, "accuracy")
plot_graphs(history, "loss")
summarize_diagnostics(history)<jupyter_output><empty_output><jupyter_text>Get the reversed index from the tokenizer<jupyter_code>reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])
# get the first 5 values by the key 1 to 5:
for (key, value) in enumerate(word_index.items()):
  if key == 5:
    break
  else:
    print(f'{reverse_word_index.get(key)}')
def decode_sentence(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])

print(decode_sentence(training_padded[0]))
print(training_sentences[2])
print(labels[2])
# get the embedding layer
e = model.layers[0]
weights = e.get_weights()[0]
print(weights.shape) # shape: (vocab_size, embedding_dim)
import io

out_v = io.open('vecs.tsv', 'w', encoding='utf-8')
out_m = io.open('meta.tsv', 'w', encoding='utf-8')
for word_num in range(1, vocab_size):
  word = reverse_word_index[word_num]
  embeddings = weights[word_num]
  out_m.write(word + "\n")
  out_v.write('\t'.join([str(x) for x in embeddings]) + "\n")
out_v.close()
out_m.close()
try:
  from google.colab import files
except ImportError:
  pass
else:
  files.download('vecs.tsv')
  files.download('meta.tsv')
sentence = ["granny starting to fear spiders in the garden might be real", "game of thrones season finale showing this sunday night","Her voice was dripping with sarcasm and, Cole noted with amusement, jealousy."]
sequences = tokenizer.texts_to_sequences(sentence)
padded = pad_sequences(sequences, maxlen=max_length, padding=padding_type, truncating=trunc_type)
score = model.predict(padded)
print(score)
predict_class = ['non-sacrasm', 'sarcasm']
for sent_score in score:
  if sent_score >= 0.5:
    print(predict_class[1])
  else:
    print(predict_class[0])<jupyter_output>[[9.9454981e-01]
 [4.4951244e-07]
 [9.9704629e-01]]
sarcasm
non-sacrasm
sarcasm<jupyter_text>Use a LSTM Text Generation by LSTM<jupyter_code>import tensorflow as tf

from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Embedding, LSTM, Dense, Bidirectional
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
import numpy as np
!wget --no-check-certificate \
    https://storage.googleapis.com/laurencemoroney-blog.appspot.com/irish-lyrics-eof.txt \
    -O ./irish-lyrics-eof.txt
tokenizer = Tokenizer()
data = open('./irish-lyrics-eof.txt').read()
corpus = data.lower().split("\n")
tokenizer.fit_on_texts(corpus)
total_words = len(tokenizer.word_index) + 1

print(tokenizer.word_index)
print(total_words)
input_sequences = []
token_list = tokenizer.texts_to_sequences([corpus[1]])[0]
for line in corpus:
  # remove the outer list wrapper
	token_list = tokenizer.texts_to_sequences([line])[0] 
	for i in range(1, len(token_list)):
		n_gram_sequence = token_list[:i+1]
		input_sequences.append(n_gram_sequence)

# pad sequences 
max_sequence_len = max([len(x) for x in input_sequences])
input_sequences = np.array(pad_sequences(input_sequences, maxlen=max_sequence_len, padding='pre'))

# create predictors and label
xs, labels = input_sequences[:,:-1],input_sequences[:,-1]

ys = tf.keras.utils.to_categorical(labels, num_classes=total_words)
# show some words' index in the tokenizer
print(tokenizer.word_index['in'])
print(tokenizer.word_index['the'])
print(tokenizer.word_index['town'])
print(tokenizer.word_index['of'])
print(tokenizer.word_index['athy'])
print(tokenizer.word_index['one'])
print(tokenizer.word_index['jeremy'])
print(tokenizer.word_index['lanigan'])
print(f'Total lines: {len(corpus)}')
print(f'Total amount of word in the corpus {total_words}')
print(xs.shape)
print(f'Max sequence length: {max_sequence_len}')
print('The 6th row of the sequence\n')
print(xs[5])
print(ys[5])
print('\n')
print('The 7th row of the sequence\n')
print(xs[6])
print(ys[6])
model = Sequential()
model.add(Embedding(total_words, 100, input_length=max_sequence_len-1))
model.add(Bidirectional(LSTM(150)))
model.add(Dense(total_words, activation='softmax'))
adam = Adam(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])
#earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto')
history = model.fit(xs, ys, epochs=10, verbose=1)
#print model.summary()
print(model)
def plot_graphs(history, string):
  plt.plot(history.history[string])
  plt.xlabel("Epochs")
  plt.ylabel(string)
  plt.show()

plot_graphs(history, 'accuracy')
seed_text = "I've got a bad feeling about this"
next_words = 100
token_list = tokenizer.texts_to_sequences([seed_text])[0]
print(token_list)
token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
token_list
predicted = model.predict_classes(token_list, verbose=0)
print(predicted)
seed_text = "I've got a bad feeling about this"
next_words = 100
  
for _ in range(next_words):
	token_list = tokenizer.texts_to_sequences([seed_text])[0]
	token_list = pad_sequences([token_list], maxlen=max_sequence_len-1, padding='pre')
	predicted = model.predict_classes(token_list, verbose=0)
	output_word = ""
	for word, index in tokenizer.word_index.items():
		if index == predicted:
			output_word = word
			break
	seed_text += " " + output_word
print(seed_text)
!pip install docx2txt
import docx2txt
 
# read in word file
result = docx2txt.process('ehr_001.docx')
print(result)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13524871355060034, 'fraction_numerical': 0.039365351629502575, 'mean_word_length': 6.265420560747663, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '72436', 'license': 'MIT', 'n_tokens_mistral': 4552, 'n_tokens_neox': 4170, 'n_words': 866, 'path': 'natural_language_processing.ipynb', 'repo_name': 'StanleyLiangYork/NLP_deepLearning', 'size': '11660'}