<jupyter_start><jupyter_text><jupyter_code>!pip install -q tensorflow-text
import tensorflow as tf
import tensorflow_datasets as tfds
import tensorflow_text as text
ds = tfds.load('imdb_reviews', split='train')
docs = []
for ex in ds.take(10):
  print(ex)
  docs.append(ex)

docs
tokenizer = tfds.features.text.Tokenizer()

vocabulary_set = set()
for doc in docs:
  some_tokens = tokenizer.tokenize(doc["text"].numpy())
  vocabulary_set.update(some_tokens)

vocab_size = len(vocabulary_set)
vocab_size
docs[1]
encoder = tfds.features.text.TokenTextEncoder(vocabulary_set)
encoded = [encoder.encode(doc["text"].numpy()) for doc in docs]
for i in range(len(encoded)):
  print(','.join([str(a) for a in encoded[i]]))
padded_inputs = tf.keras.preprocessing.sequence.pad_sequences(encoded, padding="post")
padded_inputs[0]
(train_data, test_data), info = tfds.load(
    'imdb_reviews/subwords8k', 
    split = (tfds.Split.TRAIN, tfds.Split.TEST), 
    with_info=True, as_supervised=True)
encoder = info.features['text'].encoder
encoder.subwords[:20]
train_batches = train_data.shuffle(1000).padded_batch(10)
test_batches = test_data.shuffle(1000).padded_batch(10)
train_batch, train_labels = next(iter(train_batches))
train_batch.numpy()
import tensorflow.keras as keras
import tensorflow.keras.layers as layers

embedding_dim=16

model = tf.keras.Sequential([
  layers.Embedding(encoder.vocab_size, embedding_dim),
  layers.GlobalAveragePooling1D(),
  layers.Dense(16, activation='relu'),
  layers.Dense(1)
])

model.summary()
model.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              metrics=['accuracy'])

history = model.fit(
    train_batches,
    epochs=10,
    validation_data=test_batches, validation_steps=20)
e = model.layers[0]
weights = e.get_weights()[0]
print(weights.shape) 
weights[0]
from scipy.spatial import distance

print(1 - distance.cosine(weights[2850], weights[2900]))
encoder.subwords[2850], encoder.subwords[2900]<jupyter_output>-0.8510435819625854
{'fraction_non_alphanumeric': 0.13392417528311176, 'fraction_numerical': 0.03249630723781388, 'mean_word_length': 6.006896551724138, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '140144', 'license': 'Apache-2.0', 'n_tokens_mistral': 776, 'n_tokens_neox': 731, 'n_words': 126, 'path': 'PAIGCP_Embeddings.ipynb', 'repo_name': 'cxbxmxcx/PAIGCP', 'size': '2031'}