<jupyter_start><jupyter_text>Choose GPU (this may not be needed on your computer)<jupyter_code>%env CUDA_DEVICE_ORDER=PCI_BUS_ID
%env CUDA_VISIBLE_DEVICES=0<jupyter_output>env: CUDA_DEVICE_ORDER=PCI_BUS_ID
env: CUDA_VISIBLE_DEVICES=0<jupyter_text>load packages<jupyter_code>from tfumap.umap import tfUMAP
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tqdm.autonotebook import tqdm
import umap
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load dataset<jupyter_code>from tfumap.paths import ensure_dir, MODEL_DIR, DATA_DIR
syllable_df = pd.read_pickle(DATA_DIR/'cassins'/ 'cassins.pickle')
#syllable_df= syllable_df[:1000]
syllable_df[:3]
top_labels = (
    pd.DataFrame(
        {i: [np.sum(syllable_df.labels.values == i)] for i in syllable_df.labels.unique()}
    )
    .T.sort_values(by=0, ascending=False)[:20]
    .T
)
top_labels
syllable_df = syllable_df[syllable_df.labels.isin(top_labels.columns)]
syllable_df[:3]
syllable_df = syllable_df.reset_index()
syllable_df['subset'] = 'train'
syllable_df.loc[:1000, 'subset'] = 'valid'
syllable_df.loc[1000:1999, 'subset'] = 'test'
#syllable_df.loc[:100, 'subset'] = 'valid'
#syllable_df.loc[100:199, 'subset'] = 'test'
specs = np.array(list(syllable_df.spectrogram.values))
specs = np.array([np.concatenate([np.zeros((32,1)), i], axis=1) for i in tqdm(specs)])
specs.shape
syllable_df['spectrogram'] = syllable_df['spectrogram'].astype('object')
syllable_df['spectrogram'] = list(specs)
np.shape(syllable_df['spectrogram'].values[0])
len(syllable_df)
Y_train = np.array(list(syllable_df.labels.values[syllable_df.subset == 'train']))
Y_valid = np.array(list(syllable_df.labels.values[syllable_df.subset == 'valid']))
Y_test = np.array(list(syllable_df.labels.values[syllable_df.subset == 'test']))
X_train = np.array(list(syllable_df.spectrogram.values[syllable_df.subset == 'train'])) #/ 255.
X_valid = np.array(list(syllable_df.spectrogram.values[syllable_df.subset == 'valid']))# / 255.
X_test = np.array(list(syllable_df.spectrogram.values[syllable_df.subset == 'test'])) #/ 255.
X_train_flat = X_train.reshape((len(X_train), np.product(np.shape(X_train)[1:])))
from sklearn.preprocessing import OrdinalEncoder
enc = OrdinalEncoder()
Y_train = enc.fit_transform([[i] for i in Y_train]).astype('int').flatten()
plt.matshow(X_train[10])
X_train[0].shape<jupyter_output><empty_output><jupyter_text>create models<jupyter_code>dims = (32,32,1)
from tensorflow.keras.layers import (
    Conv2D,
    Reshape,
    Bidirectional,
    Dense,
    RepeatVector,
    TimeDistributed,
    LSTM
)
n_components=64
#shape_final = (8,2,128)
encoder = tf.keras.Sequential([
    tf.keras.layers.InputLayer(input_shape=dims),
    Conv2D(
        filters=32, kernel_size=3, strides=(2, 2), activation=tf.nn.leaky_relu, padding="same"
    ),
    Conv2D(
        filters=64, kernel_size=3, strides=(2, 2), activation=tf.nn.leaky_relu, padding="same"
    ),
    Conv2D(
        filters=128, kernel_size=3, strides=(2, 1), activation=tf.nn.leaky_relu, padding="same"
    ),
    Conv2D(
        filters=128, kernel_size=3, strides=(2, 1), activation=tf.nn.leaky_relu, padding="same"
    ),
    Reshape(target_shape=(8, 2*128)),
    Bidirectional(LSTM(units=100, activation="relu")),
    Dense(units=512),
    Dense(units=n_components),
])
decoder = tf.keras.Sequential([
    tf.keras.layers.InputLayer(input_shape=(n_components)),
    Dense(units=512),
    RepeatVector(8),
    Bidirectional(LSTM(units=100, activation="relu", return_sequences=True)),
    TimeDistributed(Dense(2*128)),
    Reshape(target_shape=(8,2,128)),
    tf.keras.layers.Conv2DTranspose(
        filters=128, kernel_size=3, strides=(1, 2), padding="SAME", activation=tf.nn.leaky_relu
    ),
    tf.keras.layers.Conv2DTranspose(
        filters=128, kernel_size=3, strides=(1, 2), padding="SAME", activation=tf.nn.leaky_relu
    ),
    tf.keras.layers.Conv2DTranspose(
        filters=64, kernel_size=3, strides=(2, 2), padding="SAME", activation=tf.nn.leaky_relu
    ),
    tf.keras.layers.Conv2DTranspose(
        filters=32, kernel_size=3, strides=(2, 2), padding="SAME", activation=tf.nn.leaky_relu
    ),
    tf.keras.layers.Conv2DTranspose(
        filters=1, kernel_size=3, strides=(1, 1), padding="SAME", activation="tanh"
    ),
    Reshape(target_shape=(32, 32, 1)),
])<jupyter_output><empty_output><jupyter_text>Prepare metric<jupyter_code>from tfumap.dtw_mse import build_dtw_mse
dtw_metric = build_dtw_mse(X_train[0].shape)<jupyter_output><empty_output><jupyter_text>Create model and train<jupyter_code>embedder = tfUMAP(
    metric = dtw_metric,
    direct_embedding=False,
    verbose=True,
    negative_sample_rate=5,
    training_epochs=5,
    encoder=encoder,
    dims=dims,
    n_components=n_components,
)
z = embedder.fit_transform(X_train_flat)
z = embedder.transform(X_train_flat)<jupyter_output>WARNING: Logging before flag parsing goes to stderr.
W0714 13:47:33.828815 140328589719296 base_layer.py:1790] Layer conv2d is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.

If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.

To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.<jupyter_text>Plot model output<jupyter_code>fig, ax = plt.subplots( figsize=(8, 8))
sc = ax.scatter(
    z[:, 0],
    z[:, 1],
    c=Y_train[:len(z)],
    cmap="tab20",
    s=0.1,
    alpha=0.5,
    rasterized=True,
)
ax.axis('equal')
ax.set_title("UMAP in Tensorflow embedding", fontsize=20)
plt.colorbar(sc, ax=ax);<jupyter_output><empty_output><jupyter_text>View loss<jupyter_code>from tfumap.umap import retrieve_tensors
import seaborn as sns
loss_df = retrieve_tensors(embedder.tensorboard_logdir)
loss_df[:3]
ax = sns.lineplot(x="step", y="val", hue="group", data=loss_df[loss_df.variable=='umap_loss'])
ax.set_xscale('log')
ax.set_yscale('log')<jupyter_output><empty_output><jupyter_text>Save output<jupyter_code>from tfumap.paths import ensure_dir, MODEL_DIR
output_dir = MODEL_DIR/'projections'/ 'cassins' / '64'/ 'seq2seq'
ensure_dir(output_dir)
embedder.save(output_dir)
loss_df.to_pickle(output_dir / 'loss_df.pickle')
np.save(output_dir / 'z.npy', z)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13863465866466618, 'fraction_numerical': 0.03360840210052513, 'mean_word_length': 5.788187372708758, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '44448', 'license': 'MIT', 'n_tokens_mistral': 2691, 'n_tokens_neox': 2527, 'n_words': 449, 'path': 'notebooks/dataset-projections/64/cassins/cassins-embedding-seq2seq-dtw.ipynb', 'repo_name': 'timsainb/ParametricUMAP_paper', 'size': '6665'}