<jupyter_start><jupyter_text>Colab initialization- install the pipeline in the colab runtime- download files neccessary for this example<jupyter_code>!pip3 install -U pip > /dev/null
!pip3 install -U bio_embeddings[all] > /dev/null
!wget http://data.bioembeddings.com/public/embeddings/notebooks/pipeline_output_example/mapping_file.csv --output-document mapping_file.csv
!wget http://data.bioembeddings.com/public/embeddings/notebooks/pipeline_output_example/reduced_embeddings_file.h5 --output-document reduced_embeddings_file.h5
!wget http://data.bioembeddings.com/public/embeddings/notebooks/custom_data/annotation_file.csv --output-document annotation_file.csv<jupyter_output><empty_output><jupyter_text>Visualize sequence spaces drawn by embeddingsIn this notebook, we use the output of the _embed_ stage to draw custom t-SNE sequence space plots.<jupyter_code>import h5py
import numpy as np
from pandas import read_csv
from bio_embeddings.project import tsne_reduce
from bio_embeddings.visualize import render_3D_scatter_plotly
mapping_file = read_csv('mapping_file.csv', index_col=0)
embeddings = []
with h5py.File('reduced_embeddings_file.h5', 'r') as f:
    for remapped_id in mapping_file.index:
        embeddings.append(np.array(f[remapped_id]))
options = {
    'perplexity': 3,
    'n_iter': 500
}

projected_embeddings = tsne_reduce(embeddings, **options)
mapping_file['component_0'] = projected_embeddings[:, 0]
mapping_file['component_1'] = projected_embeddings[:, 1]
mapping_file['component_2'] = projected_embeddings[:, 2]
annotation_file = read_csv('annotation_file.csv', index_col=0)

merged_annotation_file = annotation_file.join(mapping_file.set_index('original_id'), how='outer')
merged_annotation_file['label'].fillna('UNKNOWN', inplace=True)
figure = render_3D_scatter_plotly(merged_annotation_file)
figure.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1054039593365436, 'fraction_numerical': 0.011235955056179775, 'mean_word_length': 7.538812785388128, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '342423', 'license': 'MIT', 'n_tokens_mistral': 620, 'n_tokens_neox': 607, 'n_words': 117, 'path': 'notebooks/project_visualize_pipeline_embeddings.ipynb', 'repo_name': 'kvetab/bio_embeddings', 'size': '1869'}