<jupyter_start><jupyter_text>Embedding ReconstructionCreate embeddings readable by ungol.common.embed based on thereconstruction of a given embcompr model. This notebook heavily relieson some naming conventions.<jupyter_code>from ungol.models import embcompr
import ungol.common.embed as uce

import h5py
import torch
import numpy as np
from tqdm import tqdm_notebook as tqdm

import pickle
import pathlib
from collections import namedtuple
DEV = torch.device('cpu')
def create_embed(embed: uce.Embed, compr: embcompr.Compressor) -> np.ndarray:
    BATCH_SIZE = 128

    gen = enumerate(embed.chunks(size=BATCH_SIZE))
    bar = tqdm(gen, total=len(embed) // BATCH_SIZE)

    Y = []

    for i, x in bar:
        y = compr(torch.from_numpy(x)).detach().numpy()
        Y.append(y)

    Y = np.vstack(Y)
    assert Y.shape == embed.shape, f'{Y.shape} != {embed.shape}'
    return Y
# use naming convention as given in ../conf/embcompr.<DATASET>.conf

# opt/bow/sick
# ├── mbow.cosine-dist.h5
# ├── mbow.embedding.h5
# ├── mbow.sentences.txt
# ├── mbow.vocab.pickle
# ├── sent2vec.cosine-dist.h5
# ├── sent2vec.embedding.h5
# ├── sent2vec.sentences.txt
# ├── sent2vec.vocab.pickle
# ...


Params = namedtuple('Params', ('dataset', 'redux', 'bits'))


def _get_model_path(p: Params):
    return pathlib.Path('../opt/current') / f'{p.dataset}.{p.redux}-{p.bits}'


def load(p: Params):
    embed_in_path = pathlib.Path('../opt/bow') / p.dataset
    print(f'loading embeddings from {embed_in_path}')

    embed_in = uce.create(uce.Config(
        provider='h5py',
        file_name=str(embed_in_path / f'{p.redux}.embedding.h5'),
        vocabulary=str(embed_in_path / f'{p.redux}.vocab.pickle'),
    ))

    print(f'obtained embedding space of shape {embed_in.shape}')
    model_in_path = _get_model_path(p)
    print(f'load model from {model_in_path}')

    f_model = model_in_path / 'compressor'
    compr = embcompr.Compressor.load(str(f_model), 'model.torch', DEV)

    name = (f_model / 'model.torch').resolve().stem
    compr.eval()

    print(f'models real name is {name} (version={compr.version})')
    return embed_in, compr, name


def save(Y: np.ndarray, p: Params, f_model: str, embed: uce.Embed):
    path = _get_model_path(p)

    f_h5 = path / f'recon.{f_model}.embedding.h5'
    print(f'writing {f_h5}')
    with h5py.File(f_h5, mode='w') as fd:
        fd.create_dataset('embedding', data=Y)

    f_pickle = path / f'recon.{f_model}.vocab.pickle'
    print(f'writing {f_pickle}')
    with open(f_pickle, mode='wb') as fd:
        pickle.dump(embed.vocab, fd)
# embed_in, compr, name = load('sick', 'mbow', 256)

def run(p: Params):

    print('loading...')
    embed_in, compr, name = load(p)

    print('tranforming...')
    Y = create_embed(embed_in, compr)

    print('saving...')
    save(Y, p, name, embed_in)

    print('done')

# p = Params(dataset='sick', redux='mbow', bits=256)
# run(p)
for glob in pathlib.Path('../opt/current/').glob('*.*-*'):
    dataset, suffix = glob.name.split('.')
    redux, bits = suffix.split('-')

    p = Params(dataset=dataset, redux=redux, bits=bits)

    print(f'\n{p}\n')
    run(p)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1626144616356173, 'fraction_numerical': 0.007893905904641616, 'mean_word_length': 3.6383601756954613, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '243044', 'license': 'MIT', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1204, 'n_words': 249, 'path': 'models/notes/reconstruction.ipynb', 'repo_name': 'kantholtz/ungol', 'size': '3167'}