<jupyter_start><jupyter_text>Default DOBF embeddings<jupyter_code>MODEL_PATH = r"/home/igor/PycharmProjects/CodeGen/training_artifacts/models/DOBF_transcoder_size.pth"
params, dico, (encoder, decoder) = _reload_model(MODEL_PATH)
encoder, decoder = encoder[0], decoder[0]
obf_type = 'CLASS'
var_idxs = torch.arange(dico.obf_index[obf_type], dico.obf_index[obf_type] + 500)
var_embeddings = encoder.embeddings(var_idxs).detach().numpy().astype(np.float32)
var_embeddings_2 = TSNE(n_components=2, init='random', learning_rate='auto', random_state=0).fit_transform(var_embeddings)
t = ["class"] * 100 + ["func"] * 200 + ["var"] * 200
index = list(range(100)) + list(range(200)) + list(range(200))
df = pd.DataFrame(data={"x": var_embeddings_2[:, 0], "y": var_embeddings_2[:, 1], "type": t, "index": index})
fig = px.scatter(df, x="x", y="y", color="type", hover_data=["index"])
fig.update_layout(title="TSNE", width=700, height=700)
plotly.offline.iplot(fig)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1839430894308943, 'fraction_numerical': 0.03861788617886179, 'mean_word_length': 8.94949494949495, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '327618', 'license': 'MIT', 'n_tokens_mistral': 418, 'n_tokens_neox': 386, 'n_words': 58, 'path': 'notebooks/embeddings.ipynb', 'repo_name': 'davidenkoim/CodeGen', 'size': '984'}