<jupyter_start><jupyter_text>NCD evaluation<jupyter_code>dn = config.plots_dir
best_dims = io.load(dn + 'transformations-best_dims.pkl')
transformations = io.load(dn + 'transformations.pkl')
min_transformations = io.load(dn + 'min_transformations.pkl')
len(list(transformations.keys()))
plt.hist(best_dims)
m = 500
x_train_encoded = encoder.predict(x_train[:m], batch_size=batch_size)
x_train_encoded.shape
# utils.reload(ncd_evaluation)
grid = [0, 0.01, 0.05, 0.1, 0.2, 0.4, 0.8, 1, 0, -0.01, -0.1, -0.5, -1]
grid = np.linspace(0,1, 21)
# grid = (np.logspace(0, 1, 10) -1) / 9
# grid = np.round(grid, 2)
amt1 = None
amt2 = None
grid
# different_genre_a = True
# result_min = ncd_evaluation.cross(x_train_encoded, genre_dict, min_transformations, 
#                                   generator, grid, different_genre_a, amt1=amt1, amt2=amt2, v=1)
# io.save(result_min, config.plots_dir + 'ncd_evaluation_ABC_min.pkl')
# result = ncd_evaluation.cross(x_train_encoded, genre_dict, transformations, 
#                               generator, grid, different_genre_a, amt1=amt1, amt2=amt2, v=0)
# io.save(result, config.plots_dir + 'ncd_evaluation_ABC.pkl')
# utils.reload(ncd_evaluation)
# different_genre_a = False
# result_AB_min = ncd_evaluation.cross(x_train_encoded, genre_dict, min_transformations, 
#                                      generator, grid, different_genre_a, amt1=amt1, amt2=amt2, v=2)
# print(len(list(result_AB_min.keys())))
# io.save(result_AB_min, config.plots_dir + 'ncd_evaluation_AB_min.pkl')
# different_genre_a = False
# result_AB = ncd_evaluation.cross(x_train_encoded, genre_dict, transformations, 
#                                  generator, grid, different_genre_a, amt1=amt1, amt2=amt2, v=0)
# print(len(list(result_AB.keys())))
# io.save(result_AB, config.plots_dir + 'ncd_evaluation_AB.pkl')
# NCD to original
# amt1 = None
# amt2 = None
utils.reload(ncd_evaluation)
different_genre_a = False
result = ncd_evaluation.cross(x_train_encoded, genre_dict, transformations, 
                                     generator, grid, different_genre_a, amt1=amt1, amt2=amt2,
                                     compare_to_target=False, v=2)
print(len(list(result.keys())))
io.save(result, config.plots_dir + 'ncd_evaluation_AA.pkl')
# utils.reload(ncd_evaluation)
different_genre_a = False
result = ncd_evaluation.cross(x_train_encoded, genre_dict, min_transformations, 
                                     generator, grid, different_genre_a, amt1=amt1, amt2=amt2,
                                     compare_to_target=False, v=0)
print(len(list(result.keys())))
io.save(result, config.plots_dir + 'ncd_evaluation_AA_min.pkl')<jupyter_output>71<jupyter_text>One to one transformations<jupyter_code># dn = config.plots_dir
# # best_dims = io.load(dn + 'transformations-one-to-one-best_dims.pkl')
# transformations = io.load(dn + 'transformations-one-to-one.pkl')
# min_transformations = io.load(dn + 'min_transformations-one-to-one.pkl')
# utils.reload(ncd_evaluation)
# different_genre_a = False
# result = ncd_evaluation.cross(x_train_encoded, label_dict, min_transformations, 
#                 generator, grid, different_genre_a, amt1=amt1, amt2=1, v=1)
# io.save(result, config.plots_dir + 'ncd_evaluation-one-to-one_AB_min.pkl')
# result = ncd_evaluation.cross(x_train_encoded, label_dict, transformations, 
#             generator, grid, different_genre_a, amt1=amt1, amt2=1, v=0)
# io.save(result, config.plots_dir + 'ncd_evaluation-one-to-one_AB.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1278409090909091, 'fraction_numerical': 0.024147727272727272, 'mean_word_length': 4.066187050359712, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '597804', 'license': 'MIT', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1340, 'n_words': 199, 'path': 'src/2b-analysis-evaluate-transformations.ipynb', 'repo_name': 'voschezang/pattern-recognition', 'size': '3520'}