<jupyter_start><jupyter_text>Load autoencoder model<jupyter_code>device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
model = torch.load(fopen(config.autoencoder_model_path, "rb"), map_location=torch.device(device))
model.device = device<jupyter_output><empty_output><jupyter_text>Load data for fine-tuning and evaluation<jupyter_code>train, test = load_train_test([config.train_path, config.test_path])

input_names_train, weighted_actual_names_train, candidate_names_train = train
input_names_test, weighted_actual_names_test, candidate_names_test = test<jupyter_output><empty_output><jupyter_text>Compute near-negatives<jupyter_code>near_negatives_train = get_near_negatives(
    input_names_train, weighted_actual_names_train, candidate_names_train, k=50
)
# save near_negatives
with fopen(config.near_negatives_path, "wb") as f:
    pickle.dump(near_negatives_train, f)
# load near_negatives
with fopen(config.near_negatives_path, "rb") as f:
    near_negatives_train = pickle.load(f)
for ix, (key, values) in enumerate(near_negatives_train.items()):
    if ix > 3:
        break
    print(key, ":", " ".join(values))
ix = input_names_train.index("<ada>")
weighted_actual_names_train[ix]
print(len(input_names_train))
print(len(candidate_names_train))
print(len(input_names_test))
print(len(candidate_names_test))
batch_size = 512
train_triplet_loss(
    model,
    input_names_train,
    weighted_actual_names_train,
    near_negatives_train,
    input_names_test,
    weighted_actual_names_test,
    candidate_names_test,
    num_epochs=50,
    batch_size=batch_size,
    margin=.05,
    k=100,
    device=device,
)
torch.save(model, fopen(config.triplet_model_path, "wb"))
model = torch.load(fopen(config.triplet_model_path, "rb"), map_location=torch.device(device))<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code># metric=euclidean is what TripletMarginLoss optimizes by default
# but this means that scores will be in terms of distance, not similarity, so take this into account when computing PR at thresholds
k = 100
best_matches = get_best_autoencoder_matches(model, input_names_test, candidate_names_test, k, batch_size)<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>print(best_matches.shape)
print(best_matches[0, 0, 0])
print(best_matches[0, 0, 1])
best_matches_names = best_matches[:, :, 0]
print(best_matches_names.shape)<jupyter_output><empty_output><jupyter_text>PR Curve<jupyter_code># minimum score threshold to test
metrics.precision_weighted_recall_curve_at_threshold(
    weighted_actual_names_test, best_matches, min_threshold=0.01, max_threshold=5.0, step=0.05, distances=True
)<jupyter_output><empty_output><jupyter_text>AUC<jupyter_code>metrics.get_auc(
    weighted_actual_names_test, best_matches, min_threshold=0.01, max_threshold=5.0, step=0.05, distances=True
)<jupyter_output><empty_output><jupyter_text>Precision and recall at a threshold<jupyter_code>threshold = 1.2

print(
    "precision",
    metrics.avg_precision_at_threshold(weighted_actual_names_test, best_matches, threshold=threshold, distances=True),
)
print(
    "recall",
    metrics.avg_weighted_recall_at_threshold(
        weighted_actual_names_test, best_matches, threshold=threshold, distances=True
    ),
)
wandb.finish()
train_matches = get_best_autoencoder_matches(model, input_names_train, 
                                             candidate_names_train, k, batch_size, n_jobs=1)
metrics.precision_weighted_recall_curve_at_threshold(
    weighted_actual_names_train, train_matches, min_threshold=0.01, max_threshold=5.0, step=0.05, distances=True
)
metrics.get_auc(
    weighted_actual_names_train, train_matches, min_threshold=0.01, max_threshold=5.0, step=0.05, distances=True
)
sum(len(wan) for wan in weighted_actual_names_train)
input_names_train[:15]
import pandas as pd

pref_path = "s3://familysearch-names/processed/tree-preferred-given-aggr.csv.gz"
pref_df = pd.read_csv(pref_path)
top_names = set(["aafje", "aafke", "aage", "aagje", "aagot", "dallin", "dallan"])
pref_df[pref_df["name"].isin(top_names)]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10717749757516974, 'fraction_numerical': 0.015033947623666343, 'mean_word_length': 7.435582822085889, 'pattern_counts': {'":': 1, '<': 34, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '636830', 'license': 'MIT', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1440, 'n_words': 229, 'path': 'notebooks/51_autoencoder_triplet.ipynb', 'repo_name': 'rootsdev/nama', 'size': '4124'}