<jupyter_start><jupyter_text>embeddings<jupyter_code>model = DeepWalk()
model.fit(G)
# np.save("training_graph_embeddings.npy", model.get_embedding())
embeddings = np.load(KG = nx.read_gpickle(root_dir+"\\data\\embeddings\\training_KG_concepts.gpickle")training_graph_embeddings.npy")
def retrieve_spl(node1, node2):
    """
    Retrieves all shortest path lengths from a particular node.
    """
    node = node1
    file_ranges = [
        [int(i) for i in s.split("_")[:2]] \
        for s \
        in os.listdir("shortest_paths")
    ]
    for r in file_ranges:
        if r[0]<=node<=r[1]:
            file = root_dir+"\\data\\shortest_paths\\%d_%d_lengths.txt"%(r[0], r[1])
            break
    line = node - r[0] + 1
    lengths = linecache.getline(
        file,
        line)
    return [
        int(i) \
        if i != "nan" \
        else np.nan
        for i \
        in lengths[1:-2].split(", ")
    ][node2]<jupyter_output><empty_output><jupyter_text>In for a fair head-to-head comparison, we can't use so many edges<jupyter_code>n = len(G)
embeddings = model.get_embedding()
all_edges = G.edges
positive_samples = int(len(all_edges)*0.1) #34668
X_train_indices = random.sample(all_edges, positive_samples)
while len(X_train_indices) < positive_samples*2:
    edge = (random.randint(0, n-1), random.randint(0, n-1))
    if edge not in all_edges:
        X_train_indices.append(edge)
y_train = [1]*positive_samples + [0]*positive_samples
c = list(zip(X_train_indices, y_train))
random.shuffle(c)
X_train_indices, y_train = zip(*c)
X_train_embeddings = [
    np.concatenate([embeddings[tup[0]], embeddings[tup[1]]]) \
    for tup in X_train_indices
]
np.save(root_dir+"\\data\\training\\X_train_indices_69336.npy", X_train_indices)
np.save(root_dir+"\\data\\training\\X_train_embeddings_69336.npy", X_train_embeddings)
np.save(root_dir+"\\data\\training\\y_train_69336.npy", y_train)
X_train_embeddings = np.load(root_dir+"\\data\\training\\X_train_embeddings_69336.npy").tolist()
y_train = np.load(root_dir+"\\data\\training\\y_train_69336.npy").tolist()
clf_embeddings_69336_mlp = MLPClassifier((64, 12))
clf_embeddings_69336_mlp.fit(X_train_embeddings, y_train)
joblib.dump(clf_embeddings_69336_mlp, root_dir+"\\data\\classifiers\\deepwalk_mlp_69336.sav")
roc_auc_score(y_test, clf_embeddings_69336_mlp.predict_proba(X_test_embeddings)[:, 1])
average_precision_score(y_test, clf_embeddings_69336_mlp.predict_proba(X_test_embeddings)[:, 1])<jupyter_output><empty_output><jupyter_text>Get Geodesic data for X_train_indices. Using 90-percentile effective diameter where disconnected https://raw.githubusercontent.com/kunegis/konect-handbook/master/konect-handbook.pdf (page 30) - the number of edgesneeded on average to reach 90% of all other nodes<jupyter_code>X_train_indices = np.load(root_dir+"\\data\\training\\X_train_indices_69336.npy").tolist()
y_train = np.load(root_dir+"\\data\\training\\y_train_69336.npy").tolist()
H = G.copy()
betweenness_dict_10000 = np.load(root_dir+"\\data\\betweenness\\betweenness_dict_10000.npy", allow_pickle=True).item()
edges_to_remove = [
    X_train_indices[i] \
    for i in range(len(X_train_indices)) \
    if y_train[i] == 1
]
H.remove_edges_from(edges_to_remove)
X_train_geodesic = []
for edge in X_train_indices:
    if edge in edges_to_remove:
        try:
            path_len = nx.astar_path_length(H, edge[0], edge[1])
        except:
            path_len = 5 #90-percentile effective diameter
    else:
        path_len = retrieve_spl(edge[0], edge[1])
        
    tup = [
        betweenness_dict_10000[list(KG.nodes)[edge[0]]],
        betweenness_dict_10000[list(KG.nodes)[edge[1]]],
        path_len
    ]
    X_train_geodesic.append(tup)
clf_geodesic_69336 = GradientBoostingClassifier(
    n_estimators=100,
    random_state=0
)
clf_geodesic_69336.fit(X_train_geodesic, y_train)
joblib.dump(clf_geodesic_69336, root_dir+"\\data\\classifiers\\geodesic_gbc_69336.sav")
# joblib.load("geodesic_gbc_69336.sav")<jupyter_output><empty_output><jupyter_text>Get test data<jupyter_code>all_nodes = list(KG.nodes)
edges_cc = pd.read_csv(root_dir+"\\data\\og\\edges_cc.csv")
all_edges = [
    (edges_cc.src[i], edges_cc.dst[i]) \
    for i \
    in range(len(edges_cc))
]
training_edges = list(KG.edges())
training_edges += [i[::-1] for i in training_edges]
validation_edges = list(set(all_edges)-set(training_edges))
validation_edges = [
    pair for pair \
    in validation_edges \
    if np.nan not in pair \
    and pair[0] in all_nodes \
    and pair[1] in all_nodes
]
X_test_indices = [(all_nodes.index(pair[0]), all_nodes.index(pair[1])) for pair in validation_edges]
n = len(X_test_indices)
#obtain negative examples
while len(X_test_indices)< 2*n:
    edge = (random.randint(0, len(all_nodes)-1), random.randint(0, len(all_nodes)-1))
    if (
        edge not in X_test_indices \
        and edge[::-1] not in X_test_indices \
        and list(edge) not in X_train_indices \
        and list(edge[::-1]) not in X_train_indices
    ):
        X_test_indices.append(edge)
y_test = [1]*n + [0]*n
np.save(root_dir+"\\data\\validation\\X_test_indices.npy", X_test_indices)
np.save(root_dir+"\\data\\validation\\y_test.npy", y_test)
X_test_embeddings = [
    np.concatenate([embeddings[tup[0]], embeddings[tup[1]]]) \
    for tup in X_test_indices
]
np.save(root_dir+"\\data\\validation\\X_test_embeddings.npy", X_test_embeddings)
X_test_geodesic = []
for edge in X_test_indices:
    path_len = retrieve_spl(edge[0], edge[1])
    if path_len != path_len:
        path_len = 5
    
    tup = [
        betweenness_dict_10000[list(KG.nodes)[edge[0]]],
        betweenness_dict_10000[list(KG.nodes)[edge[1]]],
        path_len
    ]
    X_test_geodesic.append(tup)
np.save(root_dir+"\\data\\validation\\X_test_geodesic.npy", X_test_geodesic)
roc_auc_score(y_test, clf_geodesic_69336.predict_proba(X_test_geodesic)[:, 1])
average_precision_score(y_test, clf_geodesic_69336.predict_proba(X_test_geodesic)[:, 1])<jupyter_output><empty_output><jupyter_text>embedding + hadamard + lr<jupyter_code>X_train_embeddings_hadamard = [
    np.multiply(embedding[:128], embedding[128:])
    for embedding in X_train_embeddings
]
X_test_embeddings_hadamard = [
    np.multiply(embedding[:128], embedding[128:])
    for embedding in X_test_embeddings
]

clf_embeddings_69336_lr = LogisticRegressionCV()
clf_embeddings_69336_lr.fit(X_train_embeddings_hadamard, y_train)
roc_auc_score(y_test, clf_embeddings_69336.predict_proba(X_test_embeddings)[:, 1])
average_precision_score(y_test, clf_embeddings_69336_lr.predict_proba(X_test_embeddings_hadamard)[:, 1])
np.save(root_dir+"\\data\\training\\X_train_embeddings_hadamard.npy", X_train_embeddings_hadamard)
np.save(root_dir+"\\data\\validation\\X_test_embeddings_hadamard.npy", X_test_embeddings_hadamard)
joblib.dump(clf_embeddings_69336_lr, root_dir+"\\data\\classifiers\\deepwalk_lr_69336_lr.sav")<jupyter_output><empty_output><jupyter_text>visualization of predicted probabilities<jupyter_code>sns.kdeplot(clf_geodesic_69336.predict_proba(X_test_geodesic)[:, 1])
sns.kdeplot(clf_embeddings_69336_mlp.predict_proba(X_test_embeddings)[:, 1])
sns.kdeplot(clf_embeddings_69336_lr.predict_proba(X_test_embeddings_hadamard)[:, 1])
clf_embeddings_69336_lr.score(X_test_embeddings_hadamard, y_test)<jupyter_output><empty_output><jupyter_text>get top-ranked novel edges<jupyter_code>X_test_indices = np.load(root_dir+"\\data\\validation\\X_test_indices.npy").tolist()
test_probs = clf_geodesic_69336.predict_proba(X_test_geodesic)[:, 1]
def get_topk_novel(predict_proba, targets, X_test_indices, all_vertices, k):
    """
    """
    novel_probs = np.multiply(predict_proba, np.abs(np.array(targets)-1))
    top_indices = (-novel_probs).argsort()[:k].tolist()
    return [
        (all_vertices[X_test_indices[i][0]], all_vertices[X_test_indices[i][1]]) \
        for i in top_indices
    ], novel_probs[top_indices]
edges, probs = get_topk_novel(test_probs, y_test, X_test_indices, all_vertices, 1000)
data = [
    (edges[i][0]+" <-> "+edges[i][1], probs[i]) \
    for i in range(len(edges))
]
from tabulate import tabulate
header = ["Edge", "Estimated Link Probability"]
table = tabulate(data, headers = header, tablefmt = "grid", numalign = "center")
print(table)
with open(root_dir+"\\dev code\\link prediction\\novel_relations.txt", 'w') as f:
    f.write(table)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13785297271535804, 'fraction_numerical': 0.03455260335994281, 'mean_word_length': 4.991434689507495, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '510590', 'license': 'MIT', 'n_tokens_mistral': 3424, 'n_tokens_neox': 3243, 'n_words': 452, 'path': 'dev code/link prediction/Node Embeddings & Link Prediction.ipynb', 'repo_name': 'lucasmccabe/covid19-link-prediction', 'size': '8393'}