<jupyter_start><jupyter_text>IntroductionIn this example notebook, we want to implement DeepWalk algorithm for graph embedding. Zachry's karate club dataset. It is a directed graph and plotted as following.<jupyter_code>plt.figure(figsize=(10, 10))
nx.draw(nx_graph, cmap = plt.get_cmap('jet'), with_labels = True, node_color='red')<jupyter_output><empty_output><jupyter_text>Generate path and training data<jupyter_code>from keras.preprocessing.sequence import skipgrams
def batch_generator(file_path = file_path, window_size = 4, path_len = 10):
    nx_graph = nx.readwrite.adjlist.read_adjlist(file_path, create_using=nx.DiGraph())
    ig_graph = load_adjlist(file_path)
    vocab_size = len(nx_graph.nodes)
    sentences = [ig_graph.random_walk(k, path_len) for k in range(vocab_size)]
    for sent in sentences:
        couples, labels = skipgrams(sent, vocab_size, window_size = window_size, negative_samples = 1.0)
        yield couples, labels<jupyter_output><empty_output><jupyter_text>Build modelIn this section, we borrow the model we used in word2vec example and apply it directly to this case. We will use __negative sampling__ and binary loss. To simplify the implementation, I ignored the difference in probability of being sampled in this implementation. To enhence that part, please refer to __sampling_table__ in __skipgram__ function<jupyter_code>from keras.models import Model
from keras.layers import Input, Flatten, Embedding, dot, Activation
def build_model(input_dim, embedding_dim):
    target_input_token = Input(shape=[1])
    context_input_token = Input(shape=[1])
    
    target_embed = Embedding(output_dim = embedding_dim, \
                             input_dim = input_dim, input_length = 1, \
                             name = "target_embed")(target_input_token)
    context_embed = Embedding(output_dim = embedding_dim, \
                             input_dim = input_dim, input_length = 1, \
                             name = "context_embed")(context_input_token)
    
    target_vec = Flatten()(target_embed)
    context_vec = Flatten()(context_embed)
    
    y = Activation('sigmoid')(dot([target_vec, context_vec], axes = 1))
    model = Model(inputs=[target_input_token, context_input_token], outputs = [y])
    model.compile(loss = 'binary_crossentropy', optimizer = 'rmsprop')
    
    return model
def train_model(model, file_path = file_path, epochs = 3):
    for epoch in range(epochs):
        loss = 0
        print("{}/{} epochs".format(epoch, epochs))
        for couples, labels in batch_generator(file_path = file_path):
            words, contexts = zip(*couples)
            words = np.array(words, dtype=np.int32)
            contexts = np.array(contexts, dtype=np.int32)
            y = np.array(labels, dtype=np.int32)
            loss += model.train_on_batch([words, contexts], y)
        print("loss: {}".format(loss))
    print("Training finished")<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>node_num = len(nx_graph.nodes)
model = build_model(node_num, 2)
model.summary()
train_model(model, file_path)
vectors = model.get_weights()[0]
plt.figure(figsize=(10, 10))
plt.scatter(vectors[:, 0], vectors[:, 1])
for idx, [x, y] in enumerate(vectors):
    plt.text(x, y, idx + 1, fontsize = 15)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10516772438803264, 'fraction_numerical': 0.010275007555152614, 'mean_word_length': 4.0, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '281125', 'license': 'MIT', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1027, 'n_words': 255, 'path': 'Graph-Embedding/.ipynb_checkpoints/deepwalk-checkpoint.ipynb', 'repo_name': 'xiaozhi-alan-zhu/ML-practice-repo', 'size': '3309'}