<jupyter_start><jupyter_text>Sample paths strictly using node2vec<jupyter_code>%load_ext autoreload

%autoreload 2
import sys
sys.path.insert(0, '../')
from src.features import n2v
# del n2v
n2v = n2v.Node2Vec(A_tr, B_tr, P_tr)
n2v.perform_one_walk_full()
n2v.perform_one_walk_metapath()
walks = n2v.save_corpus()
%load_ext line_profiler
!wc -l node2vec_n=1_p=2_q=1_wl=100.cor
from gensim import utils

class MyCorpus(object):
    """An interator that yields sentences (lists of str)."""

    def __iter__(self):
        corpus_path = 'node2vec_n=1_p=2_q=1_wl=100.cor'
        for line in open(corpus_path):
            # assume there's one document per line, tokens separated by whitespace
            yield line.strip().split(' ')
sentences = MyCorpus()
%%time

import gensim.models

sentences = MyCorpus()
model = gensim.models.Word2Vec(sentences=sentences, min_count=1, size=200)
!ls HinDroid-with-Embeddings/data/processed/
meta_tr = pd.read_csv('HinDroid-with-Embeddings/data/processed/meta_tr.csv', index_col=0)
meta_tr.head()
y_train = meta_tr.label == 'class1'
app_vec = np.array([model.wv[f'app_{i}'] for i in range(len(meta_tr))])
app_vec
from sklearn.svm import SVC
svm = SVC(kernel='linear')
svm.fit(app_vec, y_train)
svm.score(app_vec, y_train)
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
%matplotlib inline
def tsne_plot(model):
    "Creates and TSNE model and plots it"
    labels = []
    tokens = []

    for word in model.wv.vocab:
        if 'api' in word: continue
        tokens.append(model.wv[word])
        labels.append(word)
    
    tsne_model = TSNE(n_components=2)
    new_values = tsne_model.fit_transform(tokens)

    x = []
    y = []
    for value in new_values:
        x.append(value[0])
        y.append(value[1])
        
    plt.figure(figsize=(16, 12)) 
    for i in range(len(x)):
        color = 'b' if meta_tr.label.iloc[i] == 'class1' else 'r'
        plt.scatter(x[i],y[i],c=color)
#         plt.annotate(labels[i],
#                      xy=(x[i], y[i]),
#                      xytext=(5, 2),
#                      textcoords='offset points',
#                      ha='right',
#                      va='bottom')
    plt.show()
tsne_plot(model)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1319073083778966, 'fraction_numerical': 0.01871657754010695, 'mean_word_length': 3.1268382352941178, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '599878', 'license': 'MIT', 'n_tokens_mistral': 865, 'n_tokens_neox': 834, 'n_words': 170, 'path': 'notebooks/node2vec_StellarGraph.ipynb', 'repo_name': 'davidzyx/HinDroid-with-Embeddings', 'size': '2244'}