<commit_before># -*- coding: utf-8 -*-
import logging
import sys
import os

import pandas as pd
from sklearn.manifold import TSNE
from gensim.models import KeyedVectors

from nlpia.constants import DATA_PATH
from nlpia.data.loaders import get_data


def stdout_logging(loglevel=logging.INFO):
    """Setup basic logging

    Args:
      loglevel (int): minimum loglevel for emitting messages
    """
    logformat = "[%(asctime)s] %(levelname)s:%(name)s:%(lineno)d: %(message)s"

    logging.config.dictConfig(level=loglevel, stream=sys.stdout,
                              format=logformat, datefmt="%Y-%m-%d %H:%M:%S")


def embed_wordvecs(w2v=None, df=None, vocab='name', embedder=TSNE, **kwargs):
    w2v = os.path.join(DATA_PATH, 'GoogleNews-vectors-negative300.bin') if w2v is None else w2v
    try:
        model = KeyedVectors.load_word2vec_format(w2v, binary=True) if isinstance(w2v, str) else w2v
    except IOError:
        model = os.path.join(DATA_PATH, w2v)
        model = KeyedVectors.loadWord2Vec.load_word2vec_format(model, binary=True)
    if df is None:
        df = get_data('cities')
    if isinstance(vocab, str) and vocab in df.columns:
        vocab = set([s.replace(' ', '_') for s in vocab.name] + [s.replace(' ', '_') for s in df.country])

    vocab = [word for word in vocab if word in model.wv]
    vectors = pd.DataFrame([model.wv[word] for word in vocab], index=vocab, columns=range(300))
    tsne = embedder(**kwargs)
    tsne = tsne.fit(vectors)
    return pd.DataFrame(tsne.embedding_, columns=['x', 'y'])


"""
Trial and error to produce the steps encoded in embed_vectors() above

```python
'xy'.split()
embeddings = pd.DataFrame(tsne.embedding_, columns=list('xy'))
embeddings
embeddings.plot(kind='scatter', x='x', y='y')
plt.show()
embeddings = pd.DataFrame(tsne.embedding_, columns=list('xy'), index=vocab)
embeddings
!pip install seaborn
from seborn import plt
from seaborn import plt
from seaborn import plt
plt = seaborn.plt<commit_msg>new took that uses annoy to get most similar wordvecs<commit_after># -*- coding: utf-8 -*-
import logging
import sys
import os

import pandas as pd
import numpy as np
from sklearn.manifold import TSNE
from gensim.models import KeyedVectors

from nlpia.constants import DATA_PATH
from nlpia.data.loaders import get_data
from nlpia.loaders import nlp, _parse, WV, WV_IDS, WORD2ID, VOCAB, ANN


def stdout_logging(loglevel=logging.INFO):
    """Setup basic logging

    Args:
      loglevel (int): minimum loglevel for emitting messages
    """
    logformat = "[%(asctime)s] %(levelname)s:%(name)s:%(lineno)d: %(message)s"

    logging.config.dictConfig(level=loglevel, stream=sys.stdout,
                              format=logformat, datefmt="%Y-%m-%d %H:%M:%S")


def embed_wordvecs(w2v=None, df=None, vocab='name', embedder=TSNE, **kwargs):
    w2v = os.path.join(DATA_PATH, 'GoogleNews-vectors-negative300.bin') if w2v is None else w2v
    try:
        model = KeyedVectors.load_word2vec_format(w2v, binary=True) if isinstance(w2v, str) else w2v
    except IOError:
        model = os.path.join(DATA_PATH, w2v)
        model = KeyedVectors.loadWord2Vec.load_word2vec_format(model, binary=True)
    if df is None:
        df = get_data('cities')
    if isinstance(vocab, str) and vocab in df.columns:
        vocab = set([s.replace(' ', '_') for s in vocab.name] + [s.replace(' ', '_') for s in df.country])

    vocab = [word for word in vocab if word in model.wv]
    vectors = pd.DataFrame([model.wv[word] for word in vocab], index=vocab, columns=range(300))
    tsne = embedder(**kwargs)
    tsne = tsne.fit(vectors)
    return pd.DataFrame(tsne.embedding_, columns=['x', 'y'])


def most_similar(tok, num_similar=20):
    """ FIXME: Use annoy to index word vectors and find most similar words to token str, id, or vector """
    raise NotImplementedError("Work in Progress, FIXME!")

    if _parse is None:
        nlp("hello", lang='en_core_web_lg')
        # FIXME: is the _parse variable updated by this, will WV and other globals work?
    stem = None
    idx = None
    vec = None

    if isinstance(tok, str):
        # TODO: if the tok is long or has punctuation/whitespace in it, use the spacy pipeline to tokenize it and compute the docvec
        stem = tok
        while stem not in WV_IDS:
            stem = stem[:-1]
        idx = WORD2ID[stem]
    elif isinstance(tok, int):
        idx = tok
        stem = VOCAB[idx].text
    else:
        vec = np.array(tok)
        if len(vec.shape) == 2:
            if vec.shape[0] == WV.shape[1]:
                vec = vec.mean(axis=0)
            else:
                vec = vec.mean(axis=1)
        idx = ANN.get_nns_by_vector(vec, 1)[0]
        stem = VOCAB[idx].text
    if idx is not None:
        vec = WV[idx]
        stem = VOCAB[idx].text
    return ANN.get_nns_by_vector(vec, num_similar)


"""
Trial and error to produce the steps encoded in embed_vectors() above

```python
'xy'.split()
embeddings = pd.DataFrame(tsne.embedding_, columns=list('xy'))
embeddings
embeddings.plot(kind='scatter', x='x', y='y')
plt.show()
embeddings = pd.DataFrame(tsne.embedding_, columns=list('xy'), index=vocab)
embeddings
!pip install seaborn
from seborn import plt
from seaborn import plt
from seaborn import plt
plt = seaborn.plt
{'fraction_non_alphanumeric': 0.1236722306525038, 'fraction_numerical': 0.009104704097116844, 'mean_word_length': 3.4125523012552303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4767859', 'n_tokens_mistral': 1853, 'n_tokens_neox': 1800, 'n_words': 500}