<commit_before>from __future__ import print_function, unicode_literals
import os
import six
import numpy as np

from gensim.models import Word2Vec
from sklearn.preprocessing import StandardScaler

from magpie.base.document import Document
from magpie.config import EMBEDDING_SIZE, WORD2VEC_WORKERS, MIN_WORD_COUNT, \
    WORD2VEC_CONTEXT
from magpie.utils import get_documents, save_to_disk


def train_word2vec_in_memory(docs, vec_dim=EMBEDDING_SIZE):
    """
    Builds word embeddings from documents and return a model
    :param docs: list of Document objects
    :param vec_dim: the dimensionality of the vector that's being built

    :return: trained gensim object with word embeddings
    """
    doc_sentences = map(lambda d: d.read_sentences(), docs)
    all_sentences = reduce(lambda d1, d2: d1 + d2, doc_sentences)

    # Initialize and train the model
    model = Word2Vec(
        all_sentences,
        workers=WORD2VEC_WORKERS,<commit_msg>Fixes 'NameError: reduce is not defined in Python' when attempting to use the in-memory train_word2vec in Python3.<commit_after>from __future__ import print_function, unicode_literals
import os
import six
import numpy as np
from functools import reduce

from gensim.models import Word2Vec
from sklearn.preprocessing import StandardScaler

from magpie.base.document import Document
from magpie.config import EMBEDDING_SIZE, WORD2VEC_WORKERS, MIN_WORD_COUNT, \
    WORD2VEC_CONTEXT
from magpie.utils import get_documents, save_to_disk


def train_word2vec_in_memory(docs, vec_dim=EMBEDDING_SIZE):
    """
    Builds word embeddings from documents and return a model
    :param docs: list of Document objects
    :param vec_dim: the dimensionality of the vector that's being built

    :return: trained gensim object with word embeddings
    """
    doc_sentences = map(lambda d: d.read_sentences(), docs)
    all_sentences = reduce(lambda d1, d2: d1 + d2, doc_sentences)

    # Initialize and train the model
    model = Word2Vec(
        all_sentences,
        workers=WORD2VEC_WORKERS,
{'fraction_non_alphanumeric': 0.055392156862745096, 'fraction_numerical': 0.010784313725490196, 'mean_word_length': 4.501347708894879, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4297417', 'n_tokens_mistral': 638, 'n_tokens_neox': 617, 'n_words': 205}