<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 <NAME> <<EMAIL>>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html


"""
Deep learning via word2vec's "hierarchical softmax skip-gram and CBOW models" [1]_.

The training algorithm was originally ported from the C package https://code.google.com/p/word2vec/
and extended with additional functionality.

For a blog tutorial on gensim word2vec, with an interactive web app trained on GoogleNews, visit http://radimrehurek.com/2014/02/word2vec-tutorial/

**Install Cython with `pip install cython` to use optimized word2vec training** (70x speedup [2]_).

Initialize a model with e.g.::

>>> model = Word2Vec(sentences, size=100, window=5, min_count=5, workers=4)

Persist a model to disk with::

>>> model.save(fname)
>>> model = Word2Vec.load(fname)  # you can continue training with the loaded model!

The model can also be instantiated from an existing file on disk in the word2vec C format::

  >>> model = Word2Vec.load_word2vec_format('/tmp/vectors.txt', binary=False)  # C text format
  >>> model = Word2Vec.load_word2vec_format('/tmp/vectors.bin', binary=True)  # C binary format

You can perform various syntactic/semantic NLP word tasks with the model. Some of them
are already built-in::

  >>> model.most_similar(positive=['woman', 'king'], negative=['man'])
  [('queen', 0.50882536), ...]

  >>> model.doesnt_match("breakfast cereal dinner lunch".split())
  'cereal'

  >>> model.similarity('woman', 'man')
  0.73723527

  >>> model['computer']  # raw numpy vector of a word
  array([-0.00449447, -0.00310097,  0.02421786, ...], dtype=float32)

and so on.

If you're finished training a model (=no more updates, only querying), you can do

  >>> model.init_sims(replace=True)

to trim unneeded model memory = use (much) less RAM.

.. [1] <NAME>, <NAME>, <NAME>, and <NAME>. Efficient Estimation of Word Representations in Vector Space. In Proceedings of Workshop at ICLR, 2013.
.. [2] Optimizing word2vec in gensim, http://radimrehurek.com/2013/09/word2vec-in-python-part-two-optimizing/
"""

import logging
import sys
import os
import heapq
import time
from copy import deepcopy
import threading
try:
    from queue import Queue
except ImportError:
    from Queue import Queue

from numpy import exp, dot, zeros, outer, random, dtype, get_include, float32 as REAL,\
    uint32, seterr, array, uint8, vstack, argsort, fromstring, sqrt, newaxis, ndarray, empty, sum as np_sum

logger = logging.getLogger("gensim.models.word2vec")


from gensim import utils, matutils  # utility fnc for pickling, common scipy operations etc
from six import iteritems, itervalues, string_types
from six.moves import xrange


try:
    from gensim_addons.models.word2vec_inner import train_sentence_sg, train_sentence_cbow, FAST_VERSION
except ImportError:
    try:
        # try to compile and use the faster cython version
        import pyximport
        models_dir = os.path.dirname(__file__) or os.getcwd()
        pyximport.install(setup_args={"include_dirs": [models_dir, get_include()]})
        from word2vec_inner import train_sentence_sg, train_sentence_cbow, FAST_VERSION
    except:
        # failed... fall back to plain numpy (20-80x slower training than the above)
        FAST_VERSION = -1

        def train_sentence_sg(model, sentence, alpha, work=None):
            """
            Update skip-gram hierarchical softmax model by training on a single sentence.

            The sentence is a list of Vocab objects (or None, where the corresponding
            word is not in the vocabulary. Called internally from `Word2Vec.train()`.

            """
            if model.negative:
                # precompute negative labels
                labels = zeros(model.negative+1)
                labels[0] = 1.
            for pos, word in enumerate(sentence):
                if word is None:
                    continue  # OOV word in the input sentence => skip
                reduced_window = random.randint(model.window)  # `b` in the original word2vec code

                # now go over all words from the (reduced) window, predicting each one in turn
                start = max(0, pos - model.window + reduced_window)
                for pos2, word2 in enumerate(sentence[start : pos + model.window + 1 - reduced_window], start):
                    # don't train on OOV words and on the `word` itself
                    if word2 and not (pos2 == pos):
                        l1 = model.syn0[word2.index]
                        neu1e = zeros(l1.shape)
                        if model.hs:
                            # work on the entire tree at once, to push as much work into numpy's C routines as possible (performance)
                            l2a = deepcopy(model.syn1[word.point])  # 2d matrix, codelen x layer1_size
                            fa = 1.0 / (1.0 + exp(-dot(l1, l2a.T)))  #  propagate hidden -> output
                            ga = (1 - word.code - fa) * alpha  # vector of error gradients multiplied by the learning rate
                            model.syn1[word.point] += outer(ga, l1)  # learn hidden -> output
                            neu1e += dot(ga, l2a) # save error
                        if model.negative:
                            # use this word (label = 1) + k other random words not from this sentence (label = 0)
                            word_indices = [word.index]
                            while len(word_indices) < model.negative+1:
                                w = model.table[random.randint(model.table.shape[0])]
                                if not (w == word.index or w == word2.index):
                                    word_indices.append(w)
                            # 2d matrix, k+1 x layer1_size
                            l2b = deepcopy(model.syn1neg[word_indices])
                            # propagate hidden -> output
                            fb = 1. / (1. + exp(-dot(l1, l2b.T)))
                            # vector of error gradients multiplied by the learning rate
                            gb = (labels - fb) * alpha
                            # learn hidden -> output
                            model.syn1neg[word_indices] += outer(gb, l1)
                            # learn input -> hidden
                            neu1e += dot(gb, l2b)
                        model.syn0[word2.index] += neu1e  # learn input -> hidden

            return len([word for word in sentence if word is not None])

        def train_sentence_cbow(model, sentence, alpha, work=None, neu1=None):
            """
            Update CBOW hierarchical softmax model by training on a single sentence.

            The sentence is a list of Vocab objects (or None, where the corresponding
            word is not in the vocabulary. Called internally from `Word2Vec.train()`.

            """
            if model.negative:
                # precompute negative labels
                labels = zeros(model.negative+1)
                labels[0] = 1.

            for pos, word in enumerate(sentence):
                if word is None:
                    continue  # OOV word in the input sentence => skip
                reduced_window = random.randint(model.window) # `b` in the original word2vec code
                start = max(0, pos - model.window + reduced_window)
                word2_indices = [word2.index for pos2, word2 in enumerate(sentence[start:pos+model.window+1-reduced_window], start) if (word2 and not (pos2 == pos))]
                l1 = np_sum(model.syn0[word2_indices],axis=0) # 1xlayer1_size
                if len(word2_indices) > 0:
                    l1 /= len(word2_indices)
                neu1e = zeros(l1.shape)

                if model.hs:
                    l2a = deepcopy(model.syn1[word.point]) #2d matrix, codelen x layer1_size
                    fa = 1. / (1. + exp(-dot(l1, l2a.T))) # propagate hidden -> output
                    ga = (1. - word.code - fa) * alpha # vector of error gradients multiplied by the learning rate
                    model.syn1[word.point] += outer(ga, l1) # learn hidden -> output
                    neu1e += dot(ga, l2a) # learn input -> hidden, here for all words in the window separately

                if model.negative:
                    # use this word (label = 1) + k other random words not from this sentence (label = 0)
                    word_indices = [word.index]
                    while len(word_indices) < model.negative+1:
                        w = model.table[random.randint(model.table.shape[0])]
                        if not (w == word.index or w in word2_indices):
                            word_indices.append(w)
                    l2b = deepcopy(model.syn1neg[word_indices]) # 2d matrix, k+1 x layer1_size
                    fb = 1. / (1. + exp(-dot(l1, l2b.T))) # propagate hidden -> output
                    gb = (labels - fb) * alpha # vector of error gradients multiplied by the learning rate
                    model.syn1neg[word_indices] += outer(gb, l1) # learn hidden -> output
                    neu1e += dot(gb, l2b) # learn input -> hidden, here for all words in the window separately

                model.syn0[word2_indices] += neu1e

            return len([word for word in sentence if word is not None])


class Vocab(object):
    """A single vocabulary item, used internally for constructing binary trees (incl. both word leaves and inner nodes)."""
    def __init__(self, **kwargs):
        self.count = 0
        self.__dict__.update(kwargs)

    def __lt__(self, other):  # used for sorting in a priority queue
        return self.count < other.count
<commit_msg>Remove additional constraint<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2013 <NAME> <<EMAIL>>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html


"""
Deep learning via word2vec's "hierarchical softmax skip-gram and CBOW models" [1]_.

The training algorithm was originally ported from the C package https://code.google.com/p/word2vec/
and extended with additional functionality.

For a blog tutorial on gensim word2vec, with an interactive web app trained on GoogleNews, visit http://radimrehurek.com/2014/02/word2vec-tutorial/

**Install Cython with `pip install cython` to use optimized word2vec training** (70x speedup [2]_).

Initialize a model with e.g.::

>>> model = Word2Vec(sentences, size=100, window=5, min_count=5, workers=4)

Persist a model to disk with::

>>> model.save(fname)
>>> model = Word2Vec.load(fname)  # you can continue training with the loaded model!

The model can also be instantiated from an existing file on disk in the word2vec C format::

  >>> model = Word2Vec.load_word2vec_format('/tmp/vectors.txt', binary=False)  # C text format
  >>> model = Word2Vec.load_word2vec_format('/tmp/vectors.bin', binary=True)  # C binary format

You can perform various syntactic/semantic NLP word tasks with the model. Some of them
are already built-in::

  >>> model.most_similar(positive=['woman', 'king'], negative=['man'])
  [('queen', 0.50882536), ...]

  >>> model.doesnt_match("breakfast cereal dinner lunch".split())
  'cereal'

  >>> model.similarity('woman', 'man')
  0.73723527

  >>> model['computer']  # raw numpy vector of a word
  array([-0.00449447, -0.00310097,  0.02421786, ...], dtype=float32)

and so on.

If you're finished training a model (=no more updates, only querying), you can do

  >>> model.init_sims(replace=True)

to trim unneeded model memory = use (much) less RAM.

.. [1] <NAME>, <NAME>, <NAME>, and <NAME>. Efficient Estimation of Word Representations in Vector Space. In Proceedings of Workshop at ICLR, 2013.
.. [2] Optimizing word2vec in gensim, http://radimrehurek.com/2013/09/word2vec-in-python-part-two-optimizing/
"""

import logging
import sys
import os
import heapq
import time
from copy import deepcopy
import threading
try:
    from queue import Queue
except ImportError:
    from Queue import Queue

from numpy import exp, dot, zeros, outer, random, dtype, get_include, float32 as REAL,\
    uint32, seterr, array, uint8, vstack, argsort, fromstring, sqrt, newaxis, ndarray, empty, sum as np_sum

logger = logging.getLogger("gensim.models.word2vec")


from gensim import utils, matutils  # utility fnc for pickling, common scipy operations etc
from six import iteritems, itervalues, string_types
from six.moves import xrange


try:
    from gensim_addons.models.word2vec_inner import train_sentence_sg, train_sentence_cbow, FAST_VERSION
except ImportError:
    try:
        # try to compile and use the faster cython version
        import pyximport
        models_dir = os.path.dirname(__file__) or os.getcwd()
        pyximport.install(setup_args={"include_dirs": [models_dir, get_include()]})
        from word2vec_inner import train_sentence_sg, train_sentence_cbow, FAST_VERSION
    except:
        # failed... fall back to plain numpy (20-80x slower training than the above)
        FAST_VERSION = -1

        def train_sentence_sg(model, sentence, alpha, work=None):
            """
            Update skip-gram hierarchical softmax model by training on a single sentence.

            The sentence is a list of Vocab objects (or None, where the corresponding
            word is not in the vocabulary. Called internally from `Word2Vec.train()`.

            """
            if model.negative:
                # precompute negative labels
                labels = zeros(model.negative+1)
                labels[0] = 1.
            for pos, word in enumerate(sentence):
                if word is None:
                    continue  # OOV word in the input sentence => skip
                reduced_window = random.randint(model.window)  # `b` in the original word2vec code

                # now go over all words from the (reduced) window, predicting each one in turn
                start = max(0, pos - model.window + reduced_window)
                for pos2, word2 in enumerate(sentence[start : pos + model.window + 1 - reduced_window], start):
                    # don't train on OOV words and on the `word` itself
                    if word2 and not (pos2 == pos):
                        l1 = model.syn0[word2.index]
                        neu1e = zeros(l1.shape)
                        if model.hs:
                            # work on the entire tree at once, to push as much work into numpy's C routines as possible (performance)
                            l2a = deepcopy(model.syn1[word.point])  # 2d matrix, codelen x layer1_size
                            fa = 1.0 / (1.0 + exp(-dot(l1, l2a.T)))  #  propagate hidden -> output
                            ga = (1 - word.code - fa) * alpha  # vector of error gradients multiplied by the learning rate
                            model.syn1[word.point] += outer(ga, l1)  # learn hidden -> output
                            neu1e += dot(ga, l2a) # save error
                        if model.negative:
                            # use this word (label = 1) + k other random words not from this sentence (label = 0)
                            word_indices = [word.index]
                            while len(word_indices) < model.negative+1:
                                w = model.table[random.randint(model.table.shape[0])]
                                if w != word.index:
                                    word_indices.append(w)
                            l2b = deepcopy(model.syn1neg[word_indices]) # 2d matrix, k+1 x layer1_size
                            fb = 1. / (1. + exp(-dot(l1, l2b.T))) # propagate hidden -> output
                            gb = (labels - fb) * alpha # vector of error gradients multiplied by the learning rate
                            model.syn1neg[word_indices] += outer(gb, l1) # learn hidden -> output
                            neu1e += dot(gb, l2b) # save error
                        model.syn0[word2.index] += neu1e  # learn input -> hidden

            return len([word for word in sentence if word is not None])

        def train_sentence_cbow(model, sentence, alpha, work=None, neu1=None):
            """
            Update CBOW hierarchical softmax model by training on a single sentence.

            The sentence is a list of Vocab objects (or None, where the corresponding
            word is not in the vocabulary. Called internally from `Word2Vec.train()`.

            """
            if model.negative:
                # precompute negative labels
                labels = zeros(model.negative+1)
                labels[0] = 1.

            for pos, word in enumerate(sentence):
                if word is None:
                    continue  # OOV word in the input sentence => skip
                reduced_window = random.randint(model.window) # `b` in the original word2vec code
                start = max(0, pos - model.window + reduced_window)
                word2_indices = [word2.index for pos2, word2 in enumerate(sentence[start:pos+model.window+1-reduced_window], start) if (word2 and not (pos2 == pos))]
                l1 = np_sum(model.syn0[word2_indices],axis=0) # 1xlayer1_size
                if len(word2_indices) > 0:
                    l1 /= len(word2_indices)
                neu1e = zeros(l1.shape)

                if model.hs:
                    l2a = deepcopy(model.syn1[word.point]) #2d matrix, codelen x layer1_size
                    fa = 1. / (1. + exp(-dot(l1, l2a.T))) # propagate hidden -> output
                    ga = (1. - word.code - fa) * alpha # vector of error gradients multiplied by the learning rate
                    model.syn1[word.point] += outer(ga, l1) # learn hidden -> output
                    neu1e += dot(ga, l2a) # save error

                if model.negative:
                    # use this word (label = 1) + k other random words not from this sentence (label = 0)
                    word_indices = [word.index]
                    while len(word_indices) < model.negative+1:
                        w = model.table[random.randint(model.table.shape[0])]
                        if w != word.index:
                            word_indices.append(w)
                    l2b = deepcopy(model.syn1neg[word_indices]) # 2d matrix, k+1 x layer1_size
                    fb = 1. / (1. + exp(-dot(l1, l2b.T))) # propagate hidden -> output
                    gb = (labels - fb) * alpha # vector of error gradients multiplied by the learning rate
                    model.syn1neg[word_indices] += outer(gb, l1) # learn hidden -> output
                    neu1e += dot(gb, l2b) # save error

                model.syn0[word2_indices] += neu1e # learn input -> hidden, here for all words in the window separately

            return len([word for word in sentence if word is not None])


class Vocab(object):
    """A single vocabulary item, used internally for constructing binary trees (incl. both word leaves and inner nodes)."""
    def __init__(self, **kwargs):
        self.count = 0
        self.__dict__.update(kwargs)

    def __lt__(self, other):  # used for sorting in a priority queue
        return self.count < other.count

{'fraction_non_alphanumeric': 0.0957974419211694, 'fraction_numerical': 0.023074915165753065, 'mean_word_length': 2.0343735149691113, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 105, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5597749', 'n_tokens_mistral': 5698, 'n_tokens_neox': 5267, 'n_words': 1807}