<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010 <NAME> <<EMAIL>>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""
Automated tests for checking transformation algorithms (the models package).
"""

from __future__ import with_statement

import logging
import unittest
import os
import tempfile
import itertools
import bz2
from six import iteritems, iterkeys
from six.moves import xrange, zip as izip
from collections import namedtuple, Counter

import numpy as np

from gensim import utils, matutils
from gensim.models import doc2vec
from gensim.models import word2vec
from gensim.models.doc2vec import TaggedDocument

module_path = os.path.dirname(__file__) # needed because sample data files are located in the same folder
datapath = lambda fname: os.path.join(module_path, 'test_data', fname)

logger = logging.getLogger('gensim.test.test_doc2vec')


class DocsLeeCorpus(object):
    def __init__(self, string_tags=False):
        self.string_tags = string_tags

    def _tag(self, i):
        return i if not self.string_tags else '_*%d' % i

    def __iter__(self):
        with open(datapath('lee_background.cor')) as f:
            for i, line in enumerate(f):
                yield TaggedDocument(utils.simple_preprocess(line),[self._tag(i)])

list_corpus = list(DocsLeeCorpus())

sentences = [
        ['human', 'interface', 'computer'],
        ['survey', 'user', 'computer', 'system', 'response', 'time'],
        ['eps', 'user', 'interface', 'system'],
        ['system', 'human', 'system', 'eps'],
        ['user', 'response', 'time'],
        ['trees'],
        ['graph', 'trees'],
        ['graph', 'minors', 'trees'],
        ['graph', 'minors', 'survey']
    ]

sentences = [TaggedDocument(words,[i]) for i, words in enumerate(sentences)]


def testfile():
    # temporary data will be stored to this file
    return os.path.join(tempfile.gettempdir(), 'gensim_doc2vec.tst')


class TestDoc2VecModel(unittest.TestCase):
    def test_persistence(self):
        """Test storing/loading the entire model."""
        model = doc2vec.Doc2Vec(DocsLeeCorpus(), min_count=1)
        model.save(testfile())
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile()))

    def test_load_mmap(self):
        """Test storing/loading the entire model."""
        model = doc2vec.Doc2Vec(sentences, min_count=1)

        # test storing the internal arrays into separate files
        model.save(testfile(), sep_limit=0)
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile()))

        # make sure mmaping the arrays back works, too
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile(), mmap='r'))

    def test_int_doctags(self):
        """Test doc2vec doctag alternatives"""
        corpus = DocsLeeCorpus()

        model = doc2vec.Doc2Vec(min_count=1)
        model.build_vocab(corpus)
        self.assertEqual(len(model.docvecs.doctag_syn0),300)
        self.assertEqual(model.docvecs[0].shape,(300,))
        self.assertRaises(KeyError,model.__getitem__,'_*0')

    def test_string_doctags(self):
        """Test doc2vec doctag alternatives"""
        corpus = DocsLeeCorpus(True)

        model = doc2vec.Doc2Vec(min_count=1)
        model.build_vocab(corpus)
        self.assertEqual(len(model.docvecs.doctag_syn0),300)
        self.assertEqual(model.docvecs[0].shape,(300,))
        self.assertEqual(model.docvecs['_*0'].shape,(300,))
        self.assertTrue(all(model.docvecs['_*0']==model.docvecs[0]))

    def test_empty_errors(self):
        # no input => "RuntimeError: you must first build vocabulary before training the model"
        self.assertRaises(RuntimeError, doc2vec.Doc2Vec, [])

        # input not empty, but rather completely filtered out
        self.assertRaises(RuntimeError, doc2vec.Doc2Vec, list_corpus, min_count=10000)

    def model_sanity(self, model):
        """Any non-trivial model on DocsLeeCorpus can pass these sanity checks"""
        fire1 = 0  # doc 0 sydney fires
        fire2 = 8  # doc 8 sydney fires
        tennis1 = 6  # doc 6 tennis

        # inferred vector should be top10 close to bulk-trained one
        doc0_inferred = model.infer_vector(list(DocsLeeCorpus())[0].words)
        sims_to_infer = model.docvecs.most_similar([doc0_inferred])
        self.assertTrue(fire1 in [match[0] for match in sims_to_infer])

        # fire8 should be top20 close to fire1
        sims = model.docvecs.most_similar(fire1,topn=20)
        sims = [(idx, round(dist,4)) for idx, dist in sims]
        self.assertTrue(fire2 in [match[0] for match in sims])

        # same sims should appear in lookup by vec as by index
        doc0_vec = model.docvecs[fire1]
        sims2 = model.docvecs.most_similar(positive=[doc0_vec], topn=21)
        sims2 = [(idx, round(dist,4)) for idx, dist in sims2]
        self.assertEqual(sims, sims2[1:])  # ignore first element of sims2, which is doc itself

        # tennis doc should be out-of-place among fire news
        self.assertEqual(model.docvecs.doesnt_match([fire1, tennis1, fire2]), tennis1) 

        # fire docs should be closer than fire-tennis
        self.assertTrue(model.docvecs.similarity(fire1,fire2) > model.docvecs.similarity(fire1,tennis1))
<commit_msg>np.allclose for float checks<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2010 <NAME> <<EMAIL>>
# Licensed under the GNU LGPL v2.1 - http://www.gnu.org/licenses/lgpl.html

"""
Automated tests for checking transformation algorithms (the models package).
"""

from __future__ import with_statement

import logging
import unittest
import os
import tempfile
import itertools
import bz2
from six import iteritems, iterkeys
from six.moves import xrange, zip as izip
from collections import namedtuple, Counter

import numpy as np

from gensim import utils, matutils
from gensim.models import doc2vec
from gensim.models import word2vec
from gensim.models.doc2vec import TaggedDocument

module_path = os.path.dirname(__file__) # needed because sample data files are located in the same folder
datapath = lambda fname: os.path.join(module_path, 'test_data', fname)

logger = logging.getLogger('gensim.test.test_doc2vec')


class DocsLeeCorpus(object):
    def __init__(self, string_tags=False):
        self.string_tags = string_tags

    def _tag(self, i):
        return i if not self.string_tags else '_*%d' % i

    def __iter__(self):
        with open(datapath('lee_background.cor')) as f:
            for i, line in enumerate(f):
                yield TaggedDocument(utils.simple_preprocess(line),[self._tag(i)])

list_corpus = list(DocsLeeCorpus())

sentences = [
        ['human', 'interface', 'computer'],
        ['survey', 'user', 'computer', 'system', 'response', 'time'],
        ['eps', 'user', 'interface', 'system'],
        ['system', 'human', 'system', 'eps'],
        ['user', 'response', 'time'],
        ['trees'],
        ['graph', 'trees'],
        ['graph', 'minors', 'trees'],
        ['graph', 'minors', 'survey']
    ]

sentences = [TaggedDocument(words,[i]) for i, words in enumerate(sentences)]


def testfile():
    # temporary data will be stored to this file
    return os.path.join(tempfile.gettempdir(), 'gensim_doc2vec.tst')


class TestDoc2VecModel(unittest.TestCase):
    def test_persistence(self):
        """Test storing/loading the entire model."""
        model = doc2vec.Doc2Vec(DocsLeeCorpus(), min_count=1)
        model.save(testfile())
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile()))

    def test_load_mmap(self):
        """Test storing/loading the entire model."""
        model = doc2vec.Doc2Vec(sentences, min_count=1)

        # test storing the internal arrays into separate files
        model.save(testfile(), sep_limit=0)
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile()))

        # make sure mmaping the arrays back works, too
        self.models_equal(model, doc2vec.Doc2Vec.load(testfile(), mmap='r'))

    def test_int_doctags(self):
        """Test doc2vec doctag alternatives"""
        corpus = DocsLeeCorpus()

        model = doc2vec.Doc2Vec(min_count=1)
        model.build_vocab(corpus)
        self.assertEqual(len(model.docvecs.doctag_syn0),300)
        self.assertEqual(model.docvecs[0].shape,(300,))
        self.assertRaises(KeyError,model.__getitem__,'_*0')

    def test_string_doctags(self):
        """Test doc2vec doctag alternatives"""
        corpus = DocsLeeCorpus(True)

        model = doc2vec.Doc2Vec(min_count=1)
        model.build_vocab(corpus)
        self.assertEqual(len(model.docvecs.doctag_syn0),300)
        self.assertEqual(model.docvecs[0].shape,(300,))
        self.assertEqual(model.docvecs['_*0'].shape,(300,))
        self.assertTrue(all(model.docvecs['_*0']==model.docvecs[0]))

    def test_empty_errors(self):
        # no input => "RuntimeError: you must first build vocabulary before training the model"
        self.assertRaises(RuntimeError, doc2vec.Doc2Vec, [])

        # input not empty, but rather completely filtered out
        self.assertRaises(RuntimeError, doc2vec.Doc2Vec, list_corpus, min_count=10000)

    def model_sanity(self, model):
        """Any non-trivial model on DocsLeeCorpus can pass these sanity checks"""
        fire1 = 0  # doc 0 sydney fires
        fire2 = 8  # doc 8 sydney fires
        tennis1 = 6  # doc 6 tennis

        # inferred vector should be top10 close to bulk-trained one
        doc0_inferred = model.infer_vector(list(DocsLeeCorpus())[0].words)
        sims_to_infer = model.docvecs.most_similar([doc0_inferred])
        self.assertTrue(fire1 in [match[0] for match in sims_to_infer])

        # fire8 should be top20 close to fire1
        sims = model.docvecs.most_similar(fire1,topn=20)
        self.assertTrue(fire2 in [match[0] for match in sims])

        # same sims should appear in lookup by vec as by index
        doc0_vec = model.docvecs[fire1]
        sims2 = model.docvecs.most_similar(positive=[doc0_vec], topn=21)
        sims2 = sims2[1:] # ignore first element of sims2, which is doc itself
        self.assertEqual(list(zip(*sims))[0], list(zip(*sims2))[0])  # same doc ids
        self.assertTrue(np.allclose(list(zip(*sims))[1], list(zip(*sims2))[1]))  # close-enough dists

        # tennis doc should be out-of-place among fire news
        self.assertEqual(model.docvecs.doesnt_match([fire1, tennis1, fire2]), tennis1) 

        # fire docs should be closer than fire-tennis
        self.assertTrue(model.docvecs.similarity(fire1,fire2) > model.docvecs.similarity(fire1,tennis1))

{'fraction_non_alphanumeric': 0.12128666035950804, 'fraction_numerical': 0.021665089877010407, 'mean_word_length': 3.402748854643898, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583471', 'n_tokens_mistral': 3517, 'n_tokens_neox': 3357, 'n_words': 832}