<commit_before># -*- coding: utf-8 -*-

from uuid import uuid4

from tornadoes import ESConnection
from tornado import escape
from tornado.testing import AsyncTestCase, gen_test
from tornado.ioloop import IOLoop


class ESConnectionTestBase(AsyncTestCase):

    def setUp(self):
        super(ESConnectionTestBase, self).setUp()
        self.es_connection = ESConnection("localhost", "9200", self.io_loop)
        self._set_version()

    def tearDown(self):
        if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
            self.io_loop.close(all_fds=True)
        super(AsyncTestCase, self).tearDown()

    def _set_version(self):
        self.es_connection.get_by_path("/", self.stop)
        response = self.wait()
        response = escape.json_decode(response.body)
        version = response['version']['number']
        self.version = [int(n) for n in version.split('.') if n.isdigit()]

    def _set_count_query(self, query):
        if self.version[0] < 1:
            return query['query']
        return query


class TestESConnection(ESConnectionTestBase):

    def test_simple_search(self):
        self.es_connection.get_by_path("/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 1)
        self.assertEqual(response["hits"]["hits"][0]["_id"], u'http://localhost/noticia/2/fast')

    def test_search_for_specific_type_with_query(self):
        self.es_connection.search(callback=self.stop,
                                  source={"query": {"term": {"ID": "171171"}}},
                                  type="materia", index="teste")

        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 1)
        self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')

    def test_search_specific_index(self):
        self.es_connection.search(callback=self.stop, index="outroteste")
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 14)

    def test_search_apecific_type(self):
        self.es_connection.search(self.stop, type='galeria')
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 2)

    def test_should_access_specific_document(self):
        self.es_connection.get(index="teste", type="materia", uid="171171", callback=self.stop)
        response = self.wait()["_source"]
        self.assertEqual(response['Portal'], "G1")
        self.assertEqual(response['Macrotema'], "Noticias")

    def test_should_accumulate_searches_before_search(self):
        source = {"query": {"text": {"_id": "171171"}}}
        self.es_connection.multi_search("teste", source=source)
        source = {"query": {"text": {"body": "multisearch"}}}
        self.es_connection.multi_search("neverEndIndex", source=source)

        self.assertListEqual(['{"index": "teste"}\n{"query": {"text": {"_id": "171171"}}}',
                              '{"index": "neverEndIndex"}\n{"query": {"text": {"body": "multisearch"}}}'
                              ], self.es_connection.bulk.bulk_list)

    def test_should_generate_empty_header_with_no_index_specified(self):
        source = {"query": {"text": {"_id": "171171"}}}
        self.es_connection.multi_search(index=None, source=source)
        source = {"query": {"text": {"body": "multisearch"}}}
        self.es_connection.multi_search(index=None, source=source)

        self.assertListEqual(['{}\n{"query": {"text": {"_id": "171171"}}}',
                              '{}\n{"query": {"text": {"body": "multisearch"}}}'
                              ], self.es_connection.bulk.bulk_list)

    def test_should_make_two_searches(self):
        self._make_multisearch()
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'], "171171")
        self.assertFalse("hits" in response['responses'][1])
<commit_msg>refact(tests.multi_search): use term instead of text<commit_after># -*- coding: utf-8 -*-

from uuid import uuid4

from tornadoes import ESConnection
from tornado import escape
from tornado.testing import AsyncTestCase, gen_test
from tornado.ioloop import IOLoop


class ESConnectionTestBase(AsyncTestCase):

    def setUp(self):
        super(ESConnectionTestBase, self).setUp()
        self.es_connection = ESConnection("localhost", "9200", self.io_loop)
        self._set_version()

    def tearDown(self):
        if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
            self.io_loop.close(all_fds=True)
        super(AsyncTestCase, self).tearDown()

    def _set_version(self):
        self.es_connection.get_by_path("/", self.stop)
        response = self.wait()
        response = escape.json_decode(response.body)
        version = response['version']['number']
        self.version = [int(n) for n in version.split('.') if n.isdigit()]

    def _set_count_query(self, query):
        if self.version[0] < 1:
            return query['query']
        return query


class TestESConnection(ESConnectionTestBase):

    def test_simple_search(self):
        self.es_connection.get_by_path("/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 1)
        self.assertEqual(response["hits"]["hits"][0]["_id"], u'http://localhost/noticia/2/fast')

    def test_search_for_specific_type_with_query(self):
        self.es_connection.search(callback=self.stop,
                                  source={"query": {"term": {"ID": "171171"}}},
                                  type="materia", index="teste")

        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 1)
        self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')

    def test_search_specific_index(self):
        self.es_connection.search(callback=self.stop, index="outroteste")
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 14)

    def test_search_apecific_type(self):
        self.es_connection.search(self.stop, type='galeria')
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response["hits"]["total"], 2)

    def test_should_access_specific_document(self):
        self.es_connection.get(index="teste", type="materia", uid="171171", callback=self.stop)
        response = self.wait()["_source"]
        self.assertEqual(response['Portal'], "G1")
        self.assertEqual(response['Macrotema'], "Noticias")

    def test_should_accumulate_searches_before_search(self):
        source = {"query": {"term": {"_id": "171171"}}}
        self.es_connection.multi_search("teste", source=source)
        source = {"query": {"term": {"body": "multisearch"}}}
        self.es_connection.multi_search("neverEndIndex", source=source)

        self.assertListEqual(['{"index": "teste"}\n{"query": {"term": {"_id": "171171"}}}',
                              '{"index": "neverEndIndex"}\n{"query": {"term": {"body": "multisearch"}}}'
                              ], self.es_connection.bulk.bulk_list)

    def test_should_generate_empty_header_with_no_index_specified(self):
        source = {"query": {"term": {"_id": "171171"}}}
        self.es_connection.multi_search(index=None, source=source)
        source = {"query": {"term": {"body": "multisearch"}}}
        self.es_connection.multi_search(index=None, source=source)

        self.assertListEqual(['{}\n{"query": {"term": {"_id": "171171"}}}',
                              '{}\n{"query": {"term": {"body": "multisearch"}}}'
                              ], self.es_connection.bulk.bulk_list)

    def test_should_make_two_searches(self):
        self._make_multisearch()
        response = self._verify_status_code_and_return_response()
        self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'], "171171")
        self.assertFalse("hits" in response['responses'][1])

{'fraction_non_alphanumeric': 0.15695773283274797, 'fraction_numerical': 0.016713091922005572, 'mean_word_length': 3.35778364116095, 'pattern_counts': {'":': 58, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2750705', 'n_tokens_mistral': 2610, 'n_tokens_neox': 2527, 'n_words': 391}