<commit_before>import uuid

from django.test import SimpleTestCase
from elasticsearch.exceptions import ConnectionError

from corehq.apps.export.esaccessors import get_case_export_base_query
from corehq.apps.export.export import (
    _get_export_documents,
)
from corehq.apps.export.filters import (
    GroupOwnerFilter,
    IsClosedFilter,
    OwnerFilter,
    OR,
    FormSubmittedByFilter,
)
from corehq.apps.export.models.new import (
    CaseExportInstance,
    FormExportInstance)
from corehq.apps.export.tests.util import (

    DEFAULT_USER,
    DEFAULT_CASE_TYPE,
    DEFAULT_XMLNS,
    DEFAULT_APP_ID,
    DOMAIN,
    new_case,
    new_form,
)
from corehq.apps.groups.models import Group
from corehq.pillows.case import CasePillow
from corehq.pillows.group import GroupPillow
from corehq.pillows.xform import XFormPillow
from corehq.util.elastic import ensure_index_deleted
from corehq.util.test_utils import trap_extra_setup
from pillowtop.es_utils import completely_initialize_pillow_index


class ExportFilterTest(SimpleTestCase):

    def test_or_filter(self):
        self.assertEqual(
            OR(OwnerFilter("foo"), OwnerFilter("bar")).to_es_filter(),
            {
                'or': (
                    {'term': {'owner_id': 'foo'}},
                    {'term': {'owner_id': 'bar'}}
                )
            }
        )


class ExportFilterResultTest(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        form_pillow = XFormPillow(online=False)
        case_pillow = CasePillow(online=False)
        group_pillow = GroupPillow(online=False)
        cls.pillows = [form_pillow, case_pillow, group_pillow]

        with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
            for pillow in cls.pillows:
                completely_initialize_pillow_index(pillow)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group.get_id
        group_pillow.send_robust(group.to_json())

        form = new_form(form={"meta": {"userID": None}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": ""}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"deviceID": "abc"}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": uuid.uuid4().hex}})
        form_pillow.send_robust(form.to_json())



        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)

    @classmethod
    def tearDownClass(cls):
        for pillow in cls.pillows:
            ensure_index_deleted(pillow.es_index)
<commit_msg>remove test dependency on GroupPillow<commit_after>import uuid

from django.test import SimpleTestCase
from elasticsearch.exceptions import ConnectionError

from corehq.apps.export.esaccessors import get_case_export_base_query
from corehq.apps.export.export import (
    _get_export_documents,
)
from corehq.apps.export.filters import (
    GroupOwnerFilter,
    IsClosedFilter,
    OwnerFilter,
    OR,
    FormSubmittedByFilter,
)
from corehq.apps.export.models.new import (
    CaseExportInstance,
    FormExportInstance)
from corehq.apps.export.tests.util import (

    DEFAULT_USER,
    DEFAULT_CASE_TYPE,
    DEFAULT_XMLNS,
    DEFAULT_APP_ID,
    DOMAIN,
    new_case,
    new_form,
)
from corehq.apps.groups.models import Group
from corehq.elastic import get_es_new, send_to_elasticsearch
from corehq.pillows.case import CasePillow
from corehq.pillows.mappings.group_mapping import GROUP_INDEX_INFO
from corehq.pillows.xform import XFormPillow
from corehq.util.elastic import ensure_index_deleted
from corehq.util.test_utils import trap_extra_setup
from pillowtop.es_utils import completely_initialize_pillow_index, initialize_index_and_mapping


class ExportFilterTest(SimpleTestCase):

    def test_or_filter(self):
        self.assertEqual(
            OR(OwnerFilter("foo"), OwnerFilter("bar")).to_es_filter(),
            {
                'or': (
                    {'term': {'owner_id': 'foo'}},
                    {'term': {'owner_id': 'bar'}}
                )
            }
        )


class ExportFilterResultTest(SimpleTestCase):

    @classmethod
    def setUpClass(cls):
        form_pillow = XFormPillow(online=False)
        case_pillow = CasePillow(online=False)
        cls.pillows = [form_pillow, case_pillow]

        es = get_es_new()
        with trap_extra_setup(ConnectionError, msg="cannot connect to elasicsearch"):
            for pillow in cls.pillows:
                completely_initialize_pillow_index(pillow)
            initialize_index_and_mapping(es, GROUP_INDEX_INFO)

        case = new_case(closed=True)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False)
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=True, owner_id="foo")
        case_pillow.send_robust(case.to_json())

        case = new_case(closed=False, owner_id="bar")
        case_pillow.send_robust(case.to_json())

        group = Group(_id=uuid.uuid4().hex, users=["foo", "bar"])
        cls.group_id = group._id
        send_to_elasticsearch('groups', group.to_json())

        form = new_form(form={"meta": {"userID": None}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": ""}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"deviceID": "abc"}})
        form_pillow.send_robust(form.to_json())

        form = new_form(form={"meta": {"userID": uuid.uuid4().hex}})
        form_pillow.send_robust(form.to_json())

        for pillow in cls.pillows:
            pillow.get_es_new().indices.refresh(pillow.es_index)
        es.indices.refresh(GROUP_INDEX_INFO.index)

    @classmethod
    def tearDownClass(cls):
        for pillow in cls.pillows:
            ensure_index_deleted(pillow.es_index)

{'fraction_non_alphanumeric': 0.10664556962025316, 'fraction_numerical': 0.0006329113924050633, 'mean_word_length': 3.1476377952755907, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '246077', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2066, 'n_words': 337}