<commit_before>from __future__ import absolute_import
from __future__ import print_function

from corehq.apps.es import case_search
from corehq.apps.es.case_search import CaseSearchES
from corehq.apps.users.models import CommCareUser
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors

from custom.enikshay.case_utils import (
    CASE_TYPE_TEST,
    get_occurrence_case_from_test,
    get_person_case_from_occurrence,
)
from custom.enikshay.const import (
    ENROLLED_IN_PRIVATE,
)
from custom.enikshay.management.commands.base_data_dump import BaseDataDump

DOMAIN = "enikshay"


class Command(BaseDataDump):
    """
    1. Other Tests
    https://docs.google.com/spreadsheets/d/1OPp0oFlizDnIyrn7Eiv11vUp8IBmc73hES7qqT-mKKA/edit#gid=939368538
    """
    TASK_NAME = "data_dumps_other_tests"
    INPUT_FILE_NAME = "data_dumps_other_tests.csv"

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.case_type = CASE_TYPE_TEST
        self.case_accessor = CaseAccessors(DOMAIN)

    def get_case_ids_query(self, case_type):
        """
        All open and closed test cases
        1) whose host/host = a person case (open or closed)
        with person.dataset = 'real' and self.enrolled_in_private != 'true' AND
        2)
        with self.test_category != 'clinical' and
        self.test_type_value != 'dst'
        and self.test_type_value != 'culture' and
        self.test_type_value != 'sl_line_probe_assay'
        and self.test_type_value != 'fl_line_probe_assay' and
        self.test_type_value != 'cbnaat' and
        self.test_type_value != 'microscopy-fluorescent' and
        self.test_type_value !=  'microscopy-zn'
        """
        reject_test_types = ['clinical', 'dst', 'culture', 'sl_line_probe_assay', 'fl_line_probe_assay',
                             'cbnaat', 'microscopy-fluorescent', 'microscopy-zn']
        return (CaseSearchES()
                .domain(DOMAIN)
                .case_type(case_type)
                .NOT(case_search.case_property_filter(
                    "test_type_value", reject_test_types)
                ))

    def include_case_in_dump(self, test_case):
        person = self.get_person(test_case)
        return (
            person and
            person.get_case_property('dataset') == 'real' and
            person.get_case_property(ENROLLED_IN_PRIVATE) != 'true'
        )

    def get_custom_value(self, column_name, test_case):
        if column_name == "Commcare UUID":
            return test_case.case_id
        elif column_name == "Date of Episode Creation":
            return self.get_episode(test_case).opened_on
        elif column_name == "Criteria for Testing":
            return (
                test_case.get_case_property('rft_drtb_diagnosis') or
                test_case.get_case_property('rft_drtb_diagnosis_ext_dst') or<commit_msg>fix check for property name for clinical tests [skip ci]<commit_after>from __future__ import absolute_import
from __future__ import print_function

from corehq.apps.es import (
    case_search,
    queries,
)
from corehq.apps.es.case_search import CaseSearchES
from corehq.apps.users.models import CommCareUser
from corehq.form_processor.interfaces.dbaccessors import CaseAccessors

from custom.enikshay.case_utils import (
    CASE_TYPE_TEST,
    get_occurrence_case_from_test,
    get_person_case_from_occurrence,
)
from custom.enikshay.const import (
    ENROLLED_IN_PRIVATE,
)
from custom.enikshay.management.commands.base_data_dump import BaseDataDump

DOMAIN = "enikshay"


class Command(BaseDataDump):
    """
    1. Other Tests
    https://docs.google.com/spreadsheets/d/1OPp0oFlizDnIyrn7Eiv11vUp8IBmc73hES7qqT-mKKA/edit#gid=939368538
    """
    TASK_NAME = "data_dumps_other_tests"
    INPUT_FILE_NAME = "data_dumps_other_tests.csv"

    def __init__(self, *args, **kwargs):
        super(Command, self).__init__(*args, **kwargs)
        self.case_type = CASE_TYPE_TEST
        self.case_accessor = CaseAccessors(DOMAIN)

    def get_case_ids_query(self, case_type):
        """
        All open and closed test cases
        1) whose host/host = a person case (open or closed)
        with person.dataset = 'real' and self.enrolled_in_private != 'true' AND
        2)
        with self.test_category != 'clinical' and
        self.test_type_value != 'dst'
        and self.test_type_value != 'culture' and
        self.test_type_value != 'sl_line_probe_assay'
        and self.test_type_value != 'fl_line_probe_assay' and
        self.test_type_value != 'cbnaat' and
        self.test_type_value != 'microscopy-fluorescent' and
        self.test_type_value !=  'microscopy-zn'
        """
        reject_test_types = ['dst', 'culture', 'sl_line_probe_assay', 'fl_line_probe_assay',
                             'cbnaat', 'microscopy-fluorescent', 'microscopy-zn']
        return (CaseSearchES()
                .domain(DOMAIN)
                .case_type(case_type)
                .case_property_query("test_category", "clinical", clause=queries.MUST_NOT)
                .NOT(case_search.case_property_filter(
                    "test_type_value", reject_test_types))
                )

    def include_case_in_dump(self, test_case):
        person = self.get_person(test_case)
        return (
            person and
            person.get_case_property('dataset') == 'real' and
            person.get_case_property(ENROLLED_IN_PRIVATE) != 'true'
        )

    def get_custom_value(self, column_name, test_case):
        if column_name == "Commcare UUID":
            return test_case.case_id
        elif column_name == "Date of Episode Creation":
            return self.get_episode(test_case).opened_on
        elif column_name == "Criteria for Testing":
            return (
                test_case.get_case_property('rft_drtb_diagnosis') or
                test_case.get_case_property('rft_drtb_diagnosis_ext_dst') or
{'fraction_non_alphanumeric': 0.0906639703404112, 'fraction_numerical': 0.007077856420626896, 'mean_word_length': 2.876551273677335, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6814702', 'n_tokens_mistral': 1991, 'n_tokens_neox': 1890, 'n_words': 347}