<commit_before>from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from jsonobject import DateTimeProperty

from corehq import Domain
from corehq.apps.commtrack.util import get_commtrack_location_id
from corehq.apps.locations.models import Location
from corehq.apps.reports.cache import request_cache
from corehq.apps.reports.filters.fixtures import AsyncLocationFilter
from corehq.apps.users.models import CommCareUser
from corehq.elastic import ES_URLS
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.standard import ProjectReport, ProjectReportParametersMixin, DatespanMixin
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dont_use.fields import StrongFilterUsersField
from corehq.apps.reports.generic import ElasticProjectInspectionReport
from corehq.apps.reports.standard.monitoring import MultiFormDrilldownMixin
from corehq.elastic import es_query
from custom.m4change.constants import BOOKING_FORMS, FOLLOW_UP_FORMS, BOOKED_AND_UNBOOKED_DELIVERY_FORMS, IMMUNIZATION_FORMS, \
    REJECTION_REASON_DISPLAY_NAMES
from custom.m4change.models import McctStatus
from custom.m4change.reports import get_location_hierarchy_by_id
from custom.m4change.utils import get_case_by_id, get_user_by_id, get_property, get_form_ids_by_status
from custom.m4change.constants import EMPTY_FIELD
from corehq.apps.reports.tasks import export_all_rows_task


CASE_FORM_SCRIPT_FILTER_NAMESPACES = BOOKED_AND_UNBOOKED_DELIVERY_FORMS + IMMUNIZATION_FORMS


def _get_date_range(range):
    if range is not None:
        dates = str(range).split(_(" to "))
        return (dates[0], dates[1])
    return None


def _get_report_query(start_date, end_date, filtered_case_ids):
    return {
        "query": {
            "bool": {
                "must": [
                    {"range": {"form.meta.timeEnd": {"from": start_date, "to": end_date, "include_upper": True}}},
                    {"term": {"doc_type": "xforminstance"}},
                ],
                "should": [
                    {"terms": {"xmlns.exact": CASE_FORM_SCRIPT_FILTER_NAMESPACES}},
                    {"range": {"form.visits": {"gte": 1, "lte": 4}}}
                ]
            }
        },
        "filter": {
            "and": [
                {"not": {"missing": {"field": "form.case.@case_id"}}},
                {"terms": {"form.case.@case_id": filtered_case_ids}}
            ]
        }
    }


def calculate_form_data(self, form):
    try:
        case_id = form["form"]["case"]["@case_id"]
        case = get_case_by_id(case_id)
    except KeyError:
        case = EMPTY_FIELD

    amount_due = EMPTY_FIELD
    service_type = EMPTY_FIELD
    visits = form["form"].get("visits")<commit_msg>fix mCCT report showing incorrect results<commit_after>from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.utils.safestring import mark_safe
from django.utils.translation import ugettext as _
from jsonobject import DateTimeProperty

from corehq import Domain
from corehq.apps.commtrack.util import get_commtrack_location_id
from corehq.apps.locations.models import Location
from corehq.apps.reports.cache import request_cache
from corehq.apps.reports.filters.fixtures import AsyncLocationFilter
from corehq.apps.users.models import CommCareUser
from corehq.elastic import ES_URLS
from corehq.apps.reports.standard import CustomProjectReport
from corehq.apps.reports.standard import ProjectReport, ProjectReportParametersMixin, DatespanMixin
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dont_use.fields import StrongFilterUsersField
from corehq.apps.reports.generic import ElasticProjectInspectionReport
from corehq.apps.reports.standard.monitoring import MultiFormDrilldownMixin
from corehq.elastic import es_query
from custom.m4change.constants import BOOKING_FORMS, FOLLOW_UP_FORMS, BOOKED_AND_UNBOOKED_DELIVERY_FORMS, IMMUNIZATION_FORMS, \
    REJECTION_REASON_DISPLAY_NAMES
from custom.m4change.models import McctStatus
from custom.m4change.reports import get_location_hierarchy_by_id
from custom.m4change.utils import get_case_by_id, get_user_by_id, get_property, get_form_ids_by_status
from custom.m4change.constants import EMPTY_FIELD
from corehq.apps.reports.tasks import export_all_rows_task


CASE_FORM_SCRIPT_FILTER_NAMESPACES = BOOKED_AND_UNBOOKED_DELIVERY_FORMS + IMMUNIZATION_FORMS


def _get_date_range(range):
    if range is not None:
        dates = str(range).split(_(" to "))
        return (dates[0], dates[1])
    return None


def _get_report_query(start_date, end_date, filtered_case_ids):
    return {
        "query": {
            "bool": {
                "must": [
                    {"range": {"form.meta.timeEnd": {"from": start_date, "to": end_date, "include_upper": True}}},
                    {"term": {"doc_type": "xforminstance"}}
                ]
            }
        },
        "filter": {
            "and": [
                {"not": {"missing": {"field": "form.case.@case_id"}}},
                {"terms": {"form.case.@case_id": filtered_case_ids}},
                {"script":{
                    "script": """
                    if (namespaces contains _source.xmlns) {
                        return true;
                    } else if (_source.form.?visits >= 1 && _source.form.?visits <= 4) {
                        return true;
                    }
                    return false;
                    """,
                    "params": {
                        "namespaces": CASE_FORM_SCRIPT_FILTER_NAMESPACES
                    }
                }}
            ]
        }
    }


def calculate_form_data(self, form):
    try:
        case_id = form["form"]["case"]["@case_id"]
        case = get_case_by_id(case_id)
    except KeyError:
        case = EMPTY_FIELD

    amount_due = EMPTY_FIELD
    service_type = EMPTY_FIELD
    visits = form["form"].get("visits")
{'fraction_non_alphanumeric': 0.0918918918918919, 'fraction_numerical': 0.0029484029484029483, 'mean_word_length': 3.1480978260869565, 'pattern_counts': {'":': 45, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1879154', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1851, 'n_words': 385}