<commit_before>from collections import defaultdict
from datetime import date, datetime, timedelta
from typing import Dict, Iterator

from django.db.models import Q
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _

from dateutil.rrule import DAILY, FR, MO, SA, TH, TU, WE, rrule

from corehq.apps.reports.datatables import DataTablesColumn, DataTablesHeader
from corehq.apps.reports.filters.dates import DatespanFilter
from corehq.apps.reports.generic import GenericTabularReport
from corehq.apps.reports.standard import CustomProjectReport, DatespanMixin
from corehq.form_processor.models import XFormInstanceSQL
from custom.abt.reports.filters_2020 import (
    LevelFourFilter,
    LevelOneFilter,
    LevelThreeFilter,
    LevelTwoFilter,
    SubmissionStatusFilter,
)
from custom.abt.reports.fixture_utils import get_locations

INDICATORS_FORM_XMLNS = 'http://openrosa.org/formdesigner/00CEB41B-2967-4370-9EA3-BFD9BD7AF785'


class LatePmt2020Report(GenericTabularReport, CustomProjectReport, DatespanMixin):
    report_title = "Late PMT"
    slug = 'late_pmt_2020'
    name = "Late PMT 2020"

    languages = (
        'en',
        'fra',
        'por'
    )

    fields = [
        DatespanFilter,
        LevelOneFilter,
        LevelTwoFilter,
        LevelThreeFilter,
        LevelFourFilter,
        SubmissionStatusFilter,
    ]

    @property
    def report_config(self):
        return {
            'domain': self.domain,
            'startdate': self.startdate,
            'enddate': self.enddate,
            'level_1': self.request.GET.get('level_1', ''),
            'level_2': self.request.GET.get('level_2', ''),
            'level_3': self.request.GET.get('level_3', ''),
            'level_4': self.request.GET.get('level_4', ''),
            'submission_status': self.request.GET.get('submission_status', '')
        }

    @property
    def startdate(self):
        return self.request.datespan.startdate

    @property
    def enddate(self):
        return self.request.datespan.end_of_end_day

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn(_("Missing Report Date")),
            DataTablesColumn(_("Name")),
            DataTablesColumn(_("Country")),
            DataTablesColumn(_("Level 1")),
            DataTablesColumn(_("Level 2")),
            DataTablesColumn(_("Level 3")),
            DataTablesColumn(_("Level 4")),
            DataTablesColumn(_("Submission Status")),
        )

    @cached_property
    def pmts_submitted_by_date(self) -> Dict[datetime.date, set]:
        pmts_submitted = defaultdict(set)
        forms = iter_forms_by_xmlns_received_on(
            self.domain, INDICATORS_FORM_XMLNS,
            midnight_starting(self.startdate),
            midnight_ending(self.enddate),
        )
        for form in forms:
            location_id = form.form_data['location_operation_site']
            pmts_submitted[form.received_on.date()].add(location_id)
        return pmts_submitted

    @property
    def rows(self):
        def _to_report_format(date, location, error_msg):
            return [
                date.strftime("%Y-%m-%d"),
                location.name,
                location.country,
                location.level_1,
                location.level_2,
                location.level_3,
                location.level_4,
                error_msg
            ]

        show_all = self.report_config['submission_status'] != 'missing_pmt_data'
        dates = rrule(
            DAILY,
            dtstart=self.startdate,
            until=self.enddate,
            byweekday=(MO, TU, WE, TH, FR, SA)
        )
        rows = []
        for date in dates:
            for location in get_locations(self.domain, self.report_config):
                pmt_submitted = location.id in self.pmts_submitted_by_date[date.date()]
                error_msg = '' if pmt_submitted else _('Incorrect or no PMT data submitted')
                if show_all or not pmt_submitted:
                    rows.append(_to_report_format(date, location, error_msg))
        return rows


def iter_forms_by_xmlns_received_on(
    domain: str,
    xmlns: str,
    start_datetime: datetime,
    end_datetime: datetime,
) -> Iterator[XFormInstanceSQL]:
    """
    Iterates form submissions of a given ``xmlns`` from
    ``start_datetime`` (incl) to ``end_datetime`` (excl).<commit_msg>Avoid name conflicts<commit_after>from collections import defaultdict
from datetime import date, datetime, timedelta
from typing import Dict, Iterator

from django.db.models import Q
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _

from dateutil.rrule import DAILY, FR, MO, SA, TH, TU, WE, rrule

from corehq.apps.reports.datatables import DataTablesColumn, DataTablesHeader
from corehq.apps.reports.filters.dates import DatespanFilter
from corehq.apps.reports.generic import GenericTabularReport
from corehq.apps.reports.standard import CustomProjectReport, DatespanMixin
from corehq.form_processor.models import XFormInstanceSQL
from custom.abt.reports.filters_2020 import (
    LevelFourFilter,
    LevelOneFilter,
    LevelThreeFilter,
    LevelTwoFilter,
    SubmissionStatusFilter,
)
from custom.abt.reports.fixture_utils import get_locations

INDICATORS_FORM_XMLNS = 'http://openrosa.org/formdesigner/00CEB41B-2967-4370-9EA3-BFD9BD7AF785'


class LatePmt2020Report(GenericTabularReport, CustomProjectReport, DatespanMixin):
    report_title = "Late PMT"
    slug = 'late_pmt_2020'
    name = "Late PMT 2020"

    languages = (
        'en',
        'fra',
        'por'
    )

    fields = [
        DatespanFilter,
        LevelOneFilter,
        LevelTwoFilter,
        LevelThreeFilter,
        LevelFourFilter,
        SubmissionStatusFilter,
    ]

    @property
    def report_config(self):
        return {
            'domain': self.domain,
            'startdate': self.startdate,
            'enddate': self.enddate,
            'level_1': self.request.GET.get('level_1', ''),
            'level_2': self.request.GET.get('level_2', ''),
            'level_3': self.request.GET.get('level_3', ''),
            'level_4': self.request.GET.get('level_4', ''),
            'submission_status': self.request.GET.get('submission_status', '')
        }

    @property
    def startdate(self):
        return self.request.datespan.startdate

    @property
    def enddate(self):
        return self.request.datespan.end_of_end_day

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn(_("Missing Report Date")),
            DataTablesColumn(_("Name")),
            DataTablesColumn(_("Country")),
            DataTablesColumn(_("Level 1")),
            DataTablesColumn(_("Level 2")),
            DataTablesColumn(_("Level 3")),
            DataTablesColumn(_("Level 4")),
            DataTablesColumn(_("Submission Status")),
        )

    @cached_property
    def pmts_submitted_by_date(self) -> Dict[datetime.date, set]:
        pmts_submitted = defaultdict(set)
        forms = iter_forms_by_xmlns_received_on(
            self.domain, INDICATORS_FORM_XMLNS,
            midnight_starting(self.startdate),
            midnight_ending(self.enddate),
        )
        for form in forms:
            location_id = form.form_data['location_operation_site']
            pmts_submitted[form.received_on.date()].add(location_id)
        return pmts_submitted

    @property
    def rows(self):
        def _to_report_format(date_, location, error_msg):
            return [
                date_.strftime("%Y-%m-%d"),
                location.name,
                location.country,
                location.level_1,
                location.level_2,
                location.level_3,
                location.level_4,
                error_msg
            ]

        show_all = self.report_config['submission_status'] != 'missing_pmt_data'
        dates = rrule(
            DAILY,
            dtstart=self.startdate,
            until=self.enddate,
            byweekday=(MO, TU, WE, TH, FR, SA)
        )
        rows = []
        for date_ in dates:
            for location in get_locations(self.domain, self.report_config):
                pmt_submitted = location.id in self.pmts_submitted_by_date[date_.date()]
                error_msg = '' if pmt_submitted else _('Incorrect or no PMT data submitted')
                if show_all or not pmt_submitted:
                    rows.append(_to_report_format(date_, location, error_msg))
        return rows


def iter_forms_by_xmlns_received_on(
    domain: str,
    xmlns: str,
    start_datetime: datetime,
    end_datetime: datetime,
) -> Iterator[XFormInstanceSQL]:
    """
    Iterates form submissions of a given ``xmlns`` from
    ``start_datetime`` (incl) to ``end_datetime`` (excl).
{'fraction_non_alphanumeric': 0.09802158273381295, 'fraction_numerical': 0.011465827338129496, 'mean_word_length': 2.4794681267109895, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5812938', 'n_tokens_mistral': 2739, 'n_tokens_neox': 2608, 'n_words': 519}