<commit_before>from datetime import datetime
import re
from django.contrib import messages
from django.core.mail.message import EmailMessage
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models.aggregates import Count
from django.http.response import Http404
from django.shortcuts import redirect
from django.template.loader import render_to_string
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext_noop as _
from corehq.apps.domain.decorators import require_superuser
from corehq.apps.hqwebapp.views import BasePageView
from corehq.apps.hqpillow_retry.filters import DatePropFilter, AttemptsFilter, PillowErrorFilter
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dispatcher import AdminReportDispatcher
from corehq.apps.reports.generic import GenericTabularReport, GetParamsMixin
from corehq.apps.reports.standard import DatespanMixin
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.web import get_url_base
from pillow_retry.models import PillowError
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime

SOURCE_SINGLE = 'single'
ACTION_RESET = 'reset'
ACTION_DELETE = 'delete'
ACTION_SEND = 'send'
ACTIONS = [ACTION_RESET, ACTION_DELETE, ACTION_SEND]


class PillowErrorsReport(GenericTabularReport, DatespanMixin, GetParamsMixin):
    dispatcher = AdminReportDispatcher
    slug = 'pillow_errors'
    name = _('PillowTop Errors')
    section_name = _("ADMINREPORT")
    asynchronous = False
    ajax_pagination = True
    base_template = 'reports/base_template.html'
    needs_filters = False

    fields = (
        'corehq.apps.reports.filters.dates.DatespanFilter',
        'corehq.apps.hqpillow_retry.filters.DatePropFilter',
        'corehq.apps.hqpillow_retry.filters.PillowErrorFilter',
        'corehq.apps.hqpillow_retry.filters.AttemptsFilter',
    )

    report_template_path = 'hqpillow_retry/pillow_errors.html'

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn('Error ID', sortable=False),
            DataTablesColumn('Doc ID', sortable=False),
            DataTablesColumn('Pillow Class', sortable=True),
            DataTablesColumn('Created', sortable=True),
            DataTablesColumn('Next attempt', sortable=True),
            DataTablesColumn('Attempts (current / total)', sortable=True),
            DataTablesColumn('Error type', sortable=True),
            DataTablesColumn('Doc type', sortable=False),
            DataTablesColumn('Domain(s)', sortable=False),
            DataTablesColumn('Select', sortable=False),
        )

    @property
    @memoized
    def pillow_error_filter(self):
        return PillowErrorFilter(self.request, None)

    @property
    @memoized
    def pillow_error_vals(self):
        return {item['slug']: item['value'] for item in self.pillow_error_filter.GET_values}

    @property
    def pillow(self):
        return self.pillow_error_vals.get('pillow')

    @property
    def error(self):
        return self.pillow_error_vals.get('error')

    @property
    def sort_descending(self):
        return self.request.GET.get('sSortDir_0', 'asc') == 'desc'

    @property
    def sort_field(self):
        sort_fields = [
            'pillow',
            'date_created',
            'date_next_attempt',
            'current_attempt',
            'error_type'
        ]
        sort_index = int(self.request.GET.get('iSortCol_0', 2))
        sort_index = 1 if sort_index == 0 else sort_index - 1
        field = sort_fields[sort_index]
        return field if not self.sort_descending else '-{0}'.format(field)

    @property
    def date_field_filter(self):
        return DatePropFilter.get_value(self.request, None)

    @property
    def filter_attempts(self):
        return AttemptsFilter.get_value(self.request, None)

    @property
    def shared_pagination_GET_params(self):
        return self.pillow_error_filter.shared_pagination_GET_params + [
            dict(name='startdate', value=self.datespan.startdate_display),
            dict(name='enddate', value=self.datespan.enddate_display),
            dict(name=DatePropFilter.slug, value=self.date_field_filter),
            dict(name=AttemptsFilter.slug, value=str(self.filter_attempts)),
        ]

    @property
    def total_records(self):
        query = self.get_query()
        return query.aggregate(Count('id'))['id__count']

    def get_query(self):
        query = PillowError.objects
        if self.pillow:
            query = query.filter(pillow=self.pillow)
        if self.error:
            query = query.filter(error_type=self.error)

        if self.date_field_filter:
            q = self.date_field_filter + '__gte'
            query = query.filter(**{q: self.datespan.startdate})
            q = self.date_field_filter + '__lte'
            query = query.filter(**{q: self.datespan.enddate_adjusted})

        if self.filter_attempts:
            query = query.filter(current_attempt__gt=settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS)

        return query

    @property
    def rows(self):
        query = self.get_query()
        query = query.order_by(self.sort_field)

        errors = query[self.pagination.start:(self.pagination.start+self.pagination.count)]
        for error in errors:
            yield [
                self.make_traceback_link(error),
                self.make_search_link(error),
                error.pillow.split('.')[-1],
                naturaltime(error.date_created),
                naturaltime(error.date_next_attempt),
                '{0} / {1}'.format(error.current_attempt, error.total_attempts),
                error.error_type,
                error.doc_type,
                error.domains,
                self.make_checkbox(error)
            ]

    def make_traceback_link(self, error):
        return '<a href="{0}?error={1}" target="_blank">{2}</a>'.format(
            reverse(EditPillowError.urlname),
            error.id,
            error.id
        )

    def make_search_link(self, error):
        return (
            '{text}<a href="{search_url}?q={doc_id}" target="_blank" title="{search_title}">'
            '<i class="icon-search"></i></a>'
            '&nbsp;<a href="{raw_url}?id={doc_id}" target="_blank" title="{raw_title}">'
            '<i class="icon-file"></i></a>'
        ).format(<commit_msg>more informative next attempt value<commit_after>from datetime import datetime
import re
from django.contrib import messages
from django.core.mail.message import EmailMessage
from django.core.urlresolvers import reverse
from django.db import transaction
from django.db.models.aggregates import Count
from django.http.response import Http404
from django.shortcuts import redirect
from django.template.loader import render_to_string
from django.utils.decorators import method_decorator
from django.utils.translation import ugettext_noop as _
from corehq.apps.domain.decorators import require_superuser
from corehq.apps.hqwebapp.views import BasePageView
from corehq.apps.hqpillow_retry.filters import DatePropFilter, AttemptsFilter, PillowErrorFilter
from corehq.apps.reports.datatables import DataTablesHeader, DataTablesColumn
from corehq.apps.reports.dispatcher import AdminReportDispatcher
from corehq.apps.reports.generic import GenericTabularReport, GetParamsMixin
from corehq.apps.reports.standard import DatespanMixin
from dimagi.utils.decorators.memoized import memoized
from dimagi.utils.web import get_url_base
from pillow_retry.models import PillowError
from django.conf import settings
from django.contrib.humanize.templatetags.humanize import naturaltime

SOURCE_SINGLE = 'single'
ACTION_RESET = 'reset'
ACTION_DELETE = 'delete'
ACTION_SEND = 'send'
ACTIONS = [ACTION_RESET, ACTION_DELETE, ACTION_SEND]


class PillowErrorsReport(GenericTabularReport, DatespanMixin, GetParamsMixin):
    dispatcher = AdminReportDispatcher
    slug = 'pillow_errors'
    name = _('PillowTop Errors')
    section_name = _("ADMINREPORT")
    asynchronous = False
    ajax_pagination = True
    base_template = 'reports/base_template.html'
    needs_filters = False

    fields = (
        'corehq.apps.reports.filters.dates.DatespanFilter',
        'corehq.apps.hqpillow_retry.filters.DatePropFilter',
        'corehq.apps.hqpillow_retry.filters.PillowErrorFilter',
        'corehq.apps.hqpillow_retry.filters.AttemptsFilter',
    )

    report_template_path = 'hqpillow_retry/pillow_errors.html'

    @property
    def headers(self):
        return DataTablesHeader(
            DataTablesColumn('Error ID', sortable=False),
            DataTablesColumn('Doc ID', sortable=False),
            DataTablesColumn('Pillow Class', sortable=True),
            DataTablesColumn('Created', sortable=True),
            DataTablesColumn('Next attempt', sortable=True),
            DataTablesColumn('Attempts (current / total)', sortable=True),
            DataTablesColumn('Error type', sortable=True),
            DataTablesColumn('Doc type', sortable=False),
            DataTablesColumn('Domain(s)', sortable=False),
            DataTablesColumn('Select', sortable=False),
        )

    @property
    @memoized
    def pillow_error_filter(self):
        return PillowErrorFilter(self.request, None)

    @property
    @memoized
    def pillow_error_vals(self):
        return {item['slug']: item['value'] for item in self.pillow_error_filter.GET_values}

    @property
    def pillow(self):
        return self.pillow_error_vals.get('pillow')

    @property
    def error(self):
        return self.pillow_error_vals.get('error')

    @property
    def sort_descending(self):
        return self.request.GET.get('sSortDir_0', 'asc') == 'desc'

    @property
    def sort_field(self):
        sort_fields = [
            'pillow',
            'date_created',
            'date_next_attempt',
            'current_attempt',
            'error_type'
        ]
        sort_index = int(self.request.GET.get('iSortCol_0', 2))
        sort_index = 1 if sort_index == 0 else sort_index - 1
        field = sort_fields[sort_index]
        return field if not self.sort_descending else '-{0}'.format(field)

    @property
    def date_field_filter(self):
        return DatePropFilter.get_value(self.request, None)

    @property
    def filter_attempts(self):
        return AttemptsFilter.get_value(self.request, None)

    @property
    def shared_pagination_GET_params(self):
        return self.pillow_error_filter.shared_pagination_GET_params + [
            dict(name='startdate', value=self.datespan.startdate_display),
            dict(name='enddate', value=self.datespan.enddate_display),
            dict(name=DatePropFilter.slug, value=self.date_field_filter),
            dict(name=AttemptsFilter.slug, value=str(self.filter_attempts)),
        ]

    @property
    def total_records(self):
        query = self.get_query()
        return query.aggregate(Count('id'))['id__count']

    def get_query(self):
        query = PillowError.objects
        if self.pillow:
            query = query.filter(pillow=self.pillow)
        if self.error:
            query = query.filter(error_type=self.error)

        if self.date_field_filter:
            q = self.date_field_filter + '__gte'
            query = query.filter(**{q: self.datespan.startdate})
            q = self.date_field_filter + '__lte'
            query = query.filter(**{q: self.datespan.enddate_adjusted})

        if self.filter_attempts:
            query = query.filter(current_attempt__gt=settings.PILLOW_RETRY_QUEUE_MAX_PROCESSING_ATTEMPTS)

        return query

    @property
    def rows(self):
        query = self.get_query()
        query = query.order_by(self.sort_field)

        next_deploy = _('Next Deploy')
        errors = query[self.pagination.start:(self.pagination.start+self.pagination.count)]
        for error in errors:
            yield [
                self.make_traceback_link(error),
                self.make_search_link(error),
                error.pillow.split('.')[-1],
                naturaltime(error.date_created),
                naturaltime(error.date_next_attempt) if error.has_next_attempt() else next_deploy,
                '{0} / {1}'.format(error.current_attempt, error.total_attempts),
                error.error_type,
                error.doc_type,
                error.domains,
                self.make_checkbox(error)
            ]

    def make_traceback_link(self, error):
        return '<a href="{0}?error={1}" target="_blank">{2}</a>'.format(
            reverse(EditPillowError.urlname),
            error.id,
            error.id
        )

    def make_search_link(self, error):
        return (
            '{text}<a href="{search_url}?q={doc_id}" target="_blank" title="{search_title}">'
            '<i class="icon-search"></i></a>'
            '&nbsp;<a href="{raw_url}?id={doc_id}" target="_blank" title="{raw_title}">'
            '<i class="icon-file"></i></a>'
        ).format(
{'fraction_non_alphanumeric': 0.10378510378510379, 'fraction_numerical': 0.002442002442002442, 'mean_word_length': 3.005195599022005, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7230437', 'n_tokens_mistral': 4017, 'n_tokens_neox': 3871, 'n_words': 716}