<commit_before># -*- coding: utf-8 -*-
"""
Models for the FOIA application
"""

from django.contrib.auth.models import User, AnonymousUser
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models, connection
from django.db.models import Q, Sum
from django.template.defaultfilters import escape, linebreaks, slugify
from django.template.loader import render_to_string

import actstream
from datetime import datetime, date, timedelta
from hashlib import md5
import logging
from taggit.managers import TaggableManager
from unidecode import unidecode

from muckrock.agency.models import Agency
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.settings import MAILGUN_SERVER_NAME
from muckrock.tags.models import Tag, TaggedItemBase
from muckrock import task
from muckrock import fields
from muckrock import utils

logger = logging.getLogger(__name__)

class FOIARequestQuerySet(models.QuerySet):
    """Object manager for FOIA requests"""
    # pylint: disable=too-many-public-methods

    def get_submitted(self):
        """Get all submitted FOIA requests"""
        return self.exclude(status='started')

    def get_done(self):
        """Get all FOIA requests with responses"""
        return self.filter(status='done').exclude(date_done=None)

    def get_editable(self):
        """Get all editable FOIA requests"""
        return self.filter(status='started')

    def get_viewable(self, user):
        """Get all viewable FOIA requests for given user"""

        if user.is_staff:
            return self.all()

        # Requests are visible if you own them, or if they are not drafts and not embargoed
        if user.is_authenticated():
            return self.filter(Q(user=user) |
                               (~Q(status='started') &
                                ~Q(embargo=True, date_embargo=None) &
                                ~Q(embargo=True, date_embargo__gte=date.today())))
        else:
            # anonymous user, filter out drafts and embargoes
            return self.exclude(status='started') \
                       .exclude(embargo=True, date_embargo=None) \
                       .exclude(embargo=True, date_embargo__gte=date.today())

    def get_public(self):
        """Get all publically viewable FOIA requests"""
        return self.get_viewable(AnonymousUser())

    def get_overdue(self):
        """Get all overdue FOIA requests"""
        return self.filter(status__in=['ack', 'processed'], date_due__lt=date.today())

    def get_manual_followup(self):
        """Get old requests which require us to follow up on with the agency"""

        return [
            f for f in self.get_overdue()
            if f.communications.all().reverse()[0].date + timedelta(15) < datetime.now()
        ]

    def get_followup(self):
        """Get requests that need follow up emails sent"""
        return self.filter(status__in=['ack', 'processed'],
                           date_followup__lte=date.today(),
                           disable_autofollowups=False)

    def get_open(self):
        """Get requests which we are awaiting a response from"""
        return self.filter(status__in=['ack', 'processed', 'appealing'])

    def get_undated(self):
        """Get requests which have an undated file"""
        return self.filter(~Q(files=None) & Q(files__date=None)).distinct()

    def organization(self, organization):
        """Get requests belonging to an organization's members."""
        members = organization.members.select_related('user').all()
        users = [member.user for member in members]
        return self.prefetch_related('jurisdiction').filter(user__in=users)

STATUS = (
    ('started', 'Draft'),
    ('submitted', 'Processing'),
    ('ack', 'Awaiting Acknowledgement'),
    ('processed', 'Awaiting Response'),
    ('appealing', 'Awaiting Appeal'),
    ('fix', 'Fix Required'),
    ('payment', 'Payment Required'),
    ('rejected', 'Rejected'),
    ('no_docs', 'No Responsive Documents'),
    ('done', 'Completed'),
    ('partial', 'Partially Completed'),
    ('abandoned', 'Withdrawn'),
)

END_STATUS = ['rejected', 'no_docs', 'done', 'partial', 'abandoned']

class Action():
    """A helper class to provide interfaces for request actions"""
    # pylint: disable=too-many-arguments
    def __init__(self, test=None, link=None, title=None, action=None, desc=None, class_name=None):
        self.test = test
        self.link = link
        self.title = title
        self.action = action
        self.desc = desc
        self.class_name = class_name

    def is_possible(self):
        """Is this action possible given the current context?"""<commit_msg>replaces prefetch_related with select_related<commit_after># -*- coding: utf-8 -*-
"""
Models for the FOIA application
"""

from django.contrib.auth.models import User, AnonymousUser
from django.core.mail import EmailMultiAlternatives
from django.core.urlresolvers import reverse
from django.db import models, connection
from django.db.models import Q, Sum
from django.template.defaultfilters import escape, linebreaks, slugify
from django.template.loader import render_to_string

import actstream
from datetime import datetime, date, timedelta
from hashlib import md5
import logging
from taggit.managers import TaggableManager
from unidecode import unidecode

from muckrock.agency.models import Agency
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.settings import MAILGUN_SERVER_NAME
from muckrock.tags.models import Tag, TaggedItemBase
from muckrock import task
from muckrock import fields
from muckrock import utils

logger = logging.getLogger(__name__)

class FOIARequestQuerySet(models.QuerySet):
    """Object manager for FOIA requests"""
    # pylint: disable=too-many-public-methods

    def get_submitted(self):
        """Get all submitted FOIA requests"""
        return self.exclude(status='started')

    def get_done(self):
        """Get all FOIA requests with responses"""
        return self.filter(status='done').exclude(date_done=None)

    def get_editable(self):
        """Get all editable FOIA requests"""
        return self.filter(status='started')

    def get_viewable(self, user):
        """Get all viewable FOIA requests for given user"""

        if user.is_staff:
            return self.all()

        # Requests are visible if you own them, or if they are not drafts and not embargoed
        if user.is_authenticated():
            return self.filter(Q(user=user) |
                               (~Q(status='started') &
                                ~Q(embargo=True, date_embargo=None) &
                                ~Q(embargo=True, date_embargo__gte=date.today())))
        else:
            # anonymous user, filter out drafts and embargoes
            return self.exclude(status='started') \
                       .exclude(embargo=True, date_embargo=None) \
                       .exclude(embargo=True, date_embargo__gte=date.today())

    def get_public(self):
        """Get all publically viewable FOIA requests"""
        return self.get_viewable(AnonymousUser())

    def get_overdue(self):
        """Get all overdue FOIA requests"""
        return self.filter(status__in=['ack', 'processed'], date_due__lt=date.today())

    def get_manual_followup(self):
        """Get old requests which require us to follow up on with the agency"""

        return [
            f for f in self.get_overdue()
            if f.communications.all().reverse()[0].date + timedelta(15) < datetime.now()
        ]

    def get_followup(self):
        """Get requests that need follow up emails sent"""
        return self.filter(status__in=['ack', 'processed'],
                           date_followup__lte=date.today(),
                           disable_autofollowups=False)

    def get_open(self):
        """Get requests which we are awaiting a response from"""
        return self.filter(status__in=['ack', 'processed', 'appealing'])

    def get_undated(self):
        """Get requests which have an undated file"""
        return self.filter(~Q(files=None) & Q(files__date=None)).distinct()

    def organization(self, organization):
        """Get requests belonging to an organization's members."""
        members = organization.members.select_related('user').all()
        users = [member.user for member in members]
        return self.select_related('jurisdiction').filter(user__in=users)

STATUS = (
    ('started', 'Draft'),
    ('submitted', 'Processing'),
    ('ack', 'Awaiting Acknowledgement'),
    ('processed', 'Awaiting Response'),
    ('appealing', 'Awaiting Appeal'),
    ('fix', 'Fix Required'),
    ('payment', 'Payment Required'),
    ('rejected', 'Rejected'),
    ('no_docs', 'No Responsive Documents'),
    ('done', 'Completed'),
    ('partial', 'Partially Completed'),
    ('abandoned', 'Withdrawn'),
)

END_STATUS = ['rejected', 'no_docs', 'done', 'partial', 'abandoned']

class Action():
    """A helper class to provide interfaces for request actions"""
    # pylint: disable=too-many-arguments
    def __init__(self, test=None, link=None, title=None, action=None, desc=None, class_name=None):
        self.test = test
        self.link = link
        self.title = title
        self.action = action
        self.desc = desc
        self.class_name = class_name

    def is_possible(self):
        """Is this action possible given the current context?"""
{'fraction_non_alphanumeric': 0.11437632135306554, 'fraction_numerical': 0.0010570824524312897, 'mean_word_length': 3.046621043627032, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3605071', 'n_tokens_mistral': 2706, 'n_tokens_neox': 2574, 'n_words': 752}