<commit_before>"""
Views for muckrock project
"""
from django.contrib import messages
from django.contrib.auth.models import User
from django.core.exceptions import FieldError
from django.core.paginator import Paginator, InvalidPage
from django.db.models import Sum
from django.http import HttpResponseServerError, Http404
from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext, Context, loader
from django.utils.decorators import method_decorator
from django.views.generic.list import ListView

from muckrock.agency.models import Agency
from muckrock.foia.models import FOIARequest, FOIAFile
from muckrock.forms import MRFilterForm
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.news.models import Article

import re
from haystack.views import SearchView

class MRFilterableListView(ListView):
    """
    The main feature of MRFilterableListView is the ability to filter
    a set of request objects by a dynamic dictionary of filters and
    lookup conditions. MRFilterableListView should be used in conjunction
    with MRFilterForm, available in the `muckrock.forms` module.

    To see an example of a subclass of MRFilterableListView that adds new
    filter fields, look at `muckrock.foia.views.RequestList`.
    """

    title = ''
    template_name = 'lists/base_list.html'

    def get_filters(self):
        """
        Filters should be the same as the fields in MRFilterForm, or whichever
        subclass of MRFilterForm is being used in as this class's `filter_form`.
        Filters are an array of key-value pairs.
        Required pairs are the field name and the [lookup condition][a].

        [a]: https://docs.djangoproject.com/en/1.7/ref/models/querysets/#field-lookups
        """
        # pylint: disable=no-self-use
        return [
            {
                'field': 'user',
                'lookup': 'exact',
            },
            {
                'field': 'agency',
                'lookup': 'exact',
            },
            {
                'field': 'jurisdiction',
                'lookup': 'exact',
            },
            {
                'field': 'tags',
                'lookup': 'name__in',
            },
        ]

    def clean_filter_value(self, filter_key, filter_value):
        """Cleans filter inputs to their expected values if detected as incorrect"""
        # pylint:disable=no-self-use
        # pylint:disable=too-many-branches
        if not filter_value:
            return None

        # tags need to be parsed into an array before filtering
        if filter_key == 'tags':
            filter_value = filter_value.split(',')
        if filter_key == 'user':
            # if looking up by PK, then result will be empty
            # if looking up by username, then result will have length
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = User.objects.get(username=filter_value).pk
                except User.DoesNotExist:
                    filter_value = None
                # username is unique so only one result should be returned by get
        if filter_key == 'agency':
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = Agency.objects.get(slug=filter_value).pk
                except Agency.DoesNotExist:
                    filter_value = None
                except Agency.MultipleObjectsReturned:
                    filter_value = Agency.objects.filter(slug=filter_value)[0]
        if filter_key == 'jurisdiction':
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = Jurisdiction.objects.get(slug=filter_value).pk
                except Jurisdiction.DoesNotExist:
                    filter_value = None
                except Jurisdiction.MultipleObjectsReturned:
                    filter_value = Jurisdiction.objects.filter(slug=filter_value)[0]

        return filter_value

    def get_filter_data(self):
        """Returns a list of filter values and a url query for the filter."""
        get = self.request.GET
        filter_initials = {}
        filter_url = ''
        for filter_by in self.get_filters():
            filter_key = filter_by['field']
            filter_value = get.get(filter_key, None)
            filter_value = self.clean_filter_value(filter_key, filter_value)
            if filter_value:
                if isinstance(filter_value, list):
                    try:
                        filter_value = ', '.join(filter_value)
                    except TypeError:
                        filter_value = str(filter_value)
                kwarg = {filter_key: filter_value}
                try:
                    filter_initials.update(kwarg)
                    filter_url += '&' + str(filter_key) + '=' + str(filter_value)
                except ValueError:
                    pass
        return {
            'filter_initials': filter_initials,
            'filter_url': filter_url
        }

    def filter_list(self, objects):
        """Filters a list of objects"""
        get = self.request.GET
        kwargs = {}
        for filter_by in self.get_filters():
            filter_key = filter_by['field']
            filter_lookup = filter_by['lookup']
            filter_value = get.get(filter_key, None)
            filter_value = self.clean_filter_value(filter_key, filter_value)
            if filter_value:
                kwargs.update({'{0}__{1}'.format(filter_key, filter_lookup): filter_value})
        # tag filtering could add duplicate items to results, so .distinct() is used
        try:
            objects = objects.filter(**kwargs).distinct()
        except FieldError:
            pass
        except ValueError:
            error_msg = "Sorry, there was a problem with your filters. Please try filtering again."
            messages.error(self.request, error_msg)
        return objects

    def sort_list(self, objects):
        """Sorts the list of objects"""
        get = self.request.GET
        sort = get.get('sort', None)
        if sort:
            order = get.get('order', 'asc')
            if order != 'asc':
                sort = '-' + sort
            try:
                objects = objects.order_by(sort)
            except FieldError:
                pass
        return objects

    def get_context_data(self, **kwargs):
        """Gets basic context, including title, form, and url"""
        context = super(MRFilterableListView, self).get_context_data(**kwargs)
        filter_data = self.get_filter_data()
        context['title'] = self.title
        context['per_page'] = int(self.get_paginate_by(context['object_list']))
        try:
            context['filter_form'] = MRFilterForm(initial=filter_data['filter_initials'])<commit_msg>check foia sort field explicitly, try/catch was missing the error<commit_after>"""
Views for muckrock project
"""
from django.contrib import messages
from django.contrib.auth.models import User
from django.core.exceptions import FieldError
from django.core.paginator import Paginator, InvalidPage
from django.db.models import Sum
from django.http import HttpResponseServerError, Http404
from django.shortcuts import render_to_response, get_object_or_404, redirect
from django.template import RequestContext, Context, loader
from django.utils.decorators import method_decorator
from django.views.generic.list import ListView

from muckrock.agency.models import Agency
from muckrock.foia.models import FOIARequest, FOIAFile
from muckrock.forms import MRFilterForm
from muckrock.jurisdiction.models import Jurisdiction
from muckrock.news.models import Article

import re
from haystack.views import SearchView

class MRFilterableListView(ListView):
    """
    The main feature of MRFilterableListView is the ability to filter
    a set of request objects by a dynamic dictionary of filters and
    lookup conditions. MRFilterableListView should be used in conjunction
    with MRFilterForm, available in the `muckrock.forms` module.

    To see an example of a subclass of MRFilterableListView that adds new
    filter fields, look at `muckrock.foia.views.RequestList`.
    """

    title = ''
    template_name = 'lists/base_list.html'

    def get_filters(self):
        """
        Filters should be the same as the fields in MRFilterForm, or whichever
        subclass of MRFilterForm is being used in as this class's `filter_form`.
        Filters are an array of key-value pairs.
        Required pairs are the field name and the [lookup condition][a].

        [a]: https://docs.djangoproject.com/en/1.7/ref/models/querysets/#field-lookups
        """
        # pylint: disable=no-self-use
        return [
            {
                'field': 'user',
                'lookup': 'exact',
            },
            {
                'field': 'agency',
                'lookup': 'exact',
            },
            {
                'field': 'jurisdiction',
                'lookup': 'exact',
            },
            {
                'field': 'tags',
                'lookup': 'name__in',
            },
        ]

    def clean_filter_value(self, filter_key, filter_value):
        """Cleans filter inputs to their expected values if detected as incorrect"""
        # pylint:disable=no-self-use
        # pylint:disable=too-many-branches
        if not filter_value:
            return None

        # tags need to be parsed into an array before filtering
        if filter_key == 'tags':
            filter_value = filter_value.split(',')
        if filter_key == 'user':
            # if looking up by PK, then result will be empty
            # if looking up by username, then result will have length
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = User.objects.get(username=filter_value).pk
                except User.DoesNotExist:
                    filter_value = None
                # username is unique so only one result should be returned by get
        if filter_key == 'agency':
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = Agency.objects.get(slug=filter_value).pk
                except Agency.DoesNotExist:
                    filter_value = None
                except Agency.MultipleObjectsReturned:
                    filter_value = Agency.objects.filter(slug=filter_value)[0]
        if filter_key == 'jurisdiction':
            if len(re.findall(r'\D+', filter_value)) > 0:
                try:
                    filter_value = Jurisdiction.objects.get(slug=filter_value).pk
                except Jurisdiction.DoesNotExist:
                    filter_value = None
                except Jurisdiction.MultipleObjectsReturned:
                    filter_value = Jurisdiction.objects.filter(slug=filter_value)[0]

        return filter_value

    def get_filter_data(self):
        """Returns a list of filter values and a url query for the filter."""
        get = self.request.GET
        filter_initials = {}
        filter_url = ''
        for filter_by in self.get_filters():
            filter_key = filter_by['field']
            filter_value = get.get(filter_key, None)
            filter_value = self.clean_filter_value(filter_key, filter_value)
            if filter_value:
                if isinstance(filter_value, list):
                    try:
                        filter_value = ', '.join(filter_value)
                    except TypeError:
                        filter_value = str(filter_value)
                kwarg = {filter_key: filter_value}
                try:
                    filter_initials.update(kwarg)
                    filter_url += '&' + str(filter_key) + '=' + str(filter_value)
                except ValueError:
                    pass
        return {
            'filter_initials': filter_initials,
            'filter_url': filter_url
        }

    def filter_list(self, objects):
        """Filters a list of objects"""
        get = self.request.GET
        kwargs = {}
        for filter_by in self.get_filters():
            filter_key = filter_by['field']
            filter_lookup = filter_by['lookup']
            filter_value = get.get(filter_key, None)
            filter_value = self.clean_filter_value(filter_key, filter_value)
            if filter_value:
                kwargs.update({'{0}__{1}'.format(filter_key, filter_lookup): filter_value})
        # tag filtering could add duplicate items to results, so .distinct() is used
        try:
            objects = objects.filter(**kwargs).distinct()
        except FieldError:
            pass
        except ValueError:
            error_msg = "Sorry, there was a problem with your filters. Please try filtering again."
            messages.error(self.request, error_msg)
        return objects

    def sort_list(self, objects):
        """Sorts the list of objects"""
        get = self.request.GET
        sort = get.get('sort', None)
        if sort:
            order = get.get('order', 'asc')
            if order != 'asc':
                sort = '-' + sort
            if sort in ['title', 'status', 'date_submitted']:
                objects = objects.order_by(sort)
        return objects

    def get_context_data(self, **kwargs):
        """Gets basic context, including title, form, and url"""
        context = super(MRFilterableListView, self).get_context_data(**kwargs)
        filter_data = self.get_filter_data()
        context['title'] = self.title
        context['per_page'] = int(self.get_paginate_by(context['object_list']))
        try:
            context['filter_form'] = MRFilterForm(initial=filter_data['filter_initials'])
{'fraction_non_alphanumeric': 0.08802893309222423, 'fraction_numerical': 0.00216998191681736, 'mean_word_length': 2.039349307540119, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1409686', 'n_tokens_mistral': 3726, 'n_tokens_neox': 3546, 'n_words': 1012}