<commit_before>import datetime
import re
import json

from django.utils.six import text_type as str
from django.conf import settings
from django.core.files import File
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt
from django.utils.translation import ugettext_lazy as _
from django.http import Http404, HttpResponse
from django.template.defaultfilters import slugify
from django.contrib import messages
from django.contrib.auth import get_user_model
from django.contrib.sitemaps import Sitemap

from haystack.query import SearchQuerySet
from taggit.models import Tag

from froide.account.forms import NewUserForm
from froide.account.models import AccountManager
from froide.publicbody.forms import PublicBodyForm
from froide.publicbody.models import PublicBody, PublicBodyTag, FoiLaw, Jurisdiction
from froide.frontpage.models import FeaturedRequest
from froide.helper.utils import render_400, render_403
from froide.helper.cache import cache_anonymous_page
from froide.redaction.utils import convert_to_pdf

from .models import FoiRequest, FoiMessage, FoiEvent, FoiAttachment
from .forms import (RequestForm, ConcreteLawForm, TagFoiRequestForm,
        SendMessageForm, FoiRequestStatusForm, MakePublicBodySuggestionForm,
        PostalReplyForm, PostalAttachmentForm, MessagePublicBodySenderForm,
        EscalationMessageForm)
from .feeds import LatestFoiRequestsFeed, LatestFoiRequestsFeedAtom
from .tasks import process_mail
from .foi_mail import package_foirequest

X_ACCEL_REDIRECT_PREFIX = getattr(settings, 'X_ACCEL_REDIRECT_PREFIX', '')
User = get_user_model()


@cache_anonymous_page(15 * 60)
def index(request):
    successful_foi_requests = FoiRequest.published.successful()[:8]
    unsuccessful_foi_requests = FoiRequest.published.unsuccessful()[:8]
    featured = FeaturedRequest.objects.getFeatured()
    return render(request, 'index.html',
            {'featured': featured,
            'successful_foi_requests': successful_foi_requests,
            'unsuccessful_foi_requests': unsuccessful_foi_requests,
            'foicount': FoiRequest.published.for_list_view().count(),
            'pbcount': PublicBody.objects.get_list().count()
        })


def dashboard(request):
    if not request.user.is_staff:
        return render_403(request)
    context = {}
    user = {}
    start_date = timezone.utc.localize(datetime.datetime(2011, 7, 30))
    for u in User.objects.filter(
            is_active=True,
            date_joined__gte=start_date):
        d = u.date_joined.date().isoformat()
        user.setdefault(d, 0)
        user[d] += 1
    context['user'] = sorted([{'date': k, 'num': v, 'symbol': 'user'} for k, v in user.items()], key=lambda x: x['date'])
    total = 0
    for user in context['user']:
        total += user['num']
        user['total'] = total
    foirequest = {}
    foi_query = FoiRequest.objects.filter(
            is_foi=True,
            public_body__isnull=False,
            first_message__gte=start_date
    )
    if request.GET.get('notsameas'):
        foi_query = foi_query.filter(same_as__isnull=True)
    if request.GET.get('public'):
        foi_query = foi_query.filter(public=True)
    for u in foi_query:
        d = u.first_message.date().isoformat()
        foirequest.setdefault(d, 0)
        foirequest[d] += 1
    context['foirequest'] = sorted([{'date': k, 'num': v, 'symbol': 'user'} for k, v in foirequest.items()], key=lambda x: x['date'])
    total = 0
    for req in context['foirequest']:
        total += req['num']
        req['total'] = total
    return render(request, 'foirequest/dashboard.html', {'data': json.dumps(context)})


def list_requests(request, status=None, topic=None, tag=None,
        jurisdiction=None, public_body=None, not_foi=False, feed=None):
    context = {
        'filtered': True
    }
    manager = FoiRequest.published
    if not_foi:
        manager = FoiRequest.published_not_foi
    topic_list = PublicBodyTag.objects.get_topic_list()
    if status is None:
        status = request.GET.get(str(_('status')), None)
    status_url = status
    foi_requests = manager.for_list_view()
    if status is not None:
        func_status = FoiRequest.get_status_from_url(status)
        if func_status is None:
            raise Http404
        func, status = func_status
        foi_requests = foi_requests.filter(func(status))
        context.update({
            'status': FoiRequest.get_readable_status(status),
            'status_description': FoiRequest.get_status_description(status)
        })
    elif topic is not None:
        topic = get_object_or_404(PublicBodyTag, slug=topic)
        foi_requests = manager.for_list_view().filter(public_body__tags=topic)
        context.update({
            'topic': topic,
        })
    elif tag is not None:
        tag = get_object_or_404(Tag, slug=tag)
        foi_requests = manager.for_list_view().filter(tags=tag)
        context.update({
            'tag': tag
        })
    else:
        foi_requests = manager.for_list_view()
        context['filtered'] = False

    if jurisdiction is not None:
        jurisdiction = get_object_or_404(Jurisdiction, slug=jurisdiction)
        foi_requests = foi_requests.filter(jurisdiction=jurisdiction)
        context.update({
            'jurisdiction': jurisdiction
        })
    elif public_body is not None:
        public_body = get_object_or_404(PublicBody, slug=public_body)
        foi_requests = foi_requests.filter(public_body=public_body)
        context.update({
            'public_body': public_body
        })
        context['filtered'] = True
        context['jurisdiction_list'] = Jurisdiction.objects.get_visible()
    else:
        context['jurisdiction_list'] = Jurisdiction.objects.get_visible()
        context['filtered'] = False

    if feed is not None:
        if feed == 'rss':
            klass = LatestFoiRequestsFeed
        else:
            klass = LatestFoiRequestsFeedAtom
        return klass(foi_requests, status=status_url, topic=topic,
            tag=tag, jurisdiction=jurisdiction)(request)

    count = foi_requests.count()

    page = request.GET.get('page')
    paginator = Paginator(foi_requests, 20)
    try:
        foi_requests = paginator.page(page)
    except PageNotAnInteger:
        foi_requests = paginator.page(1)
    except EmptyPage:
        foi_requests = paginator.page(paginator.num_pages)

    context.update({
        'page_title': _("FoI Requests"),
        'count': count,
        'not_foi': not_foi,<commit_msg>Add get parameter to view up to 500 request on one page<commit_after>import datetime
import re
import json

from django.utils.six import text_type as str
from django.conf import settings
from django.core.files import File
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.utils import timezone
from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.http import require_POST
from django.views.decorators.csrf import csrf_exempt
from django.utils.translation import ugettext_lazy as _
from django.http import Http404, HttpResponse
from django.template.defaultfilters import slugify
from django.contrib import messages
from django.contrib.auth import get_user_model
from django.contrib.sitemaps import Sitemap

from haystack.query import SearchQuerySet
from taggit.models import Tag

from froide.account.forms import NewUserForm
from froide.account.models import AccountManager
from froide.publicbody.forms import PublicBodyForm
from froide.publicbody.models import PublicBody, PublicBodyTag, FoiLaw, Jurisdiction
from froide.frontpage.models import FeaturedRequest
from froide.helper.utils import render_400, render_403
from froide.helper.cache import cache_anonymous_page
from froide.redaction.utils import convert_to_pdf

from .models import FoiRequest, FoiMessage, FoiEvent, FoiAttachment
from .forms import (RequestForm, ConcreteLawForm, TagFoiRequestForm,
        SendMessageForm, FoiRequestStatusForm, MakePublicBodySuggestionForm,
        PostalReplyForm, PostalAttachmentForm, MessagePublicBodySenderForm,
        EscalationMessageForm)
from .feeds import LatestFoiRequestsFeed, LatestFoiRequestsFeedAtom
from .tasks import process_mail
from .foi_mail import package_foirequest

X_ACCEL_REDIRECT_PREFIX = getattr(settings, 'X_ACCEL_REDIRECT_PREFIX', '')
User = get_user_model()


@cache_anonymous_page(15 * 60)
def index(request):
    successful_foi_requests = FoiRequest.published.successful()[:8]
    unsuccessful_foi_requests = FoiRequest.published.unsuccessful()[:8]
    featured = FeaturedRequest.objects.getFeatured()
    return render(request, 'index.html',
            {'featured': featured,
            'successful_foi_requests': successful_foi_requests,
            'unsuccessful_foi_requests': unsuccessful_foi_requests,
            'foicount': FoiRequest.published.for_list_view().count(),
            'pbcount': PublicBody.objects.get_list().count()
        })


def dashboard(request):
    if not request.user.is_staff:
        return render_403(request)
    context = {}
    user = {}
    start_date = timezone.utc.localize(datetime.datetime(2011, 7, 30))
    for u in User.objects.filter(
            is_active=True,
            date_joined__gte=start_date):
        d = u.date_joined.date().isoformat()
        user.setdefault(d, 0)
        user[d] += 1
    context['user'] = sorted([{'date': k, 'num': v, 'symbol': 'user'} for k, v in user.items()], key=lambda x: x['date'])
    total = 0
    for user in context['user']:
        total += user['num']
        user['total'] = total
    foirequest = {}
    foi_query = FoiRequest.objects.filter(
            is_foi=True,
            public_body__isnull=False,
            first_message__gte=start_date
    )
    if request.GET.get('notsameas'):
        foi_query = foi_query.filter(same_as__isnull=True)
    if request.GET.get('public'):
        foi_query = foi_query.filter(public=True)
    for u in foi_query:
        d = u.first_message.date().isoformat()
        foirequest.setdefault(d, 0)
        foirequest[d] += 1
    context['foirequest'] = sorted([{'date': k, 'num': v, 'symbol': 'user'} for k, v in foirequest.items()], key=lambda x: x['date'])
    total = 0
    for req in context['foirequest']:
        total += req['num']
        req['total'] = total
    return render(request, 'foirequest/dashboard.html', {'data': json.dumps(context)})


def list_requests(request, status=None, topic=None, tag=None,
        jurisdiction=None, public_body=None, not_foi=False, feed=None):
    context = {
        'filtered': True
    }
    manager = FoiRequest.published
    if not_foi:
        manager = FoiRequest.published_not_foi
    topic_list = PublicBodyTag.objects.get_topic_list()
    if status is None:
        status = request.GET.get(str(_('status')), None)
    status_url = status
    foi_requests = manager.for_list_view()
    if status is not None:
        func_status = FoiRequest.get_status_from_url(status)
        if func_status is None:
            raise Http404
        func, status = func_status
        foi_requests = foi_requests.filter(func(status))
        context.update({
            'status': FoiRequest.get_readable_status(status),
            'status_description': FoiRequest.get_status_description(status)
        })
    elif topic is not None:
        topic = get_object_or_404(PublicBodyTag, slug=topic)
        foi_requests = manager.for_list_view().filter(public_body__tags=topic)
        context.update({
            'topic': topic,
        })
    elif tag is not None:
        tag = get_object_or_404(Tag, slug=tag)
        foi_requests = manager.for_list_view().filter(tags=tag)
        context.update({
            'tag': tag
        })
    else:
        foi_requests = manager.for_list_view()
        context['filtered'] = False

    if jurisdiction is not None:
        jurisdiction = get_object_or_404(Jurisdiction, slug=jurisdiction)
        foi_requests = foi_requests.filter(jurisdiction=jurisdiction)
        context.update({
            'jurisdiction': jurisdiction
        })
    elif public_body is not None:
        public_body = get_object_or_404(PublicBody, slug=public_body)
        foi_requests = foi_requests.filter(public_body=public_body)
        context.update({
            'public_body': public_body
        })
        context['filtered'] = True
        context['jurisdiction_list'] = Jurisdiction.objects.get_visible()
    else:
        context['jurisdiction_list'] = Jurisdiction.objects.get_visible()
        context['filtered'] = False

    if feed is not None:
        if feed == 'rss':
            klass = LatestFoiRequestsFeed
        else:
            klass = LatestFoiRequestsFeedAtom
        return klass(foi_requests, status=status_url, topic=topic,
            tag=tag, jurisdiction=jurisdiction)(request)

    count = foi_requests.count()

    page = request.GET.get('page')
    paginator = Paginator(foi_requests, 20)

    if request.GET.get('all') is not None:
        if count <= 500:
            paginator = Paginator(foi_requests, count)
    try:
        foi_requests = paginator.page(page)
    except PageNotAnInteger:
        foi_requests = paginator.page(1)
    except EmptyPage:
        foi_requests = paginator.page(paginator.num_pages)

    context.update({
        'page_title': _("FoI Requests"),
        'count': count,
        'not_foi': not_foi,
{'fraction_non_alphanumeric': 0.09704764688687519, 'fraction_numerical': 0.008038585209003215, 'mean_word_length': 3.3555060471037557, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7360386', 'n_tokens_mistral': 4211, 'n_tokens_neox': 4090, 'n_words': 917}