<commit_before>import urllib
from operator import attrgetter
from itertools import chain

from django.conf import settings
from django.db.models import Sum, Q
from django.utils.translation import ugettext as _
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.views.generic import ListView, TemplateView, RedirectView
from django.views.decorators.csrf import ensure_csrf_cookie
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.utils import simplejson as json, simplejson
from django.utils.decorators import method_decorator
from django.contrib.contenttypes.models import ContentType
from django.shortcuts import get_object_or_404
from backlinks.pingback.server import default_server
from actstream import actor_stream
from actstream.models import Follow
from hashnav.detail import DetailView
from models import Member, Party, Knesset
from polyorg.models import CandidateList
from utils import percentile
from laws.models import MemberVotingStatistics, Bill, VoteAction
from agendas.models import Agenda
from auxiliary.views import CsvView

from video.utils import get_videos_queryset
from datetime import date, timedelta

import logging
from auxiliary.views import GetMoreView
from auxiliary.serializers import PromiseAwareJSONEncoder


logger = logging.getLogger("open-knesset.mks")


class MemberRedirectView(RedirectView):
    "Redirect to first stats view"

    def get_redirect_url(self):
        return reverse('member-stats', kwargs={'stat_type': MemberListView.pages[0][0]})


class MemberListView(ListView):

    pages = (
        ('abc', _('By ABC')),
        ('bills_proposed', _('By number of bills proposed')),
        ('bills_pre', _('By number of bills pre-approved')),
        ('bills_first', _('By number of bills first-approved')),
        ('bills_approved', _('By number of bills approved')),
        ('votes', _('By number of votes per month')),
        ('presence', _('By average weekly hours of presence')),
        ('committees', _('By average monthly committee meetings')),
        ('graph', _('Graphical view'))
    )

    def get_queryset(self):
        return Member.current_knesset.all()

    def get_context_data(self, **kwargs):

        info = self.kwargs['stat_type']

        original_context = super(MemberListView, self).get_context_data(**kwargs)
        qs = original_context['object_list'].filter(is_current=True)

        # Do we have it in the cache ? If so, update and return
        context = cache.get('object_list_by_%s' % info) or {}

        if context:
            original_context.update(context)
            return original_context

        context['friend_pages'] = self.pages
        context['stat_type'] = info
        context['title'] = dict(self.pages)[info]
        context['csv_path'] = 'api/v2/member' + '?' + self.request.GET.urlencode() + '&format=csv&limit=0'
        context['past_mks'] = Member.current_knesset.filter(is_current=False)

        # We make sure qs are lists so that the template can get min/max
        if info == 'abc':
            pass
        elif info == 'bills_proposed':
            qs = list(
                qs.order_by('-bills_stats_proposed')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_proposed'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_proposed')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_proposed'})
            )
            context['bill_stage'] = 'proposed'
        elif info == 'bills_pre':
            qs = list(
                qs.order_by('-bills_stats_pre')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_pre'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_pre')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_pre'})
            )
            context['bill_stage'] = 'pre'
        elif info == 'bills_first':
            qs = list(
                qs.order_by('-bills_stats_first')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_first'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_first')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_first'})
            )
            context['bill_stage'] = 'first'
        elif info == 'bills_approved':
            qs = list(
                qs.order_by('-bills_stats_approved')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_approved'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_approved')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_approved'})
            )
            context['bill_stage'] = 'approved'
        elif info == 'votes':
            qs = list(qs)
            vs = list(MemberVotingStatistics.objects.all())
            vs = dict(zip([x.member_id for x in vs], vs))
            for x in qs:
                x.extra = vs[x.id].average_votes_per_month()
            qs.sort(key=lambda x: x.extra, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.voting_statistics.average_votes_per_month()
            context['past_mks'].sort(key=lambda x: x.extra, reverse=True)
        elif info == 'presence':
            qs = list(qs)
            for x in qs:
                x.extra = x.average_weekly_presence()
            qs.sort(key=lambda x: x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.average_weekly_presence()
            context['past_mks'].sort(key=lambda x: x.extra or 0, reverse=True)
        elif info == 'committees':
            qs = list(qs)
            for x in qs:
                x.extra = x.committee_meetings_per_month()
            qs.sort(key=lambda x: x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.committee_meetings_per_month()
            context['past_mks'].sort(key=lambda x: x.extra or 0, reverse=True)
        elif info == 'graph':
            pass

        context['object_list'] = qs

        if info not in ('graph', 'abc'):
            context['max_current'] = qs[0].extra

            if context['past_mks']:
                context['max_past'] = context['past_mks'][0].extra

        cache.set('object_list_by_%s' % info, context, settings.LONG_CACHE_TIME)
        original_context.update(context)
        return original_context


class MemberCsvView(CsvView):
    model = Member
    filename = 'members.csv'<commit_msg>use the db field for average presence to improve performance in member/by/presence/<commit_after>import urllib
from operator import attrgetter
from itertools import chain

from django.conf import settings
from django.db.models import Sum, Q
from django.utils.translation import ugettext as _
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.views.generic import ListView, TemplateView, RedirectView
from django.views.decorators.csrf import ensure_csrf_cookie
from django.core.cache import cache
from django.core.urlresolvers import reverse
from django.utils import simplejson as json, simplejson
from django.utils.decorators import method_decorator
from django.contrib.contenttypes.models import ContentType
from django.shortcuts import get_object_or_404
from backlinks.pingback.server import default_server
from actstream import actor_stream
from actstream.models import Follow
from hashnav.detail import DetailView
from models import Member, Party, Knesset
from polyorg.models import CandidateList
from utils import percentile
from laws.models import MemberVotingStatistics, Bill, VoteAction
from agendas.models import Agenda
from auxiliary.views import CsvView

from video.utils import get_videos_queryset
from datetime import date, timedelta

import logging
from auxiliary.views import GetMoreView
from auxiliary.serializers import PromiseAwareJSONEncoder


logger = logging.getLogger("open-knesset.mks")


class MemberRedirectView(RedirectView):
    "Redirect to first stats view"

    def get_redirect_url(self):
        return reverse('member-stats', kwargs={'stat_type': MemberListView.pages[0][0]})


class MemberListView(ListView):

    pages = (
        ('abc', _('By ABC')),
        ('bills_proposed', _('By number of bills proposed')),
        ('bills_pre', _('By number of bills pre-approved')),
        ('bills_first', _('By number of bills first-approved')),
        ('bills_approved', _('By number of bills approved')),
        ('votes', _('By number of votes per month')),
        ('presence', _('By average weekly hours of presence')),
        ('committees', _('By average monthly committee meetings')),
        ('graph', _('Graphical view'))
    )

    def get_queryset(self):
        return Member.current_knesset.all()

    def get_context_data(self, **kwargs):

        info = self.kwargs['stat_type']

        original_context = super(MemberListView,
                                 self).get_context_data(**kwargs)
        qs = original_context['object_list'].filter(
            is_current=True).select_related('current_party')

        # Do we have it in the cache ? If so, update and return
        context = cache.get('object_list_by_%s' % info) or {}

        if context:
            original_context.update(context)
            return original_context

        context['friend_pages'] = self.pages
        context['stat_type'] = info
        context['title'] = dict(self.pages)[info]
        context['csv_path'] = 'api/v2/member' + '?' + self.request.GET.urlencode() + '&format=csv&limit=0'
        context['past_mks'] = Member.current_knesset.filter(is_current=False)

        # We make sure qs are lists so that the template can get min/max
        if info == 'abc':
            pass
        elif info == 'bills_proposed':
            qs = list(
                qs.order_by('-bills_stats_proposed')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_proposed'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_proposed')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_proposed'})
            )
            context['bill_stage'] = 'proposed'
        elif info == 'bills_pre':
            qs = list(
                qs.order_by('-bills_stats_pre')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_pre'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_pre')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_pre'})
            )
            context['bill_stage'] = 'pre'
        elif info == 'bills_first':
            qs = list(
                qs.order_by('-bills_stats_first')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_first'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_first')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_first'})
            )
            context['bill_stage'] = 'first'
        elif info == 'bills_approved':
            qs = list(
                qs.order_by('-bills_stats_approved')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_approved'})
            )
            context['past_mks'] = list(
                context['past_mks'].order_by('-bills_stats_approved')
                .select_related('current_party')
                .extra(select={'extra': 'bills_stats_approved'})
            )
            context['bill_stage'] = 'approved'
        elif info == 'votes':
            qs = list(qs)
            vs = list(MemberVotingStatistics.objects.all())
            vs = dict(zip([x.member_id for x in vs], vs))
            for x in qs:
                x.extra = vs[x.id].average_votes_per_month()
            qs.sort(key=lambda x: x.extra, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.voting_statistics.average_votes_per_month()
            context['past_mks'].sort(key=lambda x: x.extra, reverse=True)
        elif info == 'presence':
            qs = qs.extra(select={'extra': 'average_weekly_presence_hours'}
                          ).order_by('-extra')
            context['past_mks'] = context['past_mks'].extra(
                select={'extra': 'average_weekly_presence_hours'}).order_by(
                    '-extra')
        elif info == 'committees':
            qs = list(qs)
            for x in qs:
                x.extra = x.committee_meetings_per_month()
            qs.sort(key=lambda x: x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.committee_meetings_per_month()
            context['past_mks'].sort(key=lambda x: x.extra or 0, reverse=True)
        elif info == 'graph':
            pass

        context['object_list'] = qs

        if info not in ('graph', 'abc'):
            context['max_current'] = qs[0].extra

            if context['past_mks']:
                context['max_past'] = context['past_mks'][0].extra

        cache.set('object_list_by_%s' % info, context, settings.LONG_CACHE_TIME)
        original_context.update(context)
        return original_context


class MemberCsvView(CsvView):
    model = Member
    filename = 'members.csv'
{'fraction_non_alphanumeric': 0.11548960593546581, 'fraction_numerical': 0.0020998110170084693, 'mean_word_length': 2.458726700556766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3770634', 'n_tokens_mistral': 4280, 'n_tokens_neox': 4046, 'n_words': 910}