<commit_before>import urllib
from operator import attrgetter

from django.conf import settings
from django.db.models import Sum, Q
from django.utils.translation import ugettext as _
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.views.generic import ListView
from django.core.cache import cache
from django.utils import simplejson as json
from tagging.models import Tag
from tagging.utils import calculate_cloud
from backlinks.pingback.server import default_server
from actstream import actor_stream

from knesset.hashnav.detail import DetailView
from knesset.mks.models import Member, Party
from knesset.mks.forms import VerbsForm
from knesset.mks.utils import percentile
from knesset.laws.models import MemberVotingStatistics, Bill, VoteAction
from knesset.agendas.models import Agenda

from knesset.video.utils import get_videos_queryset
from datetime import date, timedelta

import logging


logger = logging.getLogger("open-knesset.mks")

class MemberListView(ListView):

    model = Member

    def get_template_names(self):
        info = self.request.GET.get('info','bills_pre')
        if info=='abc':
            return ['mks/member_list.html']
        elif info=='graph':
            return ['mks/member_graph.html']
        else:
            return ['mks/member_list_with_bars.html']

    def get_context_data(self, **kwargs):
        info = self.request.GET.get('info','bills_pre')
        if info not in ['abc','bills_proposed','bills_pre',
                        'bills_first','bills_approved','votes',
                        'presence','committees','graph']:
            raise Http404()
        original_context = super(MemberListView, self).get_context_data(**kwargs)
        qs = original_context['object_list'].filter(is_current=True)

        context = cache.get('object_list_by_%s' % info) or {}
        if context:
            original_context.update(context)
            return original_context
        context['past_mks'] = Member.objects.filter(is_current=False)

        context['friend_pages'] = [['.?info=abc',_('By ABC'), False],
                              ['.?info=bills_proposed',_('By number of bills proposed'), False],
                              ['.?info=bills_pre',_('By number of bills pre-approved'), False],
                              ['.?info=bills_first',_('By number of bills first-approved'), False],
                              ['.?info=bills_approved',_('By number of bills approved'), False],
                              ['.?info=votes', _('By number of votes per month'), False],
                              ['.?info=presence', _('By average weekly hours of presence'), False],
                              ['.?info=committees', _('By average monthly committee meetings'), False],
                              ['.?info=graph', _('Graphical view'), False]]
        if info=='abc':
            context['friend_pages'][0][2] = True
            context['title'] = _('Members')
        elif info=='bills_proposed':
            qs = qs.order_by('-bills_stats_proposed')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_proposed'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_proposed')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_proposed'})
            context['friend_pages'][1][2] = True
            context['norm_factor'] = float(qs[0].bills_stats_proposed)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills proposed'))
            context['bill_stage'] = 'proposed'
        elif info=='bills_pre':
            qs = qs.order_by('-bills_stats_pre')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_pre'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_pre')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_pre'})
            context['friend_pages'][2][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills pre-approved'))
            context['bill_stage'] = 'pre'
        elif info=='bills_first':
            qs = qs.order_by('-bills_stats_first')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_first'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_first')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_first'})
            context['friend_pages'][3][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills first-approved'))
            context['bill_stage'] = 'first'
        elif info=='bills_approved':
            qs = qs.order_by('-bills_stats_approved')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_approved'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_approved')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_approved'})
            context['friend_pages'][4][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills approved'))
            context['bill_stage'] = 'approved'
        elif info=='votes':
            qs = list(qs)
            vs = list(MemberVotingStatistics.objects.all())
            vs = dict(zip([x.member_id for x in vs],vs))
            for x in qs:
                x.extra = vs[x.id].average_votes_per_month()
            qs.sort(key=lambda x:x.extra, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.voting_statistics.average_votes_per_month()
            context['past_mks'].sort(key=lambda x:x.extra, reverse=True)
            context['friend_pages'][5][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of votes per month'))
        elif info=='presence':
            qs = list(qs)
            for x in qs:
                x.extra = x.average_weekly_presence()
            qs.sort(key=lambda x:x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.average_weekly_presence()
            context['past_mks'].sort(key=lambda x:x.extra or 0, reverse=True)
            context['friend_pages'][6][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By average weekly hours of presence'))
        elif info=='committees':
            qs = list(qs)
            for x in qs:
                x.extra = x.committee_meetings_per_month()
            qs.sort(key=lambda x:x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.committee_meetings_per_month()
            context['past_mks'].sort(key=lambda x:x.extra or 0, reverse=True)
            context['friend_pages'][7][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By average monthly committee meetings'))
        elif info=='graph':
            context['friend_pages'][8][2] = True
            context['title'] = "%s %s" % (_('Members'), _('Graphical view'))
        context['object_list']=qs
        cache.set('object_list_by_%s' % info, context, settings.LONG_CACHE_TIME)
        original_context.update(context)
        return original_context

class MemberDetailView(DetailView):

    model = Member

    def calc_percentile(self,member,outdict,inprop,outvalprop,outpercentileprop):
        # store in instance var if needed, no need to access cache for each
        # call.
        #
        # If not found in the instance, than try to get from cache (and set if
        # not found), plus setting it as an instance var
        all_members = getattr(self, '_all_members', None)

        if not all_members:
            all_members = cache.get('all_members', None)
            if not all_members:
                self._all_members = all_members = list(
                    Member.objects.filter(is_current=True).values())
                cache.set('all_members', all_members, settings.LONG_CACHE_TIME)

        member_count = float(len(all_members))
        member_val = getattr(member,inprop) or 0

        avg = sum(x.get(inprop) or 0 for x in all_members) / member_count

        var = sum(((x.get(inprop) or 0) - avg) ** 2 for x in all_members) / member_count

        outdict[outvalprop] = member_val
        outdict[outpercentileprop] = percentile(avg,var,member_val) if var != 0 else 0

    def calc_bill_stats(self,member,bills_statistics,stattype):
        self.calc_percentile( member,
                              bills_statistics,
                              'bills_stats_%s' % stattype,
                              stattype,
                              '%s_percentile' % stattype)


    def get_context_data (self, **kwargs):
        context = super(MemberDetailView, self).get_context_data(**kwargs)
        member = context['object']
        if self.request.user.is_authenticated():
            p = self.request.user.get_profile()
            watched = member in p.members<commit_msg>Use select_related to pervent additional query on party<commit_after>import urllib
from operator import attrgetter

from django.conf import settings
from django.db.models import Sum, Q
from django.utils.translation import ugettext as _
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.views.generic import ListView
from django.core.cache import cache
from django.utils import simplejson as json
from tagging.models import Tag
from tagging.utils import calculate_cloud
from backlinks.pingback.server import default_server
from actstream import actor_stream

from knesset.hashnav.detail import DetailView
from knesset.mks.models import Member, Party
from knesset.mks.forms import VerbsForm
from knesset.mks.utils import percentile
from knesset.laws.models import MemberVotingStatistics, Bill, VoteAction
from knesset.agendas.models import Agenda

from knesset.video.utils import get_videos_queryset
from datetime import date, timedelta

import logging


logger = logging.getLogger("open-knesset.mks")

class MemberListView(ListView):

    model = Member

    def get_template_names(self):
        info = self.request.GET.get('info','bills_pre')
        if info=='abc':
            return ['mks/member_list.html']
        elif info=='graph':
            return ['mks/member_graph.html']
        else:
            return ['mks/member_list_with_bars.html']

    def get_context_data(self, **kwargs):
        info = self.request.GET.get('info','bills_pre')
        if info not in ['abc','bills_proposed','bills_pre',
                        'bills_first','bills_approved','votes',
                        'presence','committees','graph']:
            raise Http404()
        original_context = super(MemberListView, self).get_context_data(**kwargs)
        qs = original_context['object_list'].filter(is_current=True)

        context = cache.get('object_list_by_%s' % info) or {}
        if context:
            original_context.update(context)
            return original_context
        context['past_mks'] = Member.objects.filter(is_current=False)

        context['friend_pages'] = [['.?info=abc',_('By ABC'), False],
                              ['.?info=bills_proposed',_('By number of bills proposed'), False],
                              ['.?info=bills_pre',_('By number of bills pre-approved'), False],
                              ['.?info=bills_first',_('By number of bills first-approved'), False],
                              ['.?info=bills_approved',_('By number of bills approved'), False],
                              ['.?info=votes', _('By number of votes per month'), False],
                              ['.?info=presence', _('By average weekly hours of presence'), False],
                              ['.?info=committees', _('By average monthly committee meetings'), False],
                              ['.?info=graph', _('Graphical view'), False]]
        if info=='abc':
            context['friend_pages'][0][2] = True
            context['title'] = _('Members')
        elif info=='bills_proposed':
            qs = qs.order_by('-bills_stats_proposed')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_proposed'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_proposed')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_proposed'})
            context['friend_pages'][1][2] = True
            context['norm_factor'] = float(qs[0].bills_stats_proposed)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills proposed'))
            context['bill_stage'] = 'proposed'
        elif info=='bills_pre':
            qs = qs.order_by('-bills_stats_pre')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_pre'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_pre')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_pre'})
            context['friend_pages'][2][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills pre-approved'))
            context['bill_stage'] = 'pre'
        elif info=='bills_first':
            qs = qs.order_by('-bills_stats_first')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_first'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_first')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_first'})
            context['friend_pages'][3][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills first-approved'))
            context['bill_stage'] = 'first'
        elif info=='bills_approved':
            qs = qs.order_by('-bills_stats_approved')\
                    .select_related('current_party')\
                    .extra(select={'extra':'bills_stats_approved'})
            context['past_mks'] = context['past_mks'].order_by('-bills_stats_approved')\
                                                     .select_related('current_party')\
                                                     .extra(select={'extra':'bills_stats_approved'})
            context['friend_pages'][4][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of bills approved'))
            context['bill_stage'] = 'approved'
        elif info=='votes':
            qs = list(qs)
            vs = list(MemberVotingStatistics.objects.all())
            vs = dict(zip([x.member_id for x in vs],vs))
            for x in qs:
                x.extra = vs[x.id].average_votes_per_month()
            qs.sort(key=lambda x:x.extra, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.voting_statistics.average_votes_per_month()
            context['past_mks'].sort(key=lambda x:x.extra, reverse=True)
            context['friend_pages'][5][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By number of votes per month'))
        elif info=='presence':
            qs = list(qs)
            for x in qs:
                x.extra = x.average_weekly_presence()
            qs.sort(key=lambda x:x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.average_weekly_presence()
            context['past_mks'].sort(key=lambda x:x.extra or 0, reverse=True)
            context['friend_pages'][6][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By average weekly hours of presence'))
        elif info=='committees':
            qs = list(qs)
            for x in qs:
                x.extra = x.committee_meetings_per_month()
            qs.sort(key=lambda x:x.extra or 0, reverse=True)
            context['past_mks'] = list(context['past_mks'])
            for x in context['past_mks']:
                x.extra = x.committee_meetings_per_month()
            context['past_mks'].sort(key=lambda x:x.extra or 0, reverse=True)
            context['friend_pages'][7][2] = True
            context['norm_factor'] = float(qs[0].extra)/50.0
            context['title'] = "%s %s" % (_('Members'), _('By average monthly committee meetings'))
        elif info=='graph':
            context['friend_pages'][8][2] = True
            context['title'] = "%s %s" % (_('Members'), _('Graphical view'))
        context['object_list']=qs
        cache.set('object_list_by_%s' % info, context, settings.LONG_CACHE_TIME)
        original_context.update(context)
        return original_context

class MemberDetailView(DetailView):

    queryset = Member.objects.select_related('current_party')

    def calc_percentile(self,member,outdict,inprop,outvalprop,outpercentileprop):
        # store in instance var if needed, no need to access cache for each
        # call.
        #
        # If not found in the instance, than try to get from cache (and set if
        # not found), plus setting it as an instance var. Also removes default
        # ordering by name (we don't need it)
        all_members = getattr(self, '_all_members', None)

        if not all_members:
            all_members = cache.get('all_members', None)
            if not all_members:
                self._all_members = all_members = list(
                    Member.objects.filter(is_current=True).order_by().values())
                cache.set('all_members', all_members, settings.LONG_CACHE_TIME)

        member_count = float(len(all_members))
        member_val = getattr(member,inprop) or 0

        avg = sum(x[inprop] or 0 for x in all_members) / member_count

        var = sum(((x[inprop] or 0) - avg) ** 2 for x in all_members) / member_count

        outdict[outvalprop] = member_val
        outdict[outpercentileprop] = percentile(avg,var,member_val) if var != 0 else 0

    def calc_bill_stats(self,member,bills_statistics,stattype):
        self.calc_percentile( member,
                              bills_statistics,
                              'bills_stats_%s' % stattype,
                              stattype,
                              '%s_percentile' % stattype)


    def get_context_data (self, **kwargs):
        context = super(MemberDetailView, self).get_context_data(**kwargs)
        member = context['object']
        if self.request.user.is_authenticated():
            p = self.request.user.get_profile()
            watched = member in p.members
{'fraction_non_alphanumeric': 0.13003157063930545, 'fraction_numerical': 0.00611681136543015, 'mean_word_length': 2.159264453794608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3710038', 'n_tokens_mistral': 5900, 'n_tokens_neox': 5583, 'n_words': 1174}