<commit_before>import json
import math
import logging

from django.shortcuts import render, HttpResponse, redirect
from django.http import JsonResponse
from django.contrib.auth.decorators import login_required
from django.views.generic import View
from django.conf import settings
from django.contrib.staticfiles.templatetags.staticfiles import static

from .serializers import (AlertSerializer, HouseholdSurveyJSONSerializer,
                          TeamMemberSerializer, HouseholdMemberSerializer,
                          SurveyMapSerializer)

from .models import Alert
from .models import HouseholdSurveyJSON
from .models import Clusters
from .models import QuestionnaireSpecification
from .models import ClustersPerFirstAdminLevel
from .models import FirstAdminLevels
from .models import FirstAdminLevelsReserveClusters
from .models import TeamMember
from .models import HouseholdMember

from rest_framework import viewsets, pagination
from rest_framework.decorators import list_route
from rest_framework.response import Response

from importer import anthrocomputation
from django.utils.decorators import method_decorator
from django_dont_vary_on.decorators import dont_vary_on
from collections import Counter


class HardLimitPagination(pagination.PageNumberPagination):
    page_size = 1000
    max_page_size = 1000


class TeamMemberViewset(viewsets.ModelViewSet):
    queryset = TeamMember.objects.all()
    permission_classes = ()
    serializer_class = TeamMemberSerializer
    template_name = 'dashboard/teammember.html'
    lookup_field = 'pk'


class HouseholdMemberViewset(viewsets.ModelViewSet):
    pagination_class = pagination.PageNumberPagination
    queryset = HouseholdMember.objects.all()
    serializer_class = HouseholdMemberSerializer

    @list_route(methods=['get'])
    def age_distribution(self, request, format):

        c = HouseholdMember.children
        h = HouseholdMember.objects

        if request.GET.get('team_lead'):
            team_lead = TeamMember.objects.get(pk=int(request.GET['team_lead']))
            c = c.by_teamlead(team_lead)
            h = h.by_teamlead(team_lead)

        if request.GET.get('stratum'):
            stratum = request.GET['stratum']
            c = c.by_first_admin_level(stratum)
            h = h.by_first_admin_level(stratum)

        # This specific brand of hell is to filter out bad data.
        # Specifically when the months rounded to the year are not the same as the year entered
        # Nothing we can do about that. There are two ways of entering the same data and sometimes it doesn't match.
        months_list = [a_m[u'age_months']  for (a_m, a_y) in HouseholdMember.children.all().values_list('extra_questions', 'age_in_years') if a_m[u'age_months'] / 12 == a_y]

        children_age_distrobutions = list({'count': v, 'age_in_months': k} for k, v in Counter(months_list).items())
        return Response({
            'age_distribution': {
                'children': children_age_distrobutions,
                'household_member': h.age_distribution_in_years(),
            }
        })


class HouseholdSurveyJSONViewset(viewsets.ModelViewSet):
    queryset = HouseholdSurveyJSON.objects.prefetch_related('members').all()
    serializer_class = HouseholdSurveyJSONSerializer

    def list(request, *args, **kwargs):
        return Response([])

    def retrieve(request, *args, **kwargs):
        return Response([])

    permission_classes = ()  # Allow any


@login_required
def home(request):
    response = {}
    return render(request, 'dashboard/home.html', response)


@login_required
def fieldwork(request):
    response = {}
    return render(request, 'dashboard/fieldwork.html', response)


@login_required
def age_distribution(request):
    response = {}
    return render(request, 'dashboard/age_distribution.html', response)


@login_required
def survey_completed_teams(request):
    response = {}
    return render(request, 'dashboard/survey_completed_teams.html', response)


@login_required
def survey_completed_strata(request):
    response = {}
    return render(request, 'dashboard/survey_completed_strata.html', response)


@login_required
def missing_data(request):

    h = HouseholdMember.objects

    if request.GET.get('team_lead'):
        team_lead = TeamMember.objects.get(pk=int(request.GET['team_lead']))
        h = h.by_teamlead(team_lead)

    if request.GET.get('stratum'):
        stratum = request.GET['stratum']
        h = h.by_first_admin_level(stratum)

    return render(request, 'dashboard/missing_data.html', {
        'existing_data': HouseholdMember.missing_data(h),
    })


@login_required
def child_anthropometry(request):
    response = {}
    return render(request, 'dashboard/child_anthropometry.html', response)


@login_required
def personnel(request):
    response = {}
    return render(request, 'dashboard/personnel.html', response)


@login_required
def time_of_data_collection(request):
    response = {}
    return render(request, 'dashboard/time_of_data_collection.html', response)


class LoginRequiredView(View):

    @classmethod
    def as_view(cls, **kwargs):
        view = super(LoginRequiredView, cls).as_view(**kwargs)
        return login_required(view)


class AggregateSurveyDataJSONView(View):

    @method_decorator(dont_vary_on('Cookie'))
    def get(self, request, *args, **kwargs):
        """Generates an HTTP response with a JSON document containing
        information from all surveys
        """
        return redirect(static('aggregatesurveydatajsonview.json'))
        docs = HouseholdSurveyJSON.objects.all()
        survey_data = []
        count = docs.count()
        for doc in docs:
            survey_data.append(self._clean_json(doc))
            count -= 1
            if count % 10 == 0:
                logging.warn(count)

        return JsonResponse({'survey_data': survey_data})


    @classmethod
    def _clean_json(cls, doc):
        """Clean JSON document given containing data from a
        single survey to decrease the amount of data sent to the
        client.
        """

        output = {}

        # map the top-level attributes<commit_msg>comment old code<commit_after>import json
import math
import logging

from django.shortcuts import render, HttpResponse, redirect
from django.http import JsonResponse
from django.contrib.auth.decorators import login_required
from django.views.generic import View
from django.conf import settings
from django.contrib.staticfiles.templatetags.staticfiles import static

from .serializers import (AlertSerializer, HouseholdSurveyJSONSerializer,
                          TeamMemberSerializer, HouseholdMemberSerializer,
                          SurveyMapSerializer)

from .models import Alert
from .models import HouseholdSurveyJSON
from .models import Clusters
from .models import QuestionnaireSpecification
from .models import ClustersPerFirstAdminLevel
from .models import FirstAdminLevels
from .models import FirstAdminLevelsReserveClusters
from .models import TeamMember
from .models import HouseholdMember

from rest_framework import viewsets, pagination
from rest_framework.decorators import list_route
from rest_framework.response import Response

from importer import anthrocomputation
from django.utils.decorators import method_decorator
from django_dont_vary_on.decorators import dont_vary_on
from collections import Counter


class HardLimitPagination(pagination.PageNumberPagination):
    page_size = 1000
    max_page_size = 1000


class TeamMemberViewset(viewsets.ModelViewSet):
    queryset = TeamMember.objects.all()
    permission_classes = ()
    serializer_class = TeamMemberSerializer
    template_name = 'dashboard/teammember.html'
    lookup_field = 'pk'


class HouseholdMemberViewset(viewsets.ModelViewSet):
    pagination_class = pagination.PageNumberPagination
    queryset = HouseholdMember.objects.all()
    serializer_class = HouseholdMemberSerializer

    @list_route(methods=['get'])
    def age_distribution(self, request, format):

        c = HouseholdMember.children
        h = HouseholdMember.objects

        if request.GET.get('team_lead'):
            team_lead = TeamMember.objects.get(pk=int(request.GET['team_lead']))
            c = c.by_teamlead(team_lead)
            h = h.by_teamlead(team_lead)

        if request.GET.get('stratum'):
            stratum = request.GET['stratum']
            c = c.by_first_admin_level(stratum)
            h = h.by_first_admin_level(stratum)

        # This specific brand of hell is to filter out bad data.
        # Specifically when the months rounded to the year are not the same as the year entered
        # Nothing we can do about that. There are two ways of entering the same data and sometimes it doesn't match.
        months_list = [a_m[u'age_months']  for (a_m, a_y) in HouseholdMember.children.all().values_list('extra_questions', 'age_in_years') if a_m[u'age_months'] / 12 == a_y]

        children_age_distrobutions = list({'count': v, 'age_in_months': k} for k, v in Counter(months_list).items())
        return Response({
            'age_distribution': {
                'children': children_age_distrobutions,
                'household_member': h.age_distribution_in_years(),
            }
        })


class HouseholdSurveyJSONViewset(viewsets.ModelViewSet):
    queryset = HouseholdSurveyJSON.objects.prefetch_related('members').all()
    serializer_class = HouseholdSurveyJSONSerializer

    def list(request, *args, **kwargs):
        return Response([])

    def retrieve(request, *args, **kwargs):
        return Response([])

    permission_classes = ()  # Allow any


@login_required
def home(request):
    response = {}
    return render(request, 'dashboard/home.html', response)


@login_required
def fieldwork(request):
    response = {}
    return render(request, 'dashboard/fieldwork.html', response)


@login_required
def age_distribution(request):
    response = {}
    return render(request, 'dashboard/age_distribution.html', response)


@login_required
def survey_completed_teams(request):
    response = {}
    return render(request, 'dashboard/survey_completed_teams.html', response)


@login_required
def survey_completed_strata(request):
    response = {}
    return render(request, 'dashboard/survey_completed_strata.html', response)


@login_required
def missing_data(request):

    h = HouseholdMember.objects

    if request.GET.get('team_lead'):
        team_lead = TeamMember.objects.get(pk=int(request.GET['team_lead']))
        h = h.by_teamlead(team_lead)

    if request.GET.get('stratum'):
        stratum = request.GET['stratum']
        h = h.by_first_admin_level(stratum)

    return render(request, 'dashboard/missing_data.html', {
        'existing_data': HouseholdMember.missing_data(h),
    })


@login_required
def child_anthropometry(request):
    response = {}
    return render(request, 'dashboard/child_anthropometry.html', response)


@login_required
def personnel(request):
    response = {}
    return render(request, 'dashboard/personnel.html', response)


@login_required
def time_of_data_collection(request):
    response = {}
    return render(request, 'dashboard/time_of_data_collection.html', response)


class LoginRequiredView(View):

    @classmethod
    def as_view(cls, **kwargs):
        view = super(LoginRequiredView, cls).as_view(**kwargs)
        return login_required(view)


class AggregateSurveyDataJSONView(View):

    @method_decorator(dont_vary_on('Cookie'))
    def get(self, request, *args, **kwargs):
        """Generates an HTTP response with a JSON document containing
        information from all surveys
        """
        return redirect(static('aggregatesurveydatajsonview.json'))
        #docs = HouseholdSurveyJSON.objects.all()
        #survey_data = []
        #count = docs.count()
        #for doc in docs:
        #    survey_data.append(self._clean_json(doc))
        #    count -= 1
        #    if count % 10 == 0:
        #        logging.warn(count)
        #
        #return JsonResponse({'survey_data': survey_data})


    @classmethod
    def _clean_json(cls, doc):
        """Clean JSON document given containing data from a
        single survey to decrease the amount of data sent to the
        client.
        """

        output = {}

        # map the top-level attributes
{'fraction_non_alphanumeric': 0.0900316327358261, 'fraction_numerical': 0.0022710682131559736, 'mean_word_length': 3.5819397993311037, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '486908', 'n_tokens_mistral': 3599, 'n_tokens_neox': 3426, 'n_words': 843}