<commit_before>from collections import defaultdict
# map view
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.shortcuts import render_to_response
# http://djangosnippets.org/snippets/365/
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseForbidden,\
         HttpResponseBadRequest
from django.core.files.storage import get_storage_class
from odk_logger.models import XForm, Instance
from odk_viewer.models import DataDictionary, ParsedInstance
from odk_logger.xform_instance_parser import xform_instance_to_dict
from pyxform import Section, Question
from utils.logger_tools import response_with_mimetype_and_name,\
         disposition_ext_and_date, round_down_geopoint
from utils.viewer_tools import image_urls, image_urls_for_form
from utils.user_auth import has_permission
from django.contrib.auth.models import User
from main.models import UserProfile

from csv_writer import CsvWriter
from csv_writer import DataDictionaryWriter
from xls_writer import XlsWriter
from xls_writer import DataDictionary

import json
import os
import urllib2
import zipfile
from tempfile import NamedTemporaryFile
from time import strftime, strptime
from datetime import date

def encode(time_str):
    time = strptime(time_str, "%Y_%m_%d_%H_%M_%S")
    return strftime("%Y-%m-%d %H:%M:%S", time)

def dd_for_params(id_string, owner, request):
    start = end = None
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    if request.GET.get('start'):
        try:
            start = encode(request.GET['start'])
        except ValueError:
            # bad format
            return [False,
                HttpReponseBadRequest('Start time format must be YY_MM_DD_hh_mm_ss')
            ]
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__gte=start)
    if request.GET.get('end'):
        try:
            end = encode(request.GET['end'])
        except ValueError:
            # bad format
            return [False,
                HttpReponseBadRequest('End time format must be YY_MM_DD_hh_mm_ss')
            ]
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__lte=end)
    if start and end:
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__lte=end)
    return [True, dd]

def parse_label_for_display(pi, xpath):
    label = pi.data_dictionary.get_label(xpath)
    if not type(label) == dict:
        label = { 'Unknown': label }
    return label.items()

def average(values):
    if len(values):
        return sum(values, 0.0) / len(values)
    return None


def map_view(request, username, id_string):
    xform = XForm.objects.get(user__username=username, id_string=id_string)
    owner = User.objects.get(username=username)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    context = RequestContext(request)
    context.content_user = owner
    context.xform = xform
    context.profile, created = UserProfile.objects.get_or_create(user=owner)
    points = ParsedInstance.objects.values('lat', 'lng', 'instance').filter(
        instance__user=owner,
        instance__xform__id_string=id_string,
        lat__isnull=False,
        lng__isnull=False)
    center = {
        'lat': round_down_geopoint(average([p['lat'] for p in points])),
        'lng': round_down_geopoint(average([p['lng'] for p in points])),
        }
    def round_down_point(p):
        return {
            'lat': round_down_geopoint(p['lat']),
            'lng': round_down_geopoint(p['lng']),
            'instance': p['instance']
        }
    context.points = json.dumps([round_down_point(p) for p in list(points)])
    context.center = json.dumps(center)
    context.form_view = True
    return render_to_response('map.html', context_instance=context)


def survey_responses(request, pk):
    # todo: do a good job of displaying hierarchical data
    pi = ParsedInstance.objects.get(instance=pk)
    data = pi.to_dict()

    # get rid of keys with leading underscores
    data_for_display = {}
    for k, v in data.items():
        if not k.startswith(u"_"):
            data_for_display[k] = v

    xpaths = data_for_display.keys()
    xpaths.sort(cmp=pi.data_dictionary.get_xpath_cmp())
    label_value_pairs = [
         (parse_label_for_display(pi, xpath),
         data_for_display[xpath]) for xpath in xpaths
    ]
    languages = label_value_pairs[-1][0]
    
    return render_to_response('survey.html', {
            'label_value_pairs': label_value_pairs,
            'image_urls': image_urls(pi.instance),
            'languages': languages, 
            'default_language': languages[0][0]
            })


def csv_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    valid, dd = dd_for_params(id_string, owner, request)
    if not valid: return dd
    writer = DataDictionaryWriter(dd)
    file_path = writer.get_default_file_path()
    writer.write_to_file(file_path)
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('application/csv', id_string,
        extension='csv',
        file_path=file_path, use_local_filesystem=True)
    return response


def xls_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    # TODO: cleanup xls writer so this interface applies
    """
    valid, dd = dd_for_params(id_string, owner, request)
    if not valid: return dd
    """
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    ddw = XlsWriter()
    ddw.set_data_dictionary(dd)
    temp_file = ddw.save_workbook_to_file()
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('vnd.ms-excel', id_string,
        extension='xls')
    response.write(temp_file.getvalue())
    response['Content-Length'] = len(temp_file.getvalue())
    temp_file.close()
    return response


def zip_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('zip', id_string)
    # create zip_file
    z = zipfile.ZipFile('attachments', 'w')
    photos = image_urls_for_form(xform)
    for photo in photos:
        f = NamedTemporaryFile()
        req = urllib2.Request(photo)
        f.write(urllib2.urlopen(req).read())
        z.write(f.name)
    z.close()
    response.content = z<commit_msg>zip code fix<commit_after>from collections import defaultdict
# map view
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.shortcuts import render_to_response
# http://djangosnippets.org/snippets/365/
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseForbidden,\
         HttpResponseBadRequest
from django.core.files.storage import get_storage_class
from odk_logger.models import XForm, Instance
from odk_viewer.models import DataDictionary, ParsedInstance
from odk_logger.xform_instance_parser import xform_instance_to_dict
from pyxform import Section, Question
from utils.logger_tools import response_with_mimetype_and_name,\
         disposition_ext_and_date, round_down_geopoint
from utils.viewer_tools import image_urls, image_urls_for_form
from utils.user_auth import has_permission
from django.contrib.auth.models import User
from main.models import UserProfile

from csv_writer import CsvWriter
from csv_writer import DataDictionaryWriter
from xls_writer import XlsWriter
from xls_writer import DataDictionary

import json
import os
import urllib2
import zipfile
from tempfile import NamedTemporaryFile
from time import strftime, strptime
from datetime import date
from urlparse import urlparse

def encode(time_str):
    time = strptime(time_str, "%Y_%m_%d_%H_%M_%S")
    return strftime("%Y-%m-%d %H:%M:%S", time)

def dd_for_params(id_string, owner, request):
    start = end = None
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    if request.GET.get('start'):
        try:
            start = encode(request.GET['start'])
        except ValueError:
            # bad format
            return [False,
                HttpReponseBadRequest('Start time format must be YY_MM_DD_hh_mm_ss')
            ]
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__gte=start)
    if request.GET.get('end'):
        try:
            end = encode(request.GET['end'])
        except ValueError:
            # bad format
            return [False,
                HttpReponseBadRequest('End time format must be YY_MM_DD_hh_mm_ss')
            ]
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__lte=end)
    if start and end:
        dd.surveys_for_export = lambda d: d.surveys.filter(date_created__lte=end)
    return [True, dd]

def parse_label_for_display(pi, xpath):
    label = pi.data_dictionary.get_label(xpath)
    if not type(label) == dict:
        label = { 'Unknown': label }
    return label.items()

def average(values):
    if len(values):
        return sum(values, 0.0) / len(values)
    return None


def map_view(request, username, id_string):
    xform = XForm.objects.get(user__username=username, id_string=id_string)
    owner = User.objects.get(username=username)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    context = RequestContext(request)
    context.content_user = owner
    context.xform = xform
    context.profile, created = UserProfile.objects.get_or_create(user=owner)
    points = ParsedInstance.objects.values('lat', 'lng', 'instance').filter(
        instance__user=owner,
        instance__xform__id_string=id_string,
        lat__isnull=False,
        lng__isnull=False)
    center = {
        'lat': round_down_geopoint(average([p['lat'] for p in points])),
        'lng': round_down_geopoint(average([p['lng'] for p in points])),
        }
    def round_down_point(p):
        return {
            'lat': round_down_geopoint(p['lat']),
            'lng': round_down_geopoint(p['lng']),
            'instance': p['instance']
        }
    context.points = json.dumps([round_down_point(p) for p in list(points)])
    context.center = json.dumps(center)
    context.form_view = True
    return render_to_response('map.html', context_instance=context)


def survey_responses(request, pk):
    # todo: do a good job of displaying hierarchical data
    pi = ParsedInstance.objects.get(instance=pk)
    data = pi.to_dict()

    # get rid of keys with leading underscores
    data_for_display = {}
    for k, v in data.items():
        if not k.startswith(u"_"):
            data_for_display[k] = v

    xpaths = data_for_display.keys()
    xpaths.sort(cmp=pi.data_dictionary.get_xpath_cmp())
    label_value_pairs = [
         (parse_label_for_display(pi, xpath),
         data_for_display[xpath]) for xpath in xpaths
    ]
    languages = label_value_pairs[-1][0]
    
    return render_to_response('survey.html', {
            'label_value_pairs': label_value_pairs,
            'image_urls': image_urls(pi.instance),
            'languages': languages, 
            'default_language': languages[0][0]
            })


def csv_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    valid, dd = dd_for_params(id_string, owner, request)
    if not valid: return dd
    writer = DataDictionaryWriter(dd)
    file_path = writer.get_default_file_path()
    writer.write_to_file(file_path)
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('application/csv', id_string,
        extension='csv',
        file_path=file_path, use_local_filesystem=True)
    return response


def xls_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    # TODO: cleanup xls writer so this interface applies
    """
    valid, dd = dd_for_params(id_string, owner, request)
    if not valid: return dd
    """
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    ddw = XlsWriter()
    ddw.set_data_dictionary(dd)
    temp_file = ddw.save_workbook_to_file()
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('vnd.ms-excel', id_string,
        extension='xls')
    response.write(temp_file.getvalue())
    response['Content-Length'] = len(temp_file.getvalue())
    temp_file.close()
    return response


def zip_export(request, username, id_string):
    owner = User.objects.get(username=username)
    xform = XForm.objects.get(id_string=id_string, user=owner)
    if not has_permission(xform, owner, request):
        return HttpResponseForbidden('Not shared.')
    dd = DataDictionary.objects.get(id_string=id_string,
                                    user=owner)
    if request.GET.get('raw'):
        id_string = None
    response = response_with_mimetype_and_name('zip', id_string)
    # create zip_file
    tmp = tempfile.TemporaryFile()
    z = zipfile.ZipFile(tmp, 'w', zipfile.ZIP_DEFLATED)
    photos = image_urls_for_form(xform)
    for photo in photos:
        f = NamedTemporaryFile()
        req = urllib2.Request(photo)
        f.write(urllib2.urlopen(req).read())
        z.write(f.name, urlparse(photo).path[1:])
    z.close()
    wrapper = FileWrapper(tmp)
    response.content = wrapper
    response['Content-Length'] = tmp.tell()
    tmp.seek(0)
{'fraction_non_alphanumeric': 0.09903563367758703, 'fraction_numerical': 0.0017782641406196566, 'mean_word_length': 3.304386223138063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6675997', 'n_tokens_mistral': 4639, 'n_tokens_neox': 4571, 'n_words': 978}