<commit_before>import json
from urlparse import urlparse, urlunparse

from django import forms
from django.contrib import messages
from django.core.cache import cache
from django.core.urlresolvers import resolve, reverse
from django.db import connection
from django.db.models import Max
from django.http import QueryDict, HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, redirect
from django.template import RequestContext
from django.utils import simplejson
from django.views.decorators.cache import never_cache
from django.views.generic import View

from h1ds_summary import SUMMARY_TABLE_NAME
from h1ds_summary.forms import SummaryAttributeForm
from h1ds_summary.models import SummaryAttribute
from h1ds_summary.utils import parse_shot_str, parse_attr_str, parse_filter_str
from h1ds_summary.tasks import populate_summary_table_task, populate_attribute_task

DEFAULT_SHOT_REGEX = "last10"

class SummaryView(View):
    
    http_method_names = ['get']

    def get(self, request, shot_str="last10", attr_str="default", filter_str=None, table=SUMMARY_TABLE_NAME):

        # If there are no summary attributes, then tell the user
        if SummaryAttribute.objects.count() == 0:
            return render_to_response('h1ds_summary/no_attributes.html', {}, 
                                      context_instance=RequestContext(request))
        
        attribute_slugs = parse_attr_str(attr_str)
                                         
        show_attrs = request.GET.get('show_attr', None)
        hide_attrs = request.GET.get('hide_attr', None)

        if show_attrs or hide_attrs:
            if show_attrs:
                new_attrs = show_attrs.split('+')
                attribute_slugs.extend(new_attrs)
            if hide_attrs:
                for a in hide_attrs.split('+'):
                    try:
                        attribute_slugs.remove(a)
                    except ValueError:
                        pass
            if len(attribute_slugs) == 0:
                new_attr_str = "default"
            else:
                new_attr_str = '+'.join(attribute_slugs)
            if filter_str:
                return HttpResponseRedirect(reverse('sdfsummary', kwargs={'shot_str':shot_str, 'attr_str':new_attr_str, 'filter_str':filter_str}))
            else:
                return HttpResponseRedirect(reverse('sdsummary', kwargs={'shot_str':shot_str, 'attr_str':new_attr_str}))

        select_list = ['shot']
        select_list.extend(attribute_slugs)
        select_str = ','.join(select_list)

        excluded_attribute_slugs = SummaryAttribute.objects.exclude(slug__in=attribute_slugs).values_list('slug', flat=True)

        shot_where = parse_shot_str(shot_str)
        if shot_where == None:
            return render_to_response('h1ds_summary/no_shots.html', {}, context_instance=RequestContext(request))

        if filter_str == None:
            where = shot_where
        else:
            filter_where = parse_filter_str(filter_str)
            where = ' AND '.join([shot_where, filter_where])

        cursor = connection.cursor()
        cursor.execute("SELECT %(select)s FROM %(table)s WHERE %(where)s ORDER BY -shot" %{'table':table, 'select':select_str, 'where':where})
        data = cursor.fetchall()


        # Make a dict of format strings for summary attributes
        format_strings = {}
        for att in SummaryAttribute.objects.all():
            format_strings[att.slug] = att.format_string

        # This seems a bit messy, but  it's not clear to me how to refer
        # to a summary data value's  attribute slug name from within the
        # template, so let's attach it here...

        new_data = []
        data_headers = select_str.split(',')
        for d in data:
            new_row = []
            for j_i, j in enumerate(d):
                fstr = format_strings.get(data_headers[j_i], None)
                if fstr:
                    new_row.append((fstr %j, data_headers[j_i]))
                else:
                    new_row.append((j, data_headers[j_i]))
            new_data.append(new_row)

        return render_to_response('h1ds_summary/summary_table.html',
                                  {'data':new_data, 'data_headers':data_headers,
                                   'included_attrs':attribute_slugs,
                                   'excluded_attrs':excluded_attribute_slugs},
                                  context_instance=RequestContext(request))

class RecomputeSummaryView(View):
    """Recompute requested subset of summary database.

    Require a HTTP POST with two key-value pairs: 'return_path and either 'shot' or 'attribute'.
       return_path - URL to be redirected to after we submit the processing task to the job queue.
       shot - a single shot number, for which all attributes are recomputed.
       attribute - name (slug) of an attribute to be recomputed for all shots.

    If both shot and attribute are provided, the shot number will be processed and the attribute ignored.
    """

    http_method_names = ['post']
    
    def post(self, request, *args, **kwargs):
        return_path = request.POST.get("return_path")
        if request.POST.has_key("shot"):
            shot = [int(request.POST.get("shot")),]<commit_msg>don't fail attribute formatting on bad typecast<commit_after>import json
from urlparse import urlparse, urlunparse

from django import forms
from django.contrib import messages
from django.core.cache import cache
from django.core.urlresolvers import resolve, reverse
from django.db import connection
from django.db.models import Max
from django.http import QueryDict, HttpResponse, HttpResponseRedirect
from django.shortcuts import render_to_response, redirect
from django.template import RequestContext
from django.utils import simplejson
from django.views.decorators.cache import never_cache
from django.views.generic import View

from h1ds_summary import SUMMARY_TABLE_NAME
from h1ds_summary.forms import SummaryAttributeForm
from h1ds_summary.models import SummaryAttribute
from h1ds_summary.utils import parse_shot_str, parse_attr_str, parse_filter_str
from h1ds_summary.tasks import populate_summary_table_task, populate_attribute_task

DEFAULT_SHOT_REGEX = "last10"

class SummaryView(View):
    
    http_method_names = ['get']

    def get(self, request, shot_str="last10", attr_str="default", filter_str=None, table=SUMMARY_TABLE_NAME):

        # If there are no summary attributes, then tell the user
        if SummaryAttribute.objects.count() == 0:
            return render_to_response('h1ds_summary/no_attributes.html', {}, 
                                      context_instance=RequestContext(request))
        
        attribute_slugs = parse_attr_str(attr_str)
                                         
        show_attrs = request.GET.get('show_attr', None)
        hide_attrs = request.GET.get('hide_attr', None)

        if show_attrs or hide_attrs:
            if show_attrs:
                new_attrs = show_attrs.split('+')
                attribute_slugs.extend(new_attrs)
            if hide_attrs:
                for a in hide_attrs.split('+'):
                    try:
                        attribute_slugs.remove(a)
                    except ValueError:
                        pass
            if len(attribute_slugs) == 0:
                new_attr_str = "default"
            else:
                new_attr_str = '+'.join(attribute_slugs)
            if filter_str:
                return HttpResponseRedirect(reverse('sdfsummary', kwargs={'shot_str':shot_str, 'attr_str':new_attr_str, 'filter_str':filter_str}))
            else:
                return HttpResponseRedirect(reverse('sdsummary', kwargs={'shot_str':shot_str, 'attr_str':new_attr_str}))

        select_list = ['shot']
        select_list.extend(attribute_slugs)
        select_str = ','.join(select_list)

        excluded_attribute_slugs = SummaryAttribute.objects.exclude(slug__in=attribute_slugs).values_list('slug', flat=True)

        shot_where = parse_shot_str(shot_str)
        if shot_where == None:
            return render_to_response('h1ds_summary/no_shots.html', {}, context_instance=RequestContext(request))

        if filter_str == None:
            where = shot_where
        else:
            filter_where = parse_filter_str(filter_str)
            where = ' AND '.join([shot_where, filter_where])

        cursor = connection.cursor()
        cursor.execute("SELECT %(select)s FROM %(table)s WHERE %(where)s ORDER BY -shot" %{'table':table, 'select':select_str, 'where':where})
        data = cursor.fetchall()


        # Make a dict of format strings for summary attributes
        format_strings = {}
        for att in SummaryAttribute.objects.all():
            format_strings[att.slug] = att.format_string

        # This seems a bit messy, but  it's not clear to me how to refer
        # to a summary data value's  attribute slug name from within the
        # template, so let's attach it here...

        new_data = []
        data_headers = select_str.split(',')
        for d in data:
            new_row = []
            for j_i, j in enumerate(d):
                fstr = format_strings.get(data_headers[j_i], None)
                try:
                    val = fstr %j
                except TypeError:
                    val = j
                new_row.append((val, data_headers[j_i]))
            new_data.append(new_row)

        return render_to_response('h1ds_summary/summary_table.html',
                                  {'data':new_data, 'data_headers':data_headers,
                                   'included_attrs':attribute_slugs,
                                   'excluded_attrs':excluded_attribute_slugs},
                                  context_instance=RequestContext(request))

class RecomputeSummaryView(View):
    """Recompute requested subset of summary database.

    Require a HTTP POST with two key-value pairs: 'return_path and either 'shot' or 'attribute'.
       return_path - URL to be redirected to after we submit the processing task to the job queue.
       shot - a single shot number, for which all attributes are recomputed.
       attribute - name (slug) of an attribute to be recomputed for all shots.

    If both shot and attribute are provided, the shot number will be processed and the attribute ignored.
    """

    http_method_names = ['post']
    
    def post(self, request, *args, **kwargs):
        return_path = request.POST.get("return_path")
        if request.POST.has_key("shot"):
            shot = [int(request.POST.get("shot")),]
{'fraction_non_alphanumeric': 0.08448714326080814, 'fraction_numerical': 0.0026372798342281247, 'mean_word_length': 2.465404699738903, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5071415', 'n_tokens_mistral': 2923, 'n_tokens_neox': 2813, 'n_words': 753}