<commit_before>import json
import datetime
import calendar

import dateutil.parser

from django.db.models import Sum, Count, Min, Max, Q
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required, user_passes_test
from locksmith.hub.models import Api, Key, Report, resolve_model
from locksmith.hub.common import cycle_generator, exclude_internal_keys, exclude_internal_key_reports
from unusual.http import BadRequest

from django.conf import settings

staff_required = user_passes_test(lambda u: u.is_staff)

def _keys_issued_date_range():
    return Key.objects.aggregate(earliest=Min('issued_on'), latest=Max('issued_on'))

def _years():
    extents = _keys_issued_date_range()
    return range(extents['earliest'].year, extents['latest'].year + 1)

def parse_bool(p):
    return unicode(p).lower() in ['y', 't', 'yes', 'true']

def request_param_type_guard(request, param, parse_func, default=None):
    untyped = request.GET.get(param) or request.POST.get(param)
    if untyped is None:
        return default
    try:
        typed = parse_func(untyped)
        return typed
    except (SyntaxError, ValueError):
        raise BadRequest(content='Unparsable {0} value: {1}'.format(param, untyped))

def parse_date_param(request, param, default=None):
    return request_param_type_guard(request, param, dateutil.parser.parse, default)

def parse_bool_param(request, param, default=None):
    return request_param_type_guard(request, param, parse_bool, default)

def parse_int_param(request, param, default=None):
    return request_param_type_guard(request, param, int, default)

@login_required
def apis_list(request):
    apis = Api.objects.all()
    result = [{'id': api.id, 'name': api.name, 'deprecated': not api.push_enabled}
              for api in apis]
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')


@login_required
def calls_to_api(request,
                 api_id=None, api_name=None):

    begin_date = parse_date_param(request, 'begin_date')
    end_date = parse_date_param(request, 'end_date')
    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    qry = Report.objects.filter(api=api) 
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    if begin_date:
        qry = qry.filter(date__gte=begin_date)
    if end_date:
        qry = qry.filter(date__lte=end_date)
    qry = qry.aggregate(calls=Sum('calls'))

    result = {
        'api_id': api.id,
        'api_name': api.name,
        'calls': qry['calls']
    }
    if begin_date is not None:
        result['begin_date'] = begin_date.isoformat()
    if end_date is not None:
        result['end_date'] = end_date.isoformat()
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@login_required
def calls_to_api_yearly(request,
                        api_id=None, api_name=None):
    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    date_extents = _keys_issued_date_range()
    earliest_year = date_extents['earliest'].year
    latest_year = date_extents['latest'].year

    qry = Report.objects.filter(api=api)
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    agg = qry.aggregate(calls=Sum('calls'))
    qry = qry.extra(select={'year': 'extract(year from date)::int'})
    yearly_aggs = qry.values('year').annotate(calls=Sum('calls'))

    yearly = dict([(yr_agg['year'], yr_agg) for yr_agg in yearly_aggs])
    for year in range(earliest_year, latest_year + 1):
        yearly[year] = yearly.get(year,
                                  {'year': year, 'calls': 0})
    result = {
        'api_id': api.id,
        'api_name': api.name,
        'earliest_year': earliest_year,
        'latest_year': latest_year,
        'calls': agg['calls'],
        'yearly': yearly.values()
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@login_required
def calls_to_api_monthly(request, year,
                         api_id=None, api_name=None):
    year = int(year)

    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    qry = Report.objects.filter(api=api)
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    qry = qry.filter(date__gte=datetime.date(year, 1, 1),
                     date__lte=datetime.date(year, 12, 31))
    agg = qry.aggregate(calls=Sum('calls'))

    qry = qry.extra(select={'month': 'extract(month from date)::int'})
    monthly_aggs = qry.values('month').annotate(calls=Sum('calls'))

    monthly = dict(((m_agg['month'], m_agg) for m_agg in monthly_aggs))
    for month in range(1, 13):
        monthly[month] = monthly.get(month,
                                     {'month': month, 'calls': 0})
    result = {
        'api_id': api.id,
        'api_name': api.name,
        'calls': agg['calls'],
        'monthly': monthly.values()
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@staff_required
def keys(request):
    """Lists API keys. Compatible with jQuery DataTables."""
    iDisplayStart = parse_int_param(request, 'iDisplayStart')
    iDisplayLength = parse_int_param(request, 'iDisplayLength')
    sEcho = parse_int_param(request, 'sEcho')
    iSortCol_0 = parse_int_param(request, 'iSortCol_0')
    sSortDir_0 = request.GET.get('sSortDir_0', 'asc')
    sSearch = request.GET.get('sSearch')

    columns = ['key', 'email', 'calls', 'latest_call', 'issued_on']
    qry = Key.objects
    if sSearch not in (None, ''):
        qry = qry.filter(Q(key__icontains=sSearch) | Q(email__icontains=sSearch))
    qry = qry.values('key', 'email', 'issued_on').annotate(calls=Sum('reports__calls'),
                                                           latest_call=Max('reports__date'))
    qry = qry.filter(calls__isnull=False)
    qry = exclude_internal_keys(qry)
    # TODO: Add multi-column sorting
    if iSortCol_0 not in (None, ''):
        sort_col_field = columns[iSortCol_0]
        sort_spec = '{dir}{col}'.format(dir='-' if sSortDir_0 == 'desc' else '',
                                        col=sort_col_field)
        qry = qry.order_by(sort_spec)

    result = {
        'iTotalRecord': Key.objects.count(),
        'iTotalDisplayRecords': qry.count(),
        'sEcho': sEcho,
        'aaData': [[k['key'],
                    '<a href="{0}">{1}</a>'.format(reverse('key_analytics', args=(k['key'], )), k['email']),
                    k['calls'],
                    k['latest_call'].isoformat(),
                    k['issued_on'].date().isoformat()]
                   for k in qry[iDisplayStart:iDisplayStart+iDisplayLength]]
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')
<commit_msg>Fix typo for dataTables field.<commit_after>import json
import datetime
import calendar

import dateutil.parser

from django.db.models import Sum, Count, Min, Max, Q
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required, user_passes_test
from locksmith.hub.models import Api, Key, Report, resolve_model
from locksmith.hub.common import cycle_generator, exclude_internal_keys, exclude_internal_key_reports
from unusual.http import BadRequest

from django.conf import settings

staff_required = user_passes_test(lambda u: u.is_staff)

def _keys_issued_date_range():
    return Key.objects.aggregate(earliest=Min('issued_on'), latest=Max('issued_on'))

def _years():
    extents = _keys_issued_date_range()
    return range(extents['earliest'].year, extents['latest'].year + 1)

def parse_bool(p):
    return unicode(p).lower() in ['y', 't', 'yes', 'true']

def request_param_type_guard(request, param, parse_func, default=None):
    untyped = request.GET.get(param) or request.POST.get(param)
    if untyped is None:
        return default
    try:
        typed = parse_func(untyped)
        return typed
    except (SyntaxError, ValueError):
        raise BadRequest(content='Unparsable {0} value: {1}'.format(param, untyped))

def parse_date_param(request, param, default=None):
    return request_param_type_guard(request, param, dateutil.parser.parse, default)

def parse_bool_param(request, param, default=None):
    return request_param_type_guard(request, param, parse_bool, default)

def parse_int_param(request, param, default=None):
    return request_param_type_guard(request, param, int, default)

@login_required
def apis_list(request):
    apis = Api.objects.all()
    result = [{'id': api.id, 'name': api.name, 'deprecated': not api.push_enabled}
              for api in apis]
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')


@login_required
def calls_to_api(request,
                 api_id=None, api_name=None):

    begin_date = parse_date_param(request, 'begin_date')
    end_date = parse_date_param(request, 'end_date')
    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    qry = Report.objects.filter(api=api) 
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    if begin_date:
        qry = qry.filter(date__gte=begin_date)
    if end_date:
        qry = qry.filter(date__lte=end_date)
    qry = qry.aggregate(calls=Sum('calls'))

    result = {
        'api_id': api.id,
        'api_name': api.name,
        'calls': qry['calls']
    }
    if begin_date is not None:
        result['begin_date'] = begin_date.isoformat()
    if end_date is not None:
        result['end_date'] = end_date.isoformat()
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@login_required
def calls_to_api_yearly(request,
                        api_id=None, api_name=None):
    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    date_extents = _keys_issued_date_range()
    earliest_year = date_extents['earliest'].year
    latest_year = date_extents['latest'].year

    qry = Report.objects.filter(api=api)
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    agg = qry.aggregate(calls=Sum('calls'))
    qry = qry.extra(select={'year': 'extract(year from date)::int'})
    yearly_aggs = qry.values('year').annotate(calls=Sum('calls'))

    yearly = dict([(yr_agg['year'], yr_agg) for yr_agg in yearly_aggs])
    for year in range(earliest_year, latest_year + 1):
        yearly[year] = yearly.get(year,
                                  {'year': year, 'calls': 0})
    result = {
        'api_id': api.id,
        'api_name': api.name,
        'earliest_year': earliest_year,
        'latest_year': latest_year,
        'calls': agg['calls'],
        'yearly': yearly.values()
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@login_required
def calls_to_api_monthly(request, year,
                         api_id=None, api_name=None):
    year = int(year)

    if api_id is None and api_name is None:
        return HttpResponseBadRequest('Must specify API id or name.')

    try:
        api = resolve_model(Api, [('id', api_id), ('name', api_name)])
    except Api.DoesNotExist:
        return HttpResponseNotFound('The requested API was not found.')

    ignore_internal_keys = parse_bool_param(request, 'ignore_internal_keys', True)

    qry = Report.objects.filter(api=api)
    if ignore_internal_keys:
        qry = exclude_internal_key_reports(qry)
    qry = qry.filter(date__gte=datetime.date(year, 1, 1),
                     date__lte=datetime.date(year, 12, 31))
    agg = qry.aggregate(calls=Sum('calls'))

    qry = qry.extra(select={'month': 'extract(month from date)::int'})
    monthly_aggs = qry.values('month').annotate(calls=Sum('calls'))

    monthly = dict(((m_agg['month'], m_agg) for m_agg in monthly_aggs))
    for month in range(1, 13):
        monthly[month] = monthly.get(month,
                                     {'month': month, 'calls': 0})
    result = {
        'api_id': api.id,
        'api_name': api.name,
        'calls': agg['calls'],
        'monthly': monthly.values()
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

@staff_required
def keys(request):
    """Lists API keys. Compatible with jQuery DataTables."""
    iDisplayStart = parse_int_param(request, 'iDisplayStart')
    iDisplayLength = parse_int_param(request, 'iDisplayLength')
    sEcho = parse_int_param(request, 'sEcho')
    iSortCol_0 = parse_int_param(request, 'iSortCol_0')
    sSortDir_0 = request.GET.get('sSortDir_0', 'asc')
    sSearch = request.GET.get('sSearch')

    columns = ['key', 'email', 'calls', 'latest_call', 'issued_on']
    qry = Key.objects
    if sSearch not in (None, ''):
        qry = qry.filter(Q(key__icontains=sSearch) | Q(email__icontains=sSearch))
    qry = qry.values('key', 'email', 'issued_on').annotate(calls=Sum('reports__calls'),
                                                           latest_call=Max('reports__date'))
    qry = qry.filter(calls__isnull=False)
    qry = exclude_internal_keys(qry)
    # TODO: Add multi-column sorting
    if iSortCol_0 not in (None, ''):
        sort_col_field = columns[iSortCol_0]
        sort_spec = '{dir}{col}'.format(dir='-' if sSortDir_0 == 'desc' else '',
                                        col=sort_col_field)
        qry = qry.order_by(sort_spec)

    result = {
        'iTotalRecords': Key.objects.count(),
        'iTotalDisplayRecords': qry.count(),
        'sEcho': sEcho,
        'aaData': [[k['key'],
                    '<a href="{0}">{1}</a>'.format(reverse('key_analytics', args=(k['key'], )), k['email']),
                    k['calls'],
                    k['latest_call'].isoformat(),
                    k['issued_on'].date().isoformat()]
                   for k in qry[iDisplayStart:iDisplayStart+iDisplayLength]]
    }
    return HttpResponse(content=json.dumps(result), status=200, content_type='application/json')

{'fraction_non_alphanumeric': 0.12677574835109082, 'fraction_numerical': 0.004946727549467275, 'mean_word_length': 3.3973786949247073, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3596111', 'n_tokens_mistral': 5127, 'n_tokens_neox': 5034, 'n_words': 1066}