<commit_before>import json
import re

from django import http
from django.conf import settings
from django.db.models import Count
from django.shortcuts import redirect
from django.views.decorators.csrf import csrf_exempt

import jingo
import redisutils
from tower import ugettext as _

import amo.utils
from amo.decorators import post_required
from amo.utils import urlparams
from amo.urlresolvers import reverse
from addons.models import Addon
from versions.compare import version_int as vint
from .models import CompatReport, AppCompat
from .forms import CompatForm


def index(request, version=None):
    COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id]
    compat_dict = dict((v['main'], v) for v in COMPAT)
    if not COMPAT:
        raise http.Http404()
    if version not in compat_dict:
        return redirect('compat.index', COMPAT[0]['main'])
    qs = AppCompat.search()
    binary = None

    initial= {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
    initial.update(request.GET.items())
    form = CompatForm(initial)
    if request.GET and form.is_valid():
        if form.cleaned_data['appver']:
            app, ver = form.cleaned_data['appver'].split('-')
            if int(app) != request.APP.id or ver != version:
                new = reverse('compat.index', args=[ver], add_prefix=False)
                url = '/%s%s' % (amo.APP_IDS[int(app)].short, new)
                type_ = form.cleaned_data['type'] or None
                return redirect(urlparams(url, type=type_))

        if form.cleaned_data['type'] != 'all':
            binary = form.cleaned_data['type'] == 'binary'

    compat, app = compat_dict[version], str(request.APP.id)
    compat_queries = (
        ('prev', qs.query(top_95=True, **{
            'support.%s.max__gte' % app: vint(compat['previous'])})),
        ('top_95', qs.query(top_95=True)),
        ('all', qs),
    )
    compat_levels = [(key, version_compat(qs, compat, app, binary))
                     for key, qs in compat_queries]
    usage_addons, usage_total = usage_stats(request, compat, app, binary)
    return jingo.render(request, 'compat/index.html',
                        {'version': version,
                         'usage_addons': usage_addons,
                         'usage_total': usage_total,
                         'compat_levels': compat_levels,
                         'form': form})


def version_compat(qs, compat, app, binary):
    facets = []
    for v, prev in zip(compat['versions'], (None,) + compat['versions']):
        d = {'from': vint(v)}
        if prev:
            d['to'] = vint(prev)
        facets.append(d)
    # Pick up everything else for an Other count.
    facets.append({'to': vint(compat['versions'][-1])})
    facet = {'range': {'support.%s.max' % app: facets}}
    if binary is not None:
        facet['facet_filter'] = {'term': {'binary': binary}}
    qs = qs.facet(by_status=facet)
    result = qs[:0].raw()
    total_addons = result['hits']['total']
    ranges = result['facets']['by_status']['ranges']
    titles = compat['versions'] + (_('Other'),)
    faceted = [(v, r['count']) for v, r in zip(titles, ranges)]
    return total_addons, faceted


def usage_stats(request, compat, app, binary=None):
    # Get the list of add-ons for usage stats.
    redis = redisutils.connections['master']
    qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict()
          .filter(**{'support.%s.max__gte' % app: vint(compat['previous'])}))
    if binary is not None:
        qs = qs.filter(binary=True)
    addons = amo.utils.paginate(request, qs)
    for obj in addons.object_list:
        obj['usage'] = obj['usage'][app]
        obj['max_version'] = obj['max_version'][app]
    total = int(redis.hget('compat:%s' % app, 'total'))
    return addons, total


@csrf_exempt
@post_required
def incoming(request):
    # Turn camelCase into snake_case.
    snake_case = lambda s: re.sub('[A-Z]+', '_\g<0>', s).lower()
    try:
        data = [(snake_case(k), v)<commit_msg>filter by binary properly (bug 674992)<commit_after>import json
import re

from django import http
from django.conf import settings
from django.db.models import Count
from django.shortcuts import redirect
from django.views.decorators.csrf import csrf_exempt

import jingo
import redisutils
from tower import ugettext as _

import amo.utils
from amo.decorators import post_required
from amo.utils import urlparams
from amo.urlresolvers import reverse
from addons.models import Addon
from versions.compare import version_int as vint
from .models import CompatReport, AppCompat
from .forms import CompatForm


def index(request, version=None):
    COMPAT = [v for v in settings.COMPAT if v['app'] == request.APP.id]
    compat_dict = dict((v['main'], v) for v in COMPAT)
    if not COMPAT:
        raise http.Http404()
    if version not in compat_dict:
        return redirect('compat.index', COMPAT[0]['main'])
    qs = AppCompat.search()
    binary = None

    initial= {'appver': '%s-%s' % (request.APP.id, version), 'type': 'all'}
    initial.update(request.GET.items())
    form = CompatForm(initial)
    if request.GET and form.is_valid():
        if form.cleaned_data['appver']:
            app, ver = form.cleaned_data['appver'].split('-')
            if int(app) != request.APP.id or ver != version:
                new = reverse('compat.index', args=[ver], add_prefix=False)
                url = '/%s%s' % (amo.APP_IDS[int(app)].short, new)
                type_ = form.cleaned_data['type'] or None
                return redirect(urlparams(url, type=type_))

        if form.cleaned_data['type'] != 'all':
            binary = form.cleaned_data['type'] == 'binary'

    compat, app = compat_dict[version], str(request.APP.id)
    compat_queries = (
        ('prev', qs.query(top_95=True, **{
            'support.%s.max__gte' % app: vint(compat['previous'])})),
        ('top_95', qs.query(top_95=True)),
        ('all', qs),
    )
    compat_levels = [(key, version_compat(qs, compat, app, binary))
                     for key, qs in compat_queries]
    usage_addons, usage_total = usage_stats(request, compat, app, binary)
    return jingo.render(request, 'compat/index.html',
                        {'version': version,
                         'usage_addons': usage_addons,
                         'usage_total': usage_total,
                         'compat_levels': compat_levels,
                         'form': form})


def version_compat(qs, compat, app, binary):
    facets = []
    for v, prev in zip(compat['versions'], (None,) + compat['versions']):
        d = {'from': vint(v)}
        if prev:
            d['to'] = vint(prev)
        facets.append(d)
    # Pick up everything else for an Other count.
    facets.append({'to': vint(compat['versions'][-1])})
    facet = {'range': {'support.%s.max' % app: facets}}
    if binary is not None:
        facet['facet_filter'] = {'term': {'binary': binary}}
    qs = qs.facet(by_status=facet)
    result = qs[:0].raw()
    total_addons = result['hits']['total']
    ranges = result['facets']['by_status']['ranges']
    titles = compat['versions'] + (_('Other'),)
    faceted = [(v, r['count']) for v, r in zip(titles, ranges)]
    return total_addons, faceted


def usage_stats(request, compat, app, binary=None):
    # Get the list of add-ons for usage stats.
    redis = redisutils.connections['master']
    qs = (AppCompat.search().order_by('-usage.%s' % app).values_dict()
          .filter(**{'support.%s.max__gte' % app: vint(compat['previous'])}))
    if binary is not None:
        qs = qs.filter(binary=binary)
    addons = amo.utils.paginate(request, qs)
    for obj in addons.object_list:
        obj['usage'] = obj['usage'][app]
        obj['max_version'] = obj['max_version'][app]
    total = int(redis.hget('compat:%s' % app, 'total'))
    return addons, total


@csrf_exempt
@post_required
def incoming(request):
    # Turn camelCase into snake_case.
    snake_case = lambda s: re.sub('[A-Z]+', '_\g<0>', s).lower()
    try:
        data = [(snake_case(k), v)
{'fraction_non_alphanumeric': 0.14822158879662908, 'fraction_numerical': 0.0039657950179700086, 'mean_word_length': 2.898550724637681, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '469758', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2561, 'n_words': 652}