<commit_before>import json
import re
import datetime

from django import http
from django.shortcuts import render
from django.contrib.auth.models import Permission
from django.contrib.sites.requests import RequestSite
from django.core.urlresolvers import reverse
from django.conf import settings
from django import forms
from django.views.decorators.csrf import csrf_exempt
# explicit import because django.forms has an __all__
from django.forms.forms import DeclarativeFieldsMetaclass

from ratelimit.decorators import ratelimit

from socorro.lib import BadArgumentError, MissingArgumentError
from socorro.external.crashstorage_base import CrashIDNotFound

import crashstats
from crashstats.crashstats.decorators import track_api_pageview
from crashstats.crashstats import models
from crashstats.crashstats import utils
from crashstats.tokens.models import Token
from .cleaner import Cleaner


# List of all modules that contain models we want to expose.
MODELS_MODULES = (
    models,
    crashstats.tools.models,
    crashstats.supersearch.models,
    crashstats.symbols.models,
)


BAD_REQUEST_EXCEPTIONS = (
    BadArgumentError,
    MissingArgumentError,
    models.RequiredParameterError,
)

NOT_FOUND_EXCEPTIONS = (
    CrashIDNotFound,
)


# See http://www.iana.org/assignments/http-status-codes
REASON_PHRASES = {
    100: 'CONTINUE',
    101: 'SWITCHING PROTOCOLS',
    102: 'PROCESSING',
    200: 'OK',
    201: 'CREATED',
    202: 'ACCEPTED',
    203: 'NON-AUTHORITATIVE INFORMATION',
    204: 'NO CONTENT',
    205: 'RESET CONTENT',
    206: 'PARTIAL CONTENT',
    207: 'MULTI-STATUS',
    208: 'ALREADY REPORTED',
    226: 'IM USED',
    300: 'MULTIPLE CHOICES',
    301: 'MOVED PERMANENTLY',
    302: 'FOUND',
    303: 'SEE OTHER',
    304: 'NOT MODIFIED',
    305: 'USE PROXY',
    306: 'RESERVED',
    307: 'TEMPORARY REDIRECT',
    400: 'BAD REQUEST',
    401: 'UNAUTHORIZED',
    402: 'PAYMENT REQUIRED',
    403: 'FORBIDDEN',
    404: 'NOT FOUND',
    405: 'METHOD NOT ALLOWED',
    406: 'NOT ACCEPTABLE',
    407: 'PROXY AUTHENTICATION REQUIRED',
    408: 'REQUEST TIMEOUT',
    409: 'CONFLICT',
    410: 'GONE',
    411: 'LENGTH REQUIRED',
    412: 'PRECONDITION FAILED',
    413: 'REQUEST ENTITY TOO LARGE',
    414: 'REQUEST-URI TOO LONG',
    415: 'UNSUPPORTED MEDIA TYPE',
    416: 'REQUESTED RANGE NOT SATISFIABLE',
    417: 'EXPECTATION FAILED',
    418: "I'M A TEAPOT",
    422: 'UNPROCESSABLE ENTITY',
    423: 'LOCKED',
    424: 'FAILED DEPENDENCY',
    426: 'UPGRADE REQUIRED',
    428: 'PRECONDITION REQUIRED',
    429: 'TOO MANY REQUESTS',
    431: 'REQUEST HEADER FIELDS TOO LARGE',
    500: 'INTERNAL SERVER ERROR',
    501: 'NOT IMPLEMENTED',
    502: 'BAD GATEWAY',
    503: 'SERVICE UNAVAILABLE',
    504: 'GATEWAY TIMEOUT',
    505: 'HTTP VERSION NOT SUPPORTED',
    506: 'VARIANT ALSO NEGOTIATES',
    507: 'INSUFFICIENT STORAGE',
    508: 'LOOP DETECTED',
    510: 'NOT EXTENDED',
    511: 'NETWORK AUTHENTICATION REQUIRED',
}


class APIWhitelistError(Exception):
    pass


class MultipleStringField(forms.TypedMultipleChoiceField):
    """Field that do not validate if the field values are in self.choices"""

    def validate(self, value):
        """Nothing to do here"""
        if self.required and not value:
            raise forms.ValidationError(self.error_messages['required'])


TYPE_MAP = {
    basestring: forms.CharField,
    list: MultipleStringField,
    datetime.date: forms.DateField,
    datetime.datetime: forms.DateTimeField,
    int: forms.IntegerField,
    bool: forms.BooleanField,
}


def fancy_init(self, model, *args, **kwargs):
    self.model = model
    self.__old_init__(*args, **kwargs)
    for parameter in model().get_annotated_params():
        required = parameter['required']
        name = parameter['name']

        if parameter['type'] not in TYPE_MAP:
            raise NotImplementedError(parameter['type'])
        field_class = TYPE_MAP[parameter['type']]
        self.fields[name] = field_class(required=required)


class FormWrapperMeta(DeclarativeFieldsMetaclass):
    def __new__(cls, name, bases, attrs):
        attrs['__old_init__'] = bases[0].__init__
        attrs['__init__'] = fancy_init
        return super(FormWrapperMeta, cls).__new__(cls, name, bases, attrs)


class FormWrapper(forms.Form):
    __metaclass__ = FormWrapperMeta

    def clean(self):
        cleaned_data = super(FormWrapper, self).clean()

        for field in self.fields:
            # Because the context for all of this is the API,
            # and we're using django forms there's a mismatch to how
            # boolean fields should be handled.
            # Django forms are meant for HTML forms. A key principle
            # functionality of a HTML form and a checkbox is that
            # if the user choses to NOT check a checkbox, the browser
            # will not send `mybool=false` or `mybool=''`. It will simply
            # not send anything and then the server has to assume the user
            # chose to NOT check it because it was offerend.
            # On a web API, however, the user doesn't use checkboxes.
            # He uses `?mybool=truthy` or `&mybool=falsy`.
            # Therefore, for our boolean fields, if the value is not
            # present at all, we have to assume it to be None.
            # That makes it possible to actually set `mybool=false`
            if isinstance(self.fields[field], forms.BooleanField):
                if field not in self.data:
                    self.cleaned_data[field] = None
        return cleaned_data


# Names of models we don't want to serve at all
BLACKLIST = (
    # not because it's sensitive but because it's only used for writes
    'ReleasesFeatured',
    # only used for doing posts
    'Releases',
    # because it's only used for the admin
    'Field',
    'SuperSearchField',
    'SuperSearchMissingFields',
    # because it's very sensitive and we don't want to expose it
    'Query',
    # because it's an internal thing only
    'GraphicsReport',
    'Healthcheck',
)

<commit_msg>fixes bug 1324840 - Remove PriorityJob from public API (#3640)<commit_after>import json
import re
import datetime

from django import http
from django.shortcuts import render
from django.contrib.auth.models import Permission
from django.contrib.sites.requests import RequestSite
from django.core.urlresolvers import reverse
from django.conf import settings
from django import forms
from django.views.decorators.csrf import csrf_exempt
# explicit import because django.forms has an __all__
from django.forms.forms import DeclarativeFieldsMetaclass

from ratelimit.decorators import ratelimit

from socorro.lib import BadArgumentError, MissingArgumentError
from socorro.external.crashstorage_base import CrashIDNotFound

import crashstats
from crashstats.crashstats.decorators import track_api_pageview
from crashstats.crashstats import models
from crashstats.crashstats import utils
from crashstats.tokens.models import Token
from .cleaner import Cleaner


# List of all modules that contain models we want to expose.
MODELS_MODULES = (
    models,
    crashstats.tools.models,
    crashstats.supersearch.models,
    crashstats.symbols.models,
)


BAD_REQUEST_EXCEPTIONS = (
    BadArgumentError,
    MissingArgumentError,
    models.RequiredParameterError,
)

NOT_FOUND_EXCEPTIONS = (
    CrashIDNotFound,
)


# See http://www.iana.org/assignments/http-status-codes
REASON_PHRASES = {
    100: 'CONTINUE',
    101: 'SWITCHING PROTOCOLS',
    102: 'PROCESSING',
    200: 'OK',
    201: 'CREATED',
    202: 'ACCEPTED',
    203: 'NON-AUTHORITATIVE INFORMATION',
    204: 'NO CONTENT',
    205: 'RESET CONTENT',
    206: 'PARTIAL CONTENT',
    207: 'MULTI-STATUS',
    208: 'ALREADY REPORTED',
    226: 'IM USED',
    300: 'MULTIPLE CHOICES',
    301: 'MOVED PERMANENTLY',
    302: 'FOUND',
    303: 'SEE OTHER',
    304: 'NOT MODIFIED',
    305: 'USE PROXY',
    306: 'RESERVED',
    307: 'TEMPORARY REDIRECT',
    400: 'BAD REQUEST',
    401: 'UNAUTHORIZED',
    402: 'PAYMENT REQUIRED',
    403: 'FORBIDDEN',
    404: 'NOT FOUND',
    405: 'METHOD NOT ALLOWED',
    406: 'NOT ACCEPTABLE',
    407: 'PROXY AUTHENTICATION REQUIRED',
    408: 'REQUEST TIMEOUT',
    409: 'CONFLICT',
    410: 'GONE',
    411: 'LENGTH REQUIRED',
    412: 'PRECONDITION FAILED',
    413: 'REQUEST ENTITY TOO LARGE',
    414: 'REQUEST-URI TOO LONG',
    415: 'UNSUPPORTED MEDIA TYPE',
    416: 'REQUESTED RANGE NOT SATISFIABLE',
    417: 'EXPECTATION FAILED',
    418: "I'M A TEAPOT",
    422: 'UNPROCESSABLE ENTITY',
    423: 'LOCKED',
    424: 'FAILED DEPENDENCY',
    426: 'UPGRADE REQUIRED',
    428: 'PRECONDITION REQUIRED',
    429: 'TOO MANY REQUESTS',
    431: 'REQUEST HEADER FIELDS TOO LARGE',
    500: 'INTERNAL SERVER ERROR',
    501: 'NOT IMPLEMENTED',
    502: 'BAD GATEWAY',
    503: 'SERVICE UNAVAILABLE',
    504: 'GATEWAY TIMEOUT',
    505: 'HTTP VERSION NOT SUPPORTED',
    506: 'VARIANT ALSO NEGOTIATES',
    507: 'INSUFFICIENT STORAGE',
    508: 'LOOP DETECTED',
    510: 'NOT EXTENDED',
    511: 'NETWORK AUTHENTICATION REQUIRED',
}


class APIWhitelistError(Exception):
    pass


class MultipleStringField(forms.TypedMultipleChoiceField):
    """Field that do not validate if the field values are in self.choices"""

    def validate(self, value):
        """Nothing to do here"""
        if self.required and not value:
            raise forms.ValidationError(self.error_messages['required'])


TYPE_MAP = {
    basestring: forms.CharField,
    list: MultipleStringField,
    datetime.date: forms.DateField,
    datetime.datetime: forms.DateTimeField,
    int: forms.IntegerField,
    bool: forms.BooleanField,
}


def fancy_init(self, model, *args, **kwargs):
    self.model = model
    self.__old_init__(*args, **kwargs)
    for parameter in model().get_annotated_params():
        required = parameter['required']
        name = parameter['name']

        if parameter['type'] not in TYPE_MAP:
            raise NotImplementedError(parameter['type'])
        field_class = TYPE_MAP[parameter['type']]
        self.fields[name] = field_class(required=required)


class FormWrapperMeta(DeclarativeFieldsMetaclass):
    def __new__(cls, name, bases, attrs):
        attrs['__old_init__'] = bases[0].__init__
        attrs['__init__'] = fancy_init
        return super(FormWrapperMeta, cls).__new__(cls, name, bases, attrs)


class FormWrapper(forms.Form):
    __metaclass__ = FormWrapperMeta

    def clean(self):
        cleaned_data = super(FormWrapper, self).clean()

        for field in self.fields:
            # Because the context for all of this is the API,
            # and we're using django forms there's a mismatch to how
            # boolean fields should be handled.
            # Django forms are meant for HTML forms. A key principle
            # functionality of a HTML form and a checkbox is that
            # if the user choses to NOT check a checkbox, the browser
            # will not send `mybool=false` or `mybool=''`. It will simply
            # not send anything and then the server has to assume the user
            # chose to NOT check it because it was offerend.
            # On a web API, however, the user doesn't use checkboxes.
            # He uses `?mybool=truthy` or `&mybool=falsy`.
            # Therefore, for our boolean fields, if the value is not
            # present at all, we have to assume it to be None.
            # That makes it possible to actually set `mybool=false`
            if isinstance(self.fields[field], forms.BooleanField):
                if field not in self.data:
                    self.cleaned_data[field] = None
        return cleaned_data


# Names of models we don't want to serve at all
BLACKLIST = (
    # not because it's sensitive but because it's only used for writes
    'ReleasesFeatured',
    # only used for doing posts
    'Releases',
    # because it's only used for the admin
    'Field',
    'SuperSearchField',
    'SuperSearchMissingFields',
    # because it's very sensitive and we don't want to expose it
    'Query',
    # because it's an internal thing only
    'GraphicsReport',
    'Healthcheck',
    'Priorityjob',
)


{'fraction_non_alphanumeric': 0.09487432232626909, 'fraction_numerical': 0.029653359618859867, 'mean_word_length': 3.1187415426251692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673065', 'n_tokens_mistral': 4054, 'n_tokens_neox': 3720, 'n_words': 1191}