<commit_before>import functools
from django.core.urlresolvers import reverse
from preserialize.serialize import serialize
from restlib2.params import Parametizer, param_cleaners
from avocado.models import DataField
from avocado.events import usage
from ..base import DataResource
from .. import templates

can_change_field = lambda u: u.has_perm('avocado.change_datafield')
stats_capable = lambda x: not x.searchable and not x.internal_type == 'auto'


def field_posthook(instance, data, request):
    """Field serialization post-hook for augmenting per-instance data.

    The only two arguments the post-hook takes is instance and data. The
    remaining arguments must be partially applied using `functools.partial`
    during the request/response cycle.
    """

    uri = request.build_absolute_uri

    # Augment the links
    data['_links'] = {
        'self': {
            'href': uri(reverse('serrano:field', args=[instance.pk])),
        },
        'values': {
            'href': uri(reverse('serrano:field-values', args=[instance.pk])),
        },
    }

    if stats_capable(instance):
        data['_links']['stats'] = {
            'href': uri(reverse('serrano:field-stats', args=[instance.pk])),
        }
        data['_links']['distribution'] = {
            'href': uri(reverse('serrano:field-distribution', args=[instance.pk])),
        }

    return data


class FieldParametizer(Parametizer):
    "Supported params and their defaults for Field endpoints."

    sort = None
    order = 'asc'
    published = None
    archived = None
    brief = False
    query = ''
    limit = None

    # Not implemented
    offset = None
    page = None

    def clean_published(self, value):
        return param_cleaners.clean_bool(value)

    def clean_archived(self, value):
        return param_cleaners.clean_bool(value)

    def clean_brief(self, value):
        return param_cleaners.clean_bool(value)

    def clean_query(self, value):
        return param_cleaners.clean_string(value)

    def clean_limit(self, value):
        return param_cleaners.clean_int(value)

    def clean_offset(self, value):
        return param_cleaners.clean_int(value)

    def clean_page(self, value):
        return param_cleaners.clean_int(value)


class FieldBase(DataResource):
    model = DataField

    parametizer = FieldParametizer

    template = templates.Field

    def get_queryset(self, request):
        queryset = self.model.objects.all()
        if not can_change_field(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except self.model.DoesNotExist:
            pass

    def prepare(self, request, instance, template=None, brief=False, **params):
        if template is None:
            template = templates.BriefField if brief else self.template

        posthook = functools.partial(field_posthook, request=request)
        return serialize(instance, posthook=posthook, **template)

    def is_not_found(self, request, response, pk, *args, **kwargs):
        instance = self.get_object(request, pk=pk)
        if instance is None:
            return True
        request.instance = instance
        return False


class FieldResource(FieldBase):
    "Resource for interacting with Field instances."

    def get(self, request, pk):
        instance = request.instance
        usage.log('read', instance=instance, request=request)
        return self.prepare(request, instance)


class FieldsResource(FieldResource):
    "Field Collection Resource"

    def is_not_found(self, request, response, *args, **kwargs):
        return False

    def get(self, request):
        params = self.get_params(request)
        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied, otherwise
        # only allow for published objects.
        if can_change_field(request.user):
            filters = {}

            if params['published'] is not None:
                filters['published'] = params['published']

            if params['archived'] is not None:
                filters['archived'] = params['archived']

            if filters:
                queryset = queryset.filter(**filters)
        else:
            queryset = queryset.published()

        # If Haystack is installed, perform the search
        if params['query'] and OPTIONAL_DEPS['haystack']:
            usage.log('search', model=self.model, request=request, data={
                'query': params['query'],
            })
            results = self.model.objects.search(params['query'],
                queryset=queryset, max_results=params['limit'],
                partial=True)
            objects = (x.object for x in results)
        else:
            if params['sort'] == 'name':
                order = '-name' if params['order'] == 'desc' else 'name'
                queryset = queryset.order_by(order)

            if params['limit']:
                queryset = queryset[:params['limit']]

            objects = queryset

        return self.prepare(request, objects, **params)<commit_msg>Filter/log orphans on /api/fields/ and raise error on /api/fields/#/<commit_after>import functools
import logging
from django.http import HttpResponse
from django.core.urlresolvers import reverse
from preserialize.serialize import serialize
from restlib2.http import codes
from restlib2.params import Parametizer, param_cleaners
from avocado.models import DataField
from avocado.events import usage
from ..base import DataResource
from .. import templates

can_change_field = lambda u: u.has_perm('avocado.change_datafield')
stats_capable = lambda x: not x.searchable and not x.internal_type == 'auto'
log = logging.getLogger(__name__)

def is_field_orphaned(instance):
    return instance.model is None or instance.field is None

def field_posthook(instance, data, request):
    """Field serialization post-hook for augmenting per-instance data.

    The only two arguments the post-hook takes is instance and data. The
    remaining arguments must be partially applied using `functools.partial`
    during the request/response cycle.
    """

    uri = request.build_absolute_uri

    # Augment the links
    data['_links'] = {
        'self': {
            'href': uri(reverse('serrano:field', args=[instance.pk])),
        },
        'values': {
            'href': uri(reverse('serrano:field-values', args=[instance.pk])),
        },
    }

    if stats_capable(instance):
        data['_links']['stats'] = {
            'href': uri(reverse('serrano:field-stats', args=[instance.pk])),
        }
        data['_links']['distribution'] = {
            'href': uri(reverse('serrano:field-distribution', args=[instance.pk])),
        }

    return data


class FieldParametizer(Parametizer):
    "Supported params and their defaults for Field endpoints."

    sort = None
    order = 'asc'
    published = None
    archived = None
    brief = False
    query = ''
    limit = None

    # Not implemented
    offset = None
    page = None

    def clean_published(self, value):
        return param_cleaners.clean_bool(value)

    def clean_archived(self, value):
        return param_cleaners.clean_bool(value)

    def clean_brief(self, value):
        return param_cleaners.clean_bool(value)

    def clean_query(self, value):
        return param_cleaners.clean_string(value)

    def clean_limit(self, value):
        return param_cleaners.clean_int(value)

    def clean_offset(self, value):
        return param_cleaners.clean_int(value)

    def clean_page(self, value):
        return param_cleaners.clean_int(value)


class FieldBase(DataResource):
    model = DataField

    parametizer = FieldParametizer

    template = templates.Field

    def get_queryset(self, request):
        queryset = self.model.objects.all()
        if not can_change_field(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except self.model.DoesNotExist:
            pass

    def prepare(self, request, instance, template=None, brief=False, **params):
        if template is None:
            template = templates.BriefField if brief else self.template

        posthook = functools.partial(field_posthook, request=request)
        return serialize(instance, posthook=posthook, **template)

    def is_not_found(self, request, response, pk, *args, **kwargs):
        instance = self.get_object(request, pk=pk)
        if instance is None:
            return True
        request.instance = instance
        return False


class FieldResource(FieldBase):
    "Resource for interacting with Field instances."

    def get(self, request, pk):
        instance = request.instance
        usage.log('read', instance=instance, request=request)
        
        # If the field is an orphan then log an error before returning an error
        if is_field_orphaned(instance):
            log.error('Error occurred when retrieving orphaned field \
                    {}.{}.{} with id {}'.format(instance.app_name,
                        instance.model_name, instance.field_name, instance.pk))
            return HttpResponse(status=codes.internal_server_error,
                    content="Error occurred when retrieving orphaned field")

        return self.prepare(request, instance)


class FieldsResource(FieldResource):
    "Field Collection Resource"

    def is_not_found(self, request, response, *args, **kwargs):
        return False

    def get(self, request):
        params = self.get_params(request)
        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied, otherwise
        # only allow for published objects.
        if can_change_field(request.user):
            filters = {}

            if params['published'] is not None:
                filters['published'] = params['published']

            if params['archived'] is not None:
                filters['archived'] = params['archived']

            if filters:
                queryset = queryset.filter(**filters)
        else:
            queryset = queryset.published()

        # If Haystack is installed, perform the search
        if params['query'] and OPTIONAL_DEPS['haystack']:
            usage.log('search', model=self.model, request=request, data={
                'query': params['query'],
            })
            results = self.model.objects.search(params['query'],
                queryset=queryset, max_results=params['limit'],
                partial=True)
            objects = (x.object for x in results)
        else:
            if params['sort'] == 'name':
                order = '-name' if params['order'] == 'desc' else 'name'
                queryset = queryset.order_by(order)

            if params['limit']:
                queryset = queryset[:params['limit']]

            objects = queryset
        
        # Remove any orphaned fields before preparing the response
        orphans = [o for o in objects if is_field_orphaned(o)]
        orphan_pks = []
        for o in orphans:
            log.warning('Truncating orphaned field {}.{}.{} with id {}'.format(
                o.app_name, o.model_name, o.field_name, o.pk))
            orphan_pks.append(o.pk)
        objects = objects.exclude(pk__in=orphan_pks)

        return self.prepare(request, objects, **params)
{'fraction_non_alphanumeric': 0.10153157804164516, 'fraction_numerical': 0.0002581311306143521, 'mean_word_length': 2.5829223181257706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4530778', 'n_tokens_mistral': 3235, 'n_tokens_neox': 3108, 'n_words': 847}