<commit_before>import json
from collections import defaultdict
from decimal import Decimal
from django.conf.urls import patterns, url
from django.http import HttpResponse
from django.db.models import Q
from django.conf import settings
from django.core.urlresolvers import reverse
from restlib2 import utils
from restlib2.http import codes
from modeltree.tree import trees
from avocado.conf import settings as _settings
from avocado.models import DataField
from avocado.stats import cluster as stats_cluster
from .base import BaseResource

DATA_CHOICES_MAP = _settings.DATA_CHOICES_MAP
SQLITE_AGG_EXT = getattr(settings, 'SQLITE_AGG_EXT', False)
AGG_FUNCTIONS = ['count', 'avg', 'min', 'max', 'stddev', 'variance']

MINIMUM_OBSERVATIONS = 500
MAXIMUM_OBSERVATIONS = 50000

SAFE_METHODS = ('GET', 'HEAD', 'OPTIONS')

# Shortcuts defined ahead of time for transparency
can_change_datafield = lambda u: u.has_perm('avocado.change_datafield')


class DataFieldBase(BaseResource):
    param_defaults = {
        'query': '',
    }

    def get_queryset(self, request):
        queryset = DataField.objects.all()
        if not can_change_datafield(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except DataField.DoesNotExist:
            pass

    def is_forbidden(self, request, response, *args, **kwargs):
        "Ensure non-privileged users cannot make any changes."
        if request.method not in SAFE_METHODS and not can_change_datafield(request.user):
            return True

    def is_not_found(self, request, response, pk=None, *args, **kwargs):
        if pk:
            instance = self.get_object(request, pk=pk)
            if instance is None:
                return True
            request.instance = instance
            return False


class DataFieldResource(DataFieldBase):
    "DataField Summary Resource"

    # Template for top-level attributes
    template = {
        'fields': [
            ':pk', 'name', 'plural_name', 'description', 'keywords',
            'category', 'app_name', 'model_name', 'field_name',
            'modified', 'published', 'archived', 'operators'
        ],
        'key_map': {
            'plural_name': 'get_plural_name',
            'operators': 'operator_choices',
        },
        'related': {
            'category': {
                'fields': [':pk', 'name', 'order', 'parent_id']
            },
        },
    }

    # Template for data-related attributes
    data_template = {
        'fields': [
            'simple_type', 'internal_type', 'modified', 'enumerable',
            'searchable', 'unit', 'plural_unit'
        ],
        'key_map': {
            'modified': 'data_modified',
            'plural_unit': 'get_plural_unit',
        }
    }

    @classmethod
    def serialize(self, instance):
        obj = utils.serialize(instance, **self.template)
        obj['url'] = reverse('datafield', args=[instance.pk])
        obj['data'] = utils.serialize(instance, **self.data_template)

        obj['links'] = {
            'values': {
                'rel': 'data',
                'href': reverse('datafield-values', args=[instance.pk]),
            },
            'stats': {
                'rel': 'data',
                'href': reverse('datafield-stats', args=[instance.pk]),
            },
            'distribution': {
                'rel': 'data',
                'href': reverse('datafield-distribution', args=[instance.pk]),
            },
        }
        return obj

    def get(self, request, pk=None):
        params = self.get_params(request)

        # Process GET parameters
        sort = params.get('sort')        # default: model ordering
        direction = params.get('direction')  # default: desc
        published = params.get('published')
        archived = params.get('archived')
        query = params.get('query').strip()

        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied
        if can_change_datafield(request.user):
            filters = {}

            if published == 'true':
                filters['published'] = True
            elif published == 'false':
                filters['published'] = False

            if archived == 'true':
                filters['archived'] = True
            elif archived == 'false':
                filters['archived'] = False

            if filters:
                queryset = queryset.filter(**filters)
        # For non-privileged users, filter out the non-published and archived
        else:
            queryset = queryset.published()

        # Early exit if dealing with a single object, no need to apply sorting
        if pk:
            return self.serialize(request.instance)

        # If there is a query parameter, perform the search
        if query:
            results = DataField.objects.search(query, queryset)
            return map(lambda x: self.serialize(x.object), results)

        # Apply sorting
        if sort == 'name':
            if direction == 'asc':
                queryset = queryset.order_by('name')
            else:
                queryset = queryset.order_by('-name')

        return map(self.serialize, queryset.iterator())


class DataFieldValues(DataFieldBase):
    "DataField Values Resource"

    def get(self, request, pk):
        instance = request.instance

        params = self.get_params(request)

        tree = trees[instance.model]
        context = self.get_context(request)
        queryset = context.apply(queryset=instance.query(), tree=tree).distinct()\
            .annotate(count=Count(instance.field_name))
        query = params.get('query').strip()

        if query:
            queryset = queryset.search_values(query)

        counts = {}
        for obj in queryset.iterator():
            counts[obj[instance.field_name]] = obj['count']

        results = []
        for value in instance.values:
            results.append({
                'name': smart_unicode(DATA_CHOICES_MAP.get(value, value)),
                'value': value,
                'count': counts.get(value, 0)
            })
        return results


class DataFieldStats(DataFieldBase):
    "DataField Stats Resource"

    def get(self, request, pk):
        instance = request.instance

        stats = None

        if instance.simple_type == 'number':
            stats = instance.max().min().avg()
        elif instance.simple_type == 'string' and instance.enumerable:
            stats = instance.count(distinct=True)

        if stats is None:
            resp = {}
        else:
            resp = iter(stats).next()

        resp['links'] = {
            'parent': {
                'rel': 'parent',
                'href': reverse('datafield', args=[instance.pk]),<commit_msg>Remove inclusion of counts in DataFieldValues resource There was a reasonably significant overhead for recalculating counts on each request. This may be implemented in the future as an GET parameter<commit_after>import json
from collections import defaultdict
from decimal import Decimal
from django.conf.urls import patterns, url
from django.http import HttpResponse
from django.db.models import Q
from django.conf import settings
from django.core.urlresolvers import reverse
from restlib2 import utils
from restlib2.http import codes
from modeltree.tree import trees
from avocado.conf import settings as _settings
from avocado.models import DataField
from avocado.stats import cluster as stats_cluster
from .base import BaseResource

RAW_DATA_MAP = _settings.RAW_DATA_MAP
SQLITE_AGG_EXT = getattr(settings, 'SQLITE_AGG_EXT', False)
AGG_FUNCTIONS = ['count', 'avg', 'min', 'max', 'stddev', 'variance']

MINIMUM_OBSERVATIONS = 500
MAXIMUM_OBSERVATIONS = 50000

SAFE_METHODS = ('GET', 'HEAD', 'OPTIONS')

# Shortcuts defined ahead of time for transparency
can_change_datafield = lambda u: u.has_perm('avocado.change_datafield')


class DataFieldBase(BaseResource):
    param_defaults = {
        'query': '',
    }

    def get_queryset(self, request):
        queryset = DataField.objects.all()
        if not can_change_datafield(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except DataField.DoesNotExist:
            pass

    def is_forbidden(self, request, response, *args, **kwargs):
        "Ensure non-privileged users cannot make any changes."
        if request.method not in SAFE_METHODS and not can_change_datafield(request.user):
            return True

    def is_not_found(self, request, response, pk=None, *args, **kwargs):
        if pk:
            instance = self.get_object(request, pk=pk)
            if instance is None:
                return True
            request.instance = instance
            return False


class DataFieldResource(DataFieldBase):
    "DataField Summary Resource"

    # Template for top-level attributes
    template = {
        'fields': [
            ':pk', 'name', 'plural_name', 'description', 'keywords',
            'category', 'app_name', 'model_name', 'field_name',
            'modified', 'published', 'archived', 'operators'
        ],
        'key_map': {
            'plural_name': 'get_plural_name',
            'operators': 'operator_choices',
        },
        'related': {
            'category': {
                'fields': [':pk', 'name', 'order', 'parent_id']
            },
        },
    }

    # Template for data-related attributes
    data_template = {
        'fields': [
            'simple_type', 'internal_type', 'modified', 'enumerable',
            'searchable', 'unit', 'plural_unit'
        ],
        'key_map': {
            'modified': 'data_modified',
            'plural_unit': 'get_plural_unit',
        }
    }

    @classmethod
    def serialize(self, instance):
        obj = utils.serialize(instance, **self.template)
        obj['url'] = reverse('datafield', args=[instance.pk])
        obj['data'] = utils.serialize(instance, **self.data_template)

        obj['links'] = {
            'values': {
                'rel': 'data',
                'href': reverse('datafield-values', args=[instance.pk]),
            },
            'stats': {
                'rel': 'data',
                'href': reverse('datafield-stats', args=[instance.pk]),
            },
            'distribution': {
                'rel': 'data',
                'href': reverse('datafield-distribution', args=[instance.pk]),
            },
        }
        return obj

    def get(self, request, pk=None):
        params = self.get_params(request)

        # Process GET parameters
        sort = params.get('sort')        # default: model ordering
        direction = params.get('direction')  # default: desc
        published = params.get('published')
        archived = params.get('archived')
        query = params.get('query').strip()

        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied
        if can_change_datafield(request.user):
            filters = {}

            if published == 'true':
                filters['published'] = True
            elif published == 'false':
                filters['published'] = False

            if archived == 'true':
                filters['archived'] = True
            elif archived == 'false':
                filters['archived'] = False

            if filters:
                queryset = queryset.filter(**filters)
        # For non-privileged users, filter out the non-published and archived
        else:
            queryset = queryset.published()

        # Early exit if dealing with a single object, no need to apply sorting
        if pk:
            return self.serialize(request.instance)

        # If there is a query parameter, perform the search
        if query:
            results = DataField.objects.search(query, queryset)
            return map(lambda x: self.serialize(x.object), results)

        # Apply sorting
        if sort == 'name':
            if direction == 'asc':
                queryset = queryset.order_by('name')
            else:
                queryset = queryset.order_by('-name')

        return map(self.serialize, queryset.iterator())


class DataFieldValues(DataFieldBase):
    """DataField Values Resource

    This resource can be overriden for any datafield to use a more
    performant search implementation.
    """

    def get(self, request, pk):
        instance = request.instance

        params = self.get_params(request)
        query = params.get('query').strip()

        results = []

        # If a query term is supplied, perform the icontains search
        if query:
            for value in instance.search(query):
                results.append({
                    'label': RAW_DATA_MAP.get(value, value),
                    'value': value,
                })
        # ..otherwise use the cached choices
        else:
            for value, name in instance.choices:
                results.append({
                    'label': name,
                    'value': value,
                })
        return results


class DataFieldStats(DataFieldBase):
    "DataField Stats Resource"

    def get(self, request, pk):
        instance = request.instance

        stats = None

        if instance.simple_type == 'number':
            stats = instance.max().min().avg()
        elif instance.simple_type == 'string' and instance.enumerable:
            stats = instance.count(distinct=True)

        if stats is None:
            resp = {}
        else:
            resp = iter(stats).next()

        resp['links'] = {
            'parent': {
                'rel': 'parent',
                'href': reverse('datafield', args=[instance.pk]),
{'fraction_non_alphanumeric': 0.1111745805069618, 'fraction_numerical': 0.0014994644769725097, 'mean_word_length': 2.1214620013371963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5754101', 'n_tokens_mistral': 3859, 'n_tokens_neox': 3701, 'n_words': 995}