<commit_before>from collections import defaultdict
from decimal import Decimal
from django.conf.urls import patterns, url
from django.db import router
from django.db.models import Q, Count
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.encoding import smart_unicode
from restlib2 import utils
from modeltree.tree import trees
from avocado.conf import settings as _settings
from avocado.models import DataField
from avocado.stats import cluster
from .base import BaseResource

DATA_CHOICES_MAP = _settings.DATA_CHOICES_MAP
SQLITE_AGG_EXT = getattr(settings, 'SQLITE_AGG_EXT', False)
# AGG_FUNCTIONS = ['count', 'avg', 'min', 'max', 'stddev', 'variance']

# Apply the 'rule of thumb' for determining the appropriate number of
# clusters relative to the # of observations. Default to 50 to ensure
# a trend is somewhat visible
REQUIRED_OBSERVATIONS = 500

SAFE_METHODS = ('GET', 'HEAD', 'OPTIONS')

# Shortcuts defined ahead of time for transparency
can_change_datafield = lambda u: u.has_perm('avocado.change_datafield')


class DataFieldBase(BaseResource):
    def get_queryset(self, request):
        queryset = DataField.objects.all()
        if not can_change_datafield(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except DataField.DoesNotExist:
            pass

    def is_forbidden(self, request, response, *args, **kwargs):
        "Ensure non-privileged users cannot make any changes."
        if request.method not in SAFE_METHODS and not can_change_datafield(request.user):
            return True

    def is_not_found(self, request, response, pk=None, *args, **kwargs):
        if pk:
            instance = self.get_object(request, pk=pk)
            if instance is None:
                return True
            request.instance = instance
            return False


class DataFieldResource(DataFieldBase):
    "DataField Summary Resource"

    # Template for top-level attributes
    template = {
        'fields': [
            ':pk', 'name', 'plural_name', 'description', 'keywords',
            'category', 'app_name', 'model_name', 'field_name',
            'modified', 'published', 'archived', 'operators'
        ],
        'key_map': {
            'plural_name': 'get_plural_name',
            'operators': 'operator_choices',
        },
        'related': {
            'category': {
                'fields': [':pk', 'name', 'order', 'parent_id']
            },
        },
    }

    # Template for data-related attributes
    data_template = {
        'fields': [
            'type', 'modified', 'enumerable', 'searchable', 'unit',
            'plural_unit'
        ],
        'key_map': {
            'type': 'datatype',
            'modified': 'data_modified',
            'plural_unit': 'get_plural_unit',
        }
    }

    @classmethod
    def serialize(self, instance):
        obj = utils.serialize(instance, **self.template)
        obj['url'] = reverse('datafield', args=[instance.pk])
        obj['data'] = utils.serialize(instance, **self.data_template)

        obj['links'] = {
            'values': {
                'rel': 'data',
                'href': reverse('datafield-values', args=[instance.pk]),
            },
#            'stats': {
#                'rel': 'data',
#                'href': reverse('datafield-stats', args=[instance.pk]),
#            },
            'distribution': {
                'rel': 'data',
                'href': reverse('datafield-distribution', args=[instance.pk]),
            },
        }
        return obj

    def get(self, request, pk=None):
        # Process GET parameters
        sort = request.GET.get('sort', None)        # default: model ordering
        direction = request.GET.get('direction', None)  # default: desc
        published = request.GET.get('published', None)
        archived = request.GET.get('archived', None)
        query = request.GET.get('query', '').strip()

        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied
        if can_change_datafield(request.user):
            filters = {}

            if published == 'true':
                filters['published'] = True
            elif published == 'false':
                filters['published'] = False

            if archived == 'true':
                filters['archived'] = True
            elif archived == 'false':
                filters['archived'] = False

            if filters:
                queryset = queryset.filter(**filters)
        # For non-privileged users, filter out the non-published and archived
        else:
            queryset = queryset.published()

        # Early exit if dealing with a single object, no need to apply sorting
        if pk:
            return self.serialize(request.instance)

        # If there is a query parameter, perform the search
        if query:
            results = DataField.objects.search(query, queryset)
            return map(lambda x: self.serialize(x.object), results)

        # Apply sorting
        if sort == 'name':
            if direction == 'asc':
                queryset = queryset.order_by('name')
            else:
                queryset = queryset.order_by('-name')

        return map(self.serialize, queryset.iterator())


class DataFieldValues(DataFieldBase):
    "DataField Values Resource"

    param_defaults = {
        'query': ''
    }

    def get(self, request, pk):
        instance = request.instance

        params = self.get_params(request)

        tree = trees[instance.model]
        context = self.get_context(request)
        queryset = context.apply(queryset=instance.query(), tree=tree)

        queryset = queryset.annotate(count=Count(instance.field_name))\
            .order_by(instance.field_name)

        query = params.get('query').strip()

        if query:
            queryset = queryset.search_values(query)

        results = []

        for obj in queryset.iterator():
            value = obj[instance.field_name]
            results.append({
                'name': smart_unicode(DATA_CHOICES_MAP.get(value, value)),
                'value': value,
                'count': obj['count'],
            })
        return results


class DataFieldStats(DataFieldBase):
    "DataField Stats Resource"

    def get(self, request, pk):
        instance = request.instance

        stats = instance.count()

        if instance.datatype == 'number':
            stats = stats.avg().max().min().sum()

            # SQLite does not support STDDEV and VARIANCE, so do not include
            # them unless the extension has been installed
            db = router.db_for_read(instance.model)
            if 'sqlite' not in settings.DATABASES[db]['ENGINE'] or SQLITE_AGG_EXT:
                stats = stats.stddev().variance()<commit_msg>The DataField values resource now returns all unique values with counts relative to the DataContext specified (or none)<commit_after>from collections import defaultdict
from decimal import Decimal
from django.conf.urls import patterns, url
from django.db import router
from django.db.models import Q, Count
from django.conf import settings
from django.core.urlresolvers import reverse
from django.utils.encoding import smart_unicode
from restlib2 import utils
from modeltree.tree import trees
from avocado.conf import settings as _settings
from avocado.models import DataField
from avocado.stats import cluster
from .base import BaseResource

DATA_CHOICES_MAP = _settings.DATA_CHOICES_MAP
SQLITE_AGG_EXT = getattr(settings, 'SQLITE_AGG_EXT', False)
# AGG_FUNCTIONS = ['count', 'avg', 'min', 'max', 'stddev', 'variance']

# Apply the 'rule of thumb' for determining the appropriate number of
# clusters relative to the # of observations. Default to 50 to ensure
# a trend is somewhat visible
REQUIRED_OBSERVATIONS = 500

SAFE_METHODS = ('GET', 'HEAD', 'OPTIONS')

# Shortcuts defined ahead of time for transparency
can_change_datafield = lambda u: u.has_perm('avocado.change_datafield')


class DataFieldBase(BaseResource):
    def get_queryset(self, request):
        queryset = DataField.objects.all()
        if not can_change_datafield(request.user):
            queryset = queryset.published()
        return queryset

    def get_object(self, request, **kwargs):
        queryset = self.get_queryset(request)
        try:
            return queryset.get(**kwargs)
        except DataField.DoesNotExist:
            pass

    def is_forbidden(self, request, response, *args, **kwargs):
        "Ensure non-privileged users cannot make any changes."
        if request.method not in SAFE_METHODS and not can_change_datafield(request.user):
            return True

    def is_not_found(self, request, response, pk=None, *args, **kwargs):
        if pk:
            instance = self.get_object(request, pk=pk)
            if instance is None:
                return True
            request.instance = instance
            return False


class DataFieldResource(DataFieldBase):
    "DataField Summary Resource"

    # Template for top-level attributes
    template = {
        'fields': [
            ':pk', 'name', 'plural_name', 'description', 'keywords',
            'category', 'app_name', 'model_name', 'field_name',
            'modified', 'published', 'archived', 'operators'
        ],
        'key_map': {
            'plural_name': 'get_plural_name',
            'operators': 'operator_choices',
        },
        'related': {
            'category': {
                'fields': [':pk', 'name', 'order', 'parent_id']
            },
        },
    }

    # Template for data-related attributes
    data_template = {
        'fields': [
            'type', 'modified', 'enumerable', 'searchable', 'unit',
            'plural_unit'
        ],
        'key_map': {
            'type': 'datatype',
            'modified': 'data_modified',
            'plural_unit': 'get_plural_unit',
        }
    }

    @classmethod
    def serialize(self, instance):
        obj = utils.serialize(instance, **self.template)
        obj['url'] = reverse('datafield', args=[instance.pk])
        obj['data'] = utils.serialize(instance, **self.data_template)

        obj['links'] = {
            'values': {
                'rel': 'data',
                'href': reverse('datafield-values', args=[instance.pk]),
            },
#            'stats': {
#                'rel': 'data',
#                'href': reverse('datafield-stats', args=[instance.pk]),
#            },
            'distribution': {
                'rel': 'data',
                'href': reverse('datafield-distribution', args=[instance.pk]),
            },
        }
        return obj

    def get(self, request, pk=None):
        # Process GET parameters
        sort = request.GET.get('sort', None)        # default: model ordering
        direction = request.GET.get('direction', None)  # default: desc
        published = request.GET.get('published', None)
        archived = request.GET.get('archived', None)
        query = request.GET.get('query', '').strip()

        queryset = self.get_queryset(request)

        # For privileged users, check if any filters are applied
        if can_change_datafield(request.user):
            filters = {}

            if published == 'true':
                filters['published'] = True
            elif published == 'false':
                filters['published'] = False

            if archived == 'true':
                filters['archived'] = True
            elif archived == 'false':
                filters['archived'] = False

            if filters:
                queryset = queryset.filter(**filters)
        # For non-privileged users, filter out the non-published and archived
        else:
            queryset = queryset.published()

        # Early exit if dealing with a single object, no need to apply sorting
        if pk:
            return self.serialize(request.instance)

        # If there is a query parameter, perform the search
        if query:
            results = DataField.objects.search(query, queryset)
            return map(lambda x: self.serialize(x.object), results)

        # Apply sorting
        if sort == 'name':
            if direction == 'asc':
                queryset = queryset.order_by('name')
            else:
                queryset = queryset.order_by('-name')

        return map(self.serialize, queryset.iterator())


class DataFieldValues(DataFieldBase):
    "DataField Values Resource"

    param_defaults = {
        'query': ''
    }

    def get(self, request, pk):
        instance = request.instance
        # Distinct set of values
        values = instance.query().distinct().order_by(instance.field_name)

        params = self.get_params(request)

        tree = trees[instance.model]
        context = self.get_context(request)
        queryset = context.apply(tree=tree)

        queryset = queryset.values(instance.field_name).annotate(count=Count(instance.field_name))
        query = params.get('query').strip()

        if query:
            queryset = queryset.search_values(query)

        counts = {}
        for obj in queryset.iterator():
            counts[obj[instance.field_name]] = obj['count']

        results = []
        for obj in values.iterator():
            value = obj[instance.field_name]
            results.append({
                'name': smart_unicode(DATA_CHOICES_MAP.get(value, value)),
                'value': value,
                'count': counts.get(value, 0)
            })
        return results


class DataFieldStats(DataFieldBase):
    "DataField Stats Resource"

    def get(self, request, pk):
        instance = request.instance

        stats = instance.count()

        if instance.datatype == 'number':
            stats = stats.avg().max().min().sum()

            # SQLite does not support STDDEV and VARIANCE, so do not include
            # them unless the extension has been installed
            db = router.db_for_read(instance.model)
            if 'sqlite' not in settings.DATABASES[db]['ENGINE'] or SQLITE_AGG_EXT:
                stats = stats.stddev().variance()
{'fraction_non_alphanumeric': 0.11170919259000349, 'fraction_numerical': 0.0009087731562390772, 'mean_word_length': 2.227159936837356, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7366754', 'n_tokens_mistral': 3969, 'n_tokens_neox': 3824, 'n_words': 1032}