<commit_before>import csv

from django.conf import settings
from django.core.serializers import json
from django.http import HttpResponse
from tastypie.cache import SimpleCache
from tastypie.resources import ModelResource
from tastypie.throttle import CacheThrottle
from tastypie.serializers import Serializer

# are we using DummyCache ?
_cache = getattr(settings, 'CACHES', {})
_cache_default = _cache.get('default')
_is_dummy = _cache_default and _cache_default['BACKEND'].endswith('DummyCache')


class SmartCacheThrottle(CacheThrottle):
    "Make sure throttling works with DummyCache"

    def should_be_throttled(self, identifier, **kwargs):
        # Tastypie breaks when using dummy cache.  if cache type is dummy
        # we'll pretend the key wasn't found
        if _is_dummy:
            return False

        return super(SmartCacheThrottle, self).should_be_throttled(
            identifier, **kwargs)


class IterJSONAndCSVSerializer(Serializer):

    formats = Serializer.formats + ['csv']
    content_types = dict(
        Serializer.content_types.items() + [('csv', 'text/csv')]
    )

    def to_json(self, data, options=None):
        options = options or {}

        data = self.to_simple(data, options)
        return ''.join(json.DjangoJSONEncoder(sort_keys=True).iterencode(data))

    def to_csv(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        response = HttpResponse(mimetype='text/csv')
        response['Content-Disposition'] = 'attachment; filename=data.csv'

        response.write(u'\ufeff'.encode('utf8'))  # BOM for excel
        writer = csv.writer(response, dialect='excel')

        #   if data contains an 'objects' key, refer to it's value as a list of objects.
        #   else, treat data as a single object itself
        objects = data.get('objects', [data])
        for item in objects:
            writer.writerow([unicode(item[key]).encode(
                "utf-8", "replace") for key in item.keys()])
        return response


class BaseResource(ModelResource):

    """Adds to Meta the following options:

    * ``list_fields``: The fields to display in resources listing

    For list mode, on may add to ``extra_fields`` to GET params to get
    addtional comma separated field in case one need more fields than those
    specified in ``list_fields``. e.g:

        GET /api/v2/some_resource/?extra_fields=img_url,number_of_children
    """

    class Meta:
        cache = SimpleCache()
        throttle = SmartCacheThrottle(throttle_at=60, timeframe=60)
        serializer = IterJSONAndCSVSerializer(
            formats=['json', 'jsonp', 'xml', 'csv'])

    def _get_list_fields(self, request):
        """Helper to return list and extra fields for list mode.

        Make things easier for overriding.
        """

        field_names = getattr(self._meta, 'list_fields', None)
        if field_names:
            field_names = field_names[:]
            extra_fields = request.GET.get('extra_fields', None)

            if extra_fields:<commit_msg>remove xml from api v2 formats, because it becomes default, due to browswers sending accept:xml header, and we don't have lxml installed on the servers.<commit_after>import csv

from django.conf import settings
from django.core.serializers import json
from django.http import HttpResponse
from tastypie.cache import SimpleCache
from tastypie.resources import ModelResource
from tastypie.throttle import CacheThrottle
from tastypie.serializers import Serializer

# are we using DummyCache ?
_cache = getattr(settings, 'CACHES', {})
_cache_default = _cache.get('default')
_is_dummy = _cache_default and _cache_default['BACKEND'].endswith('DummyCache')


class SmartCacheThrottle(CacheThrottle):
    "Make sure throttling works with DummyCache"

    def should_be_throttled(self, identifier, **kwargs):
        # Tastypie breaks when using dummy cache.  if cache type is dummy
        # we'll pretend the key wasn't found
        if _is_dummy:
            return False

        return super(SmartCacheThrottle, self).should_be_throttled(
            identifier, **kwargs)


class IterJSONAndCSVSerializer(Serializer):

    formats = Serializer.formats + ['csv']
    content_types = dict(
        Serializer.content_types.items() + [('csv', 'text/csv')]
    )

    def to_json(self, data, options=None):
        options = options or {}

        data = self.to_simple(data, options)
        return ''.join(json.DjangoJSONEncoder(sort_keys=True).iterencode(data))

    def to_csv(self, data, options=None):
        options = options or {}
        data = self.to_simple(data, options)
        response = HttpResponse(mimetype='text/csv')
        response['Content-Disposition'] = 'attachment; filename=data.csv'

        response.write(u'\ufeff'.encode('utf8'))  # BOM for excel
        writer = csv.writer(response, dialect='excel')

        #   if data contains an 'objects' key, refer to it's value as a list of objects.
        #   else, treat data as a single object itself
        objects = data.get('objects', [data])
        for item in objects:
            writer.writerow([unicode(item[key]).encode(
                "utf-8", "replace") for key in item.keys()])
        return response


class BaseResource(ModelResource):

    """Adds to Meta the following options:

    * ``list_fields``: The fields to display in resources listing

    For list mode, on may add to ``extra_fields`` to GET params to get
    addtional comma separated field in case one need more fields than those
    specified in ``list_fields``. e.g:

        GET /api/v2/some_resource/?extra_fields=img_url,number_of_children
    """

    class Meta:
        cache = SimpleCache()
        throttle = SmartCacheThrottle(throttle_at=60, timeframe=60)
        serializer = IterJSONAndCSVSerializer(
            formats=['json', 'jsonp', 'csv'])

    def _get_list_fields(self, request):
        """Helper to return list and extra fields for list mode.

        Make things easier for overriding.
        """

        field_names = getattr(self._meta, 'list_fields', None)
        if field_names:
            field_names = field_names[:]
            extra_fields = request.GET.get('extra_fields', None)

            if extra_fields:
{'fraction_non_alphanumeric': 0.09808612440191387, 'fraction_numerical': 0.0023923444976076554, 'mean_word_length': 3.117531188443861, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6162941', 'n_tokens_mistral': 1831, 'n_tokens_neox': 1774, 'n_words': 532}