<commit_before>import pprint as pp
from dateutil import parser
import StringIO
import csv,json,math
from collections import defaultdict

from tastypie.serializers import Serializer
from tastypie.resources import ModelResource, ALL
from tastypie.authorization import Authorization
from tastypie.authentication import ApiKeyAuthentication
from tastypie import fields
from tastypie.bundle import Bundle
from tastypie.serializers import Serializer

from django.utils.decorators import method_decorator
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from stronghold.decorators import public
import pandas as pd

from datapoints.models import *


class CustomSerializer(Serializer):
    formats = ['json', 'csv']
    content_types = {
        'json': 'application/json',
        'csv': 'text/csv',
    }

    def campaign_region_pivot(self,list_of_dicts):

        meta = None

        try:
            objects = list_of_dicts['objects']
        except KeyError as e:
            objects = []

        if not meta:
            try:
                meta = list_of_dicts['meta']
            except KeyError as e:
                pass

        df = pd.DataFrame(objects)

        try:
            pivoted = pd.pivot_table(df, values='value', index=['region', 'campaign'],
                     columns=['indicator'],aggfunc = lambda x: x)
        except KeyError:
            pivoted = pd.DataFrame()

        return pivoted, meta


    def to_csv(self, data, options=None):

        options = options or {}
        data = self.to_simple(data, options)

        pivoted,meta = self.campaign_region_pivot(data)

        csv = StringIO.StringIO(str(pivoted.to_csv()))

        return csv


    def to_json(self, data, options=None):

        pp.pprint(options)

        options = options or {}
        data = self.to_simple(data, options)

        pivoted,meta = self.campaign_region_pivot(data)

        pivoted_dict = pivoted.to_dict()
        cleaned_dict = {} ## JSON CANT SERIALIZE TUPLE_DICTS

        for indicator,tuple_dict in pivoted_dict.iteritems():

            indicator_values = []

            for reg_camp, value in tuple_dict.iteritems():

                if type(value) == float and math.isnan(value):
                    value = None

                reg_camp_dict = {}

                reg_camp_dict['region'] = reg_camp[0]
                reg_camp_dict['campaign'] = reg_camp[1]
                reg_camp_dict['value'] = value

                indicator_values.append(reg_camp_dict)

            cleaned_dict[indicator] = indicator_values

        cleaned_dict['meta'] = meta
        json_data = json.dumps(cleaned_dict)


        return json_data
        # return json.dumps(data)




class SimpleApiResource(ModelResource):
    '''
    This is the top level class all other Resource Classes inherit from this.
    The API Key authentication is defined here and thus is required by all
    other resources.  This class enherits fro the Tastyppie "ModelResource"

    See Here: http://django-tastypie.readthedocs.org/en/latest/resources.html?highlight=modelresource
    '''<commit_msg>meta is at the top of the response<commit_after>import pprint as pp
from dateutil import parser
import StringIO
import csv,json,math
from collections import defaultdict

from tastypie.serializers import Serializer
from tastypie.resources import ModelResource, ALL
from tastypie.authorization import Authorization
from tastypie.authentication import ApiKeyAuthentication
from tastypie import fields
from tastypie.bundle import Bundle
from tastypie.serializers import Serializer

from django.utils.decorators import method_decorator
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from stronghold.decorators import public
import pandas as pd

from datapoints.models import *


class CustomSerializer(Serializer):
    formats = ['json', 'csv']
    content_types = {
        'json': 'application/json',
        'csv': 'text/csv',
    }

    def campaign_region_pivot(self,list_of_dicts):

        meta = None

        try:
            objects = list_of_dicts['objects']
        except KeyError as e:
            objects = []

        if not meta:
            try:
                meta = list_of_dicts['meta']
            except KeyError as e:
                pass

        df = pd.DataFrame(objects)

        try:
            pivoted = pd.pivot_table(df, values='value', index=['region', 'campaign'],
                     columns=['indicator'],aggfunc = lambda x: x)
        except KeyError:
            pivoted = pd.DataFrame()

        return pivoted, meta


    def to_csv(self, data, options=None):

        options = options or {}
        data = self.to_simple(data, options)

        pivoted,meta = self.campaign_region_pivot(data)

        csv = StringIO.StringIO(str(pivoted.to_csv()))

        return csv


    def to_json(self, data, options=None):

        response_dict = {}

        options = options or {}
        data = self.to_simple(data, options)

        pivoted,meta = self.campaign_region_pivot(data)

        pivoted_dict = pivoted.to_dict()
        cleaned_dict = {} ## JSON CANT SERIALIZE TUPLE_DICTS

        for indicator,tuple_dict in pivoted_dict.iteritems():

            indicator_values = []

            for reg_camp, value in tuple_dict.iteritems():

                if type(value) == float and math.isnan(value):
                    value = None

                reg_camp_dict = {}

                reg_camp_dict['region'] = reg_camp[0]
                reg_camp_dict['campaign'] = reg_camp[1]
                reg_camp_dict['value'] = value

                indicator_values.append(reg_camp_dict)

            cleaned_dict[indicator] = indicator_values

        response_dict['meta'] = meta
        response_dict['objects'] = cleaned_dict

        json_data = json.dumps(response_dict)


        return json_data
        # return json.dumps(data)




class SimpleApiResource(ModelResource):
    '''
    This is the top level class all other Resource Classes inherit from this.
    The API Key authentication is defined here and thus is required by all
    other resources.  This class enherits fro the Tastyppie "ModelResource"

    See Here: http://django-tastypie.readthedocs.org/en/latest/resources.html?highlight=modelresource
    '''
{'fraction_non_alphanumeric': 0.08196979232221523, 'fraction_numerical': 0.0006293266205160479, 'mean_word_length': 2.5257903494176372, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3708983', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1740, 'n_words': 471}