<commit_before>import json
import datetime
import traceback
from random import randint
from pprint import pprint
from collections import defaultdict

from django.core.serializers import json as djangojson
from django.db.models import Model, ManyToManyField
from django.db.models.query import RawQuerySet
from django.db.models.sql.constants import QUERY_TERMS
from django.db.models.options import Options
from django.db.models.fields import AutoField, FieldDoesNotExist
from django.db import connection

from django.forms.models import model_to_dict
from django.utils.encoding import smart_str
from django.contrib.auth.models import User, Group
from django.core import serializers

from datapoints.models import *
from datapoints.cache_tasks import CacheRefresh
from source_data.models import *
from source_data.etl_tasks.refresh_master import MasterRefresh


class v2Request(object):

    def __init__(self, request, content_type):
        '''
        The v2 request when instatiated setts up class variables for the request
        the content type and the user_id sending the request.

        Most importantly howver, the v2 api uses the dictionary below in order
        to translate the content type in the api ( api/v1/<content_type> ) into
        a database model (Region, Campaign, Indicator) as well as an optional
        function to perform filtering based on the user's permissions or any
        other transformation to the data before return to the application.
        '''

        self.request = request
        self.content_type = content_type
        self.user_id = request.user.id
        self.data = None
        self.meta = None
        self.err = None

        # Tells the API which models are avail for GET / POST / META requests #
        self.orm_mapping = {
            'refresh_master': {'orm_obj':Document,
                'permission_function':self.refresh_master_for_document},
            'campaign': {'orm_obj':CampaignAbstracted,
                'permission_function':self.apply_campaign_permissions},
            'region': {'orm_obj':Region,
                'permission_function':self.apply_region_permissions},
            'indicator': {'orm_obj':IndicatorAbstracted,
                'permission_function':None},
            'datapoint': {'orm_obj':DataPoint,
                'permission_function':None},
            'doc_mapping' : {'orm_obj':SourceObjectMap,
                'permission_function': self.pretty_doc_mapping},
            'doc_datapoint' : {'orm_obj':DocDataPoint,
                'permission_function': self.pretty_doc_datapoint},
            'synced_datapoint' : {'orm_obj':DataPointComputed,
                'permission_function': self.filter_calced_dp_by_doc_id},
            'document': {'orm_obj':Document,
                'permission_function':self.apply_document_permissions },
            'custom_dashboard': {'orm_obj':CustomDashboard,
                'permission_function':self.apply_cust_dashboard_permissions},
            # 'user_permission': {'orm_obj':UserAuthFunction,
                # 'permission_function':self.filter_permissions_to_current_user},
            'user_permission': {'orm_obj':User,
                'permission_function':None},
            'group_permission': {'orm_obj':IndicatorPermission,
                'permission_function':None},
            'geo': {'orm_obj':RegionPolygon,
                'permission_function':None},
            'group': {'orm_obj':Group,
                'permission_function':None},
            'user': {'orm_obj':User,
                'permission_function':None},
            'region_permission': {'orm_obj':RegionPermission,
                'permission_function':None},
            'user_group': {'orm_obj':UserGroup,
                'permission_function':None},
            'office': {'orm_obj':Office,
                'permission_function':None},
            'indicator_tag': {'orm_obj':IndicatorTag,
                'permission_function':None},
            'indicator_to_tag': {'orm_obj':IndicatorToTag,
                'permission_function':None},
            'campaign_type': {'orm_obj':CampaignType,
                'permission_function':None},
            'region_type': {'orm_obj':RegionType,
                'permission_function':None},
            'source_object_map' : {'orm_obj': SourceObjectMap,
                'permission_function':None},
        }


    def main(self):
        '''
        Put together a response with the data, meta, and error objects, returing
        this all to the django view.
        '''

        response_data = {
            'objects':self.data,
            'meta':self.meta,
            'error':self.err,
        }

        return response_data

    def refresh_master_for_document(self, list_of_object_ids):


        mr = MasterRefresh(self.user_id,self.document_id)
        qset = Document.objects.raw('''

            SELECT id FROM source_data_document WHERE id = %s;

        ''',[self.document_id])<commit_msg>add doc_datapoint count and all of the document info to the response of refresh_master<commit_after>import json
import datetime
import traceback
from random import randint
from pprint import pprint
from collections import defaultdict

from django.core.serializers import json as djangojson
from django.db.models import Model, ManyToManyField
from django.db.models.query import RawQuerySet
from django.db.models.sql.constants import QUERY_TERMS
from django.db.models.options import Options
from django.db.models.fields import AutoField, FieldDoesNotExist
from django.db import connection

from django.forms.models import model_to_dict
from django.utils.encoding import smart_str
from django.contrib.auth.models import User, Group
from django.core import serializers

from datapoints.models import *
from datapoints.cache_tasks import CacheRefresh
from source_data.models import *
from source_data.etl_tasks.refresh_master import MasterRefresh


class v2Request(object):

    def __init__(self, request, content_type):
        '''
        The v2 request when instatiated setts up class variables for the request
        the content type and the user_id sending the request.

        Most importantly howver, the v2 api uses the dictionary below in order
        to translate the content type in the api ( api/v1/<content_type> ) into
        a database model (Region, Campaign, Indicator) as well as an optional
        function to perform filtering based on the user's permissions or any
        other transformation to the data before return to the application.
        '''

        self.request = request
        self.content_type = content_type
        self.user_id = request.user.id
        self.data = None
        self.meta = None
        self.err = None

        # Tells the API which models are avail for GET / POST / META requests #
        self.orm_mapping = {
            'refresh_master': {'orm_obj':Document,
                'permission_function':self.refresh_master_for_document},
            'campaign': {'orm_obj':CampaignAbstracted,
                'permission_function':self.apply_campaign_permissions},
            'region': {'orm_obj':Region,
                'permission_function':self.apply_region_permissions},
            'indicator': {'orm_obj':IndicatorAbstracted,
                'permission_function':None},
            'datapoint': {'orm_obj':DataPoint,
                'permission_function':None},
            'doc_mapping' : {'orm_obj':SourceObjectMap,
                'permission_function': self.pretty_doc_mapping},
            'doc_datapoint' : {'orm_obj':DocDataPoint,
                'permission_function': self.pretty_doc_datapoint},
            'synced_datapoint' : {'orm_obj':DataPointComputed,
                'permission_function': self.filter_calced_dp_by_doc_id},
            'document': {'orm_obj':Document,
                'permission_function':self.apply_document_permissions },
            'custom_dashboard': {'orm_obj':CustomDashboard,
                'permission_function':self.apply_cust_dashboard_permissions},
            # 'user_permission': {'orm_obj':UserAuthFunction,
                # 'permission_function':self.filter_permissions_to_current_user},
            'user_permission': {'orm_obj':User,
                'permission_function':None},
            'group_permission': {'orm_obj':IndicatorPermission,
                'permission_function':None},
            'geo': {'orm_obj':RegionPolygon,
                'permission_function':None},
            'group': {'orm_obj':Group,
                'permission_function':None},
            'user': {'orm_obj':User,
                'permission_function':None},
            'region_permission': {'orm_obj':RegionPermission,
                'permission_function':None},
            'user_group': {'orm_obj':UserGroup,
                'permission_function':None},
            'office': {'orm_obj':Office,
                'permission_function':None},
            'indicator_tag': {'orm_obj':IndicatorTag,
                'permission_function':None},
            'indicator_to_tag': {'orm_obj':IndicatorToTag,
                'permission_function':None},
            'campaign_type': {'orm_obj':CampaignType,
                'permission_function':None},
            'region_type': {'orm_obj':RegionType,
                'permission_function':None},
            'source_object_map' : {'orm_obj': SourceObjectMap,
                'permission_function':None},
        }


    def main(self):
        '''
        Put together a response with the data, meta, and error objects, returing
        this all to the django view.
        '''

        response_data = {
            'objects':self.data,
            'meta':self.meta,
            'error':self.err,
        }

        return response_data

    def refresh_master_for_document(self, list_of_object_ids):


        mr = MasterRefresh(self.user_id,self.document_id)
        qset = Document.objects.raw('''

            DROP TABLE IF EXISTS _this_doc;
            CREATE TEMP TABLE _this_doc AS
            SELECT
                sd.*
            FROM
            source_data_document sd
            WHERE id = %s;

            SELECT
                td.*
                ,x.doc_datapoint_count
            FROM _this_doc td
            INNER JOIN (
                SELECT COUNT(1) as doc_datapoint_count
                FROM doc_datapoint dd
                WHERE EXISTS (
                        SELECT 1 FROM _this_doc td
                        WHERE dd.document_id = td.id
                    )
                )x
            ON 1=1;

        ''',[self.document_id])
{'fraction_non_alphanumeric': 0.09064299877254273, 'fraction_numerical': 0.001133037484656784, 'mean_word_length': 2.415672363753628, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3700863', 'n_tokens_mistral': 2841, 'n_tokens_neox': 2732, 'n_words': 659}