<commit_before>import json
import logging

from datetime import datetime

from django.contrib import messages
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse_lazy
from django.http import (HttpResponse, HttpResponseBadRequest,
                         HttpResponseForbidden, HttpResponseRedirect)
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import RedirectView, View

from ipware.ip import get_ip

from .models import BaseDataFile, DataRetrievalTask, DataFileAccessLog

logger = logging.getLogger(__name__)


class TaskUpdateView(View):
    """
    Receive and record task success/failure input.
    """

    def post(self, request):
        logger.info('Received task update with: %s', str(request.POST))

        if 'task_data' not in request.POST:
            return HttpResponseBadRequest()

        # TODO: since this is just JSON we could post the JSON directly
        task_data = json.loads(request.POST['task_data'])

        response = self.update_task(task_data)

        return HttpResponse(response)

    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(TaskUpdateView, self).dispatch(*args, **kwargs)

    @classmethod
    def update_task(cls, task_data):
        try:
            task = DataRetrievalTask.objects.get(id=task_data['task_id'])
        except DataRetrievalTask.DoesNotExist:
            logger.warning('No task for ID: %s', task_data['task_id'])

            return 'Invalid task ID!'

        if 'task_state' in task_data:
            cls.update_task_state(task, task_data['task_state'])

        if 's3_keys' in task_data:
            cls.create_datafiles(task, **task_data)

        return 'Thanks!'

    @staticmethod
    def update_task_state(task, task_state):
        # TODO: change SUCCESS to SUCCEEDED on data_processing to match
        if task_state in ['SUCCESS', 'SUCCEEDED']:
            task.status = task.TASK_SUCCEEDED
            task.complete_time = datetime.now()
        elif task_state == 'QUEUED':
            task.status = task.TASK_QUEUED
        elif task_state == 'INITIATED':
            task.status = task.TASK_INITIATED
        # TODO: change FAILURE to FAILED on data_processing to match
        elif task_state in ['FAILURE', 'FAILED']:
            task.status = task.TASK_FAILED
            task.complete_time = datetime.now()

        task.save()

    # pylint: disable=unused-argument
    @staticmethod
    def create_datafiles(task, s3_keys, **kwargs):
        datafile_model = task.datafile_model.model_class()

        assert issubclass(datafile_model, BaseDataFile), (
            '%r is not a subclass of BaseDataFile' % datafile_model)

        userdata_model = (datafile_model._meta
                          .get_field_by_name('user_data')[0]
                          .rel.to)

        user_data, _ = userdata_model.objects.get_or_create(user=task.user)

        # XXX: there's only ever one s3_key (at this point in time)
        for s3_key in s3_keys:
            data_file = datafile_model(user_data=user_data, task=task)

            data_file.file.name = s3_key
            data_file.save()


class BaseDataRetrievalView(View):
    """
    Abstract base class for a view that starts a data retrieval task.

    Class attributes that need to be defined:
        datafile_model (attribute)
            App-specific, a subclass of BaseDataFile

    Class methods that need to be defined:
        get_app_task_params(self)
            Returns a dict with app-specific task parameters. These will be
            stored in the DataRetrievalTask.app_task_params, and will be sent
            to the data processing server wehn the task is run.
    """
    datafile_model = None
    redirect_url = reverse_lazy('my-member-research-data')
    message_error = 'Sorry, our data retrieval server seems to be down.'
    message_started = "Thanks! We've submitted this import task to our server."<commit_msg>🔧 begin adding support for saving metadata<commit_after>import json
import logging

from datetime import datetime

from django.contrib import messages
from django.contrib.contenttypes.models import ContentType
from django.core.urlresolvers import reverse_lazy
from django.http import (HttpResponse, HttpResponseBadRequest,
                         HttpResponseForbidden, HttpResponseRedirect)
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import RedirectView, View

from ipware.ip import get_ip

from .models import BaseDataFile, DataRetrievalTask, DataFileAccessLog

logger = logging.getLogger(__name__)


class TaskUpdateView(View):
    """
    Receive and record task success/failure input.
    """

    def post(self, request):
        logger.info('Received task update with: %s', str(request.POST))

        if 'task_data' not in request.POST:
            return HttpResponseBadRequest()

        # TODO: since this is just JSON we could post the JSON directly
        task_data = json.loads(request.POST['task_data'])

        response = self.update_task(task_data)

        return HttpResponse(response)

    @csrf_exempt
    def dispatch(self, *args, **kwargs):
        return super(TaskUpdateView, self).dispatch(*args, **kwargs)

    @classmethod
    def update_task(cls, task_data):
        try:
            task = DataRetrievalTask.objects.get(id=task_data['task_id'])
        except DataRetrievalTask.DoesNotExist:
            logger.warning('No task for ID: %s', task_data['task_id'])

            return 'Invalid task ID!'

        if 'task_state' in task_data:
            cls.update_task_state(task, task_data['task_state'])

        if 's3_keys' in task_data:
            cls.create_datafiles(task, **task_data)

        if 'data_files' in task_data:
            cls.create_datafiles_with_metadata(task, **task_data)

        return 'Thanks!'

    @staticmethod
    def update_task_state(task, task_state):
        # TODO: change SUCCESS to SUCCEEDED on data_processing to match
        if task_state in ['SUCCESS', 'SUCCEEDED']:
            task.status = task.TASK_SUCCEEDED
            task.complete_time = datetime.now()
        elif task_state == 'QUEUED':
            task.status = task.TASK_QUEUED
        elif task_state == 'INITIATED':
            task.status = task.TASK_INITIATED
        # TODO: change FAILURE to FAILED on data_processing to match
        elif task_state in ['FAILURE', 'FAILED']:
            task.status = task.TASK_FAILED
            task.complete_time = datetime.now()

        task.save()

    @staticmethod
    def get_userdata_and_datafile_model(task):
        datafile_model = task.datafile_model.model_class()

        assert issubclass(datafile_model, BaseDataFile), (
            '%r is not a subclass of BaseDataFile' % datafile_model)

        userdata_model = (datafile_model._meta
                          .get_field_by_name('user_data')[0]
                          .rel.to)

        userdata, _ = userdata_model.objects.get_or_create(user=task.user)

        return userdata, datafile_model

    # pylint: disable=unused-argument
    def create_datafiles(self, task, s3_keys, **kwargs):
        userdata, datafile_model = self.get_userdata_and_datafile_model(task)

        for s3_key in s3_keys:
            data_file = datafile_model(userdata=userdata, task=task)

            data_file.file.name = s3_key
            data_file.save()

    def create_datafiles_with_metadata(self, task, data_files, **kwargs):
        userdata, datafile_model = self.get_userdata_and_datafile_model(task)

        for data_file in data_files:
            data_file_object = datafile_model(userdata=userdata, task=task)

            data_file_object.file.name = data_file['s3_key']

            metadata = data_file['metadata']

            data_file_object.description = metadata['description']
            data_file_object.tags = metadata['tags']

            # TODO: save other metadata fields here

            data_file_object.save()


class BaseDataRetrievalView(View):
    """
    Abstract base class for a view that starts a data retrieval task.

    Class attributes that need to be defined:
        datafile_model (attribute)
            App-specific, a subclass of BaseDataFile

    Class methods that need to be defined:
        get_app_task_params(self)
            Returns a dict with app-specific task parameters. These will be
            stored in the DataRetrievalTask.app_task_params, and will be sent
            to the data processing server wehn the task is run.
    """
    datafile_model = None
    redirect_url = reverse_lazy('my-member-research-data')
    message_error = 'Sorry, our data retrieval server seems to be down.'
    message_started = "Thanks! We've submitted this import task to our server."
{'fraction_non_alphanumeric': 0.08047283473516709, 'fraction_numerical': 0.0015912707433507615, 'mean_word_length': 2.7634730538922154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '727020', 'n_tokens_mistral': 2551, 'n_tokens_neox': 2488, 'n_words': 675}