<commit_before>##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
import pickle
import zlib
import hashlib

from amcat.models import Project, ArticleSet, TaskHandler, CodingJob
from amcat.models.authorisation import ROLE_PROJECT_METAREADER
from amcat.scripts.forms import SelectionForm
from amcat.tools.caching import cached
from amcat.tools.progress import ProgressMonitor
from django import forms
from django.contrib.auth.models import User
from django.core.cache import cache
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.urlresolvers import reverse
from django.http import QueryDict, HttpResponse
from navigator.views.scriptview import CeleryProgressUpdater

DOWNLOAD_HEADER = "Content-Disposition: attachment; "


def to_querydict(dict):
    """
    Converts a value created by querydict_dict back into a Django QueryDict value.
    """
    q = QueryDict("", mutable=True)
    for k, v in dict.items():
        q.setlist(k, v)
    q._mutable = False
    return q


class QueryActionForm(SelectionForm):
    output_type = forms.ChoiceField(choices=())
    download = forms.BooleanField(initial=False, required=False)

    def __init__(self, user, *args, **kwargs):
        super(QueryActionForm, self).__init__(*args, **kwargs)
        self.user = user


class QueryActionHandler(TaskHandler):
    @classmethod
    def serialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).serialise_arguments(arguments)
        user = arguments['user']
        arguments['user'] = None if user.is_anonymous() else user.id
        arguments['project'] = arguments['project'].id
        arguments['articlesets'] = [aset.id for aset in arguments['articlesets']]
        arguments['codingjobs'] = [cj.id for cj in arguments.get('codingjobs', [])]

        if arguments['data'] is not None and isinstance(arguments["data"], QueryDict):
            arguments['data'] = dict(arguments['data'].lists())

        return arguments

    @classmethod
    def deserialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).deserialise_arguments(arguments)
        arguments['user'] = None if arguments['user'] is None else User.objects.get(id=arguments['user'])
        arguments['project'] = Project.objects.get(id=arguments['project'])
        arguments['articlesets'] = ArticleSet.objects.filter(id__in=arguments['articlesets'])
        arguments['codingjobs'] = CodingJob.objects.filter(id__in=arguments.get('codingjobs', []))

        if arguments['data'] is not None:
            arguments['data'] = to_querydict(arguments['data'])

        return arguments

    @cached
    def get_query_action(self):
        query_action = self.task.get_class()(**self.task.get_arguments())
        query_action.get_form().full_clean()
        return query_action

    def run_task(self):
        query_action = self.get_query_action()
        updater = CeleryProgressUpdater(str(self.task.uuid))
        query_action.monitor.add_listener(updater.update)

        msg = "Task started, running {query_action.__class__.__name__}..".format(**locals())
        query_action.monitor.update(0, msg)

        try:
            form = query_action.get_form()
            query_action.before_run(form)
            return query_action.run(form)
        except Exception as e:
            raise

    def _get_content_type(self):
        """Returns content type of selected 'output_type'. This usually is a mimetype
        formatted like:

            application/json
            text/csv

        If clients want to add a specific 'meaning' to the data, the can add it using
        the separator comma:

            application/json,normalise
            text/csv,sparse

        In the latter cases only the part before the comma will be send back to the client
        as Content-Type header.
        """
        form = self.get_query_action().get_form()
        content_type = form.cleaned_data["output_type"]
        content_type = content_type.split(";")[0]
        return content_type

    def get_response(self):
        response = HttpResponse(content=self.get_result())
        response["Content-Disposition"] = "attachment"
        response["Content-Type"] = self._get_content_type()
        return response

    def get_redirect(self):
        return reverse("api:queryaction-index"), "No redirect"


class NotInCacheError(Exception):
    pass


class QueryAction(object):
    form_class = QueryActionForm
    task_handler = QueryActionHandler
    output_types = None
    required_role = ROLE_PROJECT_METAREADER
    ignore_cache_fields = ("output_type",)
    
    def __init__(self, user, project, articlesets, codingjobs=None, data=None):
        """
        @type project: amcat.models.Project
        @type articlesets: django.db.QuerySet
        @type data: NoneType, django.http.QueryDict
        """
        self.user = user
        self.project = project
        self.articlesets = articlesets
        self.codingjobs = codingjobs
        self.data = data
        self.monitor = ProgressMonitor()
        self.cache_hit = False

        assert(issubclass(self.form_class, SelectionForm))

    def _get_cache_key(self) -> str:
        """Returns a cache key (SHA256 hexdigest) of user id and form hash"""
        form_hash = self.get_form().get_hash(ignore_fields=self.ignore_cache_fields)
        return hashlib.sha256("{}|{}".format(self.user.id, form_hash).encode("ascii")).hexdigest()

    def serialize_cache_value(self, value):
        """Pickle then compress value"""
        return zlib.compress(pickle.dumps(value))

    def desearialize_cache_value(self, value):
        """Decompress then depickle value"""
        return pickle.loads(zlib.decompress(value))

    def get_cache(self):
        """Get cached value for this particular form+user. Raises NotInCacheError if not cached
        value is found."""
        cache_key = self._get_cache_key()
        value = cache.get(cache_key)
        if value is None:
            raise NotInCacheError("Cache value for {} was not found".format(cache_key))
        self.cache_hit = True
        return self.desearialize_cache_value(value)

    def set_cache(self, value):
        """Sets cache for this particular form+user to 'value'. The value is serialized
        with QueryAction.serialize_cache_value. Note that large values may not fit in
        memcached."""
        cache.set(self._get_cache_key(), self.serialize_cache_value(value))

    def get_form_kwargs(self, **kwargs):
        return dict({
            "data": self.data, "project": self.project,
            "articlesets": self.articlesets, "user": self.user,
            "codingjobs": self.codingjobs
        }, **kwargs)

    def get_form_class(self):
        return self.form_class

    def get_task_handler(self):
        return self.task_handler

    def target_project(self, form):
        return self.project
        
    @cached
    def get_form(self) -> SelectionForm:
        form = self.form_class(**self.get_form_kwargs())
        form.fields["output_type"].choices = self.output_types

        if len(self.output_types) <= 1:
            form.fields["output_type"].widget.attrs["type"] = "hidden"

        return form

    def before_run(self, form):
        self.check_permission(form)
        
    def run(self, form):
        """Needs to be overriden by subclass. Return value must be at least
        json serialisable, preferably bytes.

        @param form: cleaned form
        @type form: (subclass of) QueryActionForm"""
        raise NotImplementedError

    def run_delayed(self):
        """
        Put this task in celery queue. Returns a task handler.

        @raises: django.core.exceptions.ValidationError if form invalid
        @rtype: QueryActionHandler
        """
        form = self.get_form()

        if not form.is_valid():
            raise ValidationError(form._errors)
        self.check_permission(form)
            
        return self.get_task_handler().call(
            target_class=self.__class__, user=self.user,
            project=self.project, arguments={
                "user": self.user, "project": self.project,
                "data": self.data, "articlesets": self.articlesets,
                "codingjobs": self.codingjobs
            }
        )

    def check_permission(self, form):
        project = self.target_project(form)
        actual = project.get_role_id(self.user)
        if actual < self.required_role:
            raise PermissionDenied("User has no permission to perform {self.__class__.__name__}"
                                   " on project {project}".format(**locals()))
            
<commit_msg>Set cache timeout to 5 minutes<commit_after>##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################
import pickle
import zlib
import hashlib

from amcat.models import Project, ArticleSet, TaskHandler, CodingJob
from amcat.models.authorisation import ROLE_PROJECT_METAREADER
from amcat.scripts.forms import SelectionForm
from amcat.tools.caching import cached
from amcat.tools.progress import ProgressMonitor
from django import forms
from django.contrib.auth.models import User
from django.core.cache import cache
from django.core.exceptions import ValidationError, PermissionDenied
from django.core.urlresolvers import reverse
from django.http import QueryDict, HttpResponse
from navigator.views.scriptview import CeleryProgressUpdater

DOWNLOAD_HEADER = "Content-Disposition: attachment; "


def to_querydict(dict):
    """
    Converts a value created by querydict_dict back into a Django QueryDict value.
    """
    q = QueryDict("", mutable=True)
    for k, v in dict.items():
        q.setlist(k, v)
    q._mutable = False
    return q


class QueryActionForm(SelectionForm):
    output_type = forms.ChoiceField(choices=())
    download = forms.BooleanField(initial=False, required=False)

    def __init__(self, user, *args, **kwargs):
        super(QueryActionForm, self).__init__(*args, **kwargs)
        self.user = user


class QueryActionHandler(TaskHandler):
    @classmethod
    def serialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).serialise_arguments(arguments)
        user = arguments['user']
        arguments['user'] = None if user.is_anonymous() else user.id
        arguments['project'] = arguments['project'].id
        arguments['articlesets'] = [aset.id for aset in arguments['articlesets']]
        arguments['codingjobs'] = [cj.id for cj in arguments.get('codingjobs', [])]

        if arguments['data'] is not None and isinstance(arguments["data"], QueryDict):
            arguments['data'] = dict(arguments['data'].lists())

        return arguments

    @classmethod
    def deserialise_arguments(cls, arguments):
        arguments = super(QueryActionHandler, cls).deserialise_arguments(arguments)
        arguments['user'] = None if arguments['user'] is None else User.objects.get(id=arguments['user'])
        arguments['project'] = Project.objects.get(id=arguments['project'])
        arguments['articlesets'] = ArticleSet.objects.filter(id__in=arguments['articlesets'])
        arguments['codingjobs'] = CodingJob.objects.filter(id__in=arguments.get('codingjobs', []))

        if arguments['data'] is not None:
            arguments['data'] = to_querydict(arguments['data'])

        return arguments

    @cached
    def get_query_action(self):
        query_action = self.task.get_class()(**self.task.get_arguments())
        query_action.get_form().full_clean()
        return query_action

    def run_task(self):
        query_action = self.get_query_action()
        updater = CeleryProgressUpdater(str(self.task.uuid))
        query_action.monitor.add_listener(updater.update)

        msg = "Task started, running {query_action.__class__.__name__}..".format(**locals())
        query_action.monitor.update(0, msg)

        try:
            form = query_action.get_form()
            query_action.before_run(form)
            return query_action.run(form)
        except Exception as e:
            raise

    def _get_content_type(self):
        """Returns content type of selected 'output_type'. This usually is a mimetype
        formatted like:

            application/json
            text/csv

        If clients want to add a specific 'meaning' to the data, the can add it using
        the separator comma:

            application/json,normalise
            text/csv,sparse

        In the latter cases only the part before the comma will be send back to the client
        as Content-Type header.
        """
        form = self.get_query_action().get_form()
        content_type = form.cleaned_data["output_type"]
        content_type = content_type.split(";")[0]
        return content_type

    def get_response(self):
        response = HttpResponse(content=self.get_result())
        response["Content-Disposition"] = "attachment"
        response["Content-Type"] = self._get_content_type()
        return response

    def get_redirect(self):
        return reverse("api:queryaction-index"), "No redirect"


class NotInCacheError(Exception):
    pass


class QueryAction(object):
    form_class = QueryActionForm
    task_handler = QueryActionHandler
    output_types = None
    required_role = ROLE_PROJECT_METAREADER
    ignore_cache_fields = ("output_type",)
    
    def __init__(self, user, project, articlesets, codingjobs=None, data=None):
        """
        @type project: amcat.models.Project
        @type articlesets: django.db.QuerySet
        @type data: NoneType, django.http.QueryDict
        """
        self.user = user
        self.project = project
        self.articlesets = articlesets
        self.codingjobs = codingjobs
        self.data = data
        self.monitor = ProgressMonitor()
        self.cache_hit = False

        assert(issubclass(self.form_class, SelectionForm))

    def _get_cache_key(self) -> str:
        """Returns a cache key (SHA256 hexdigest) of user id and form hash"""
        form_hash = self.get_form().get_hash(ignore_fields=self.ignore_cache_fields)
        return hashlib.sha256("{}|{}".format(self.user.id, form_hash).encode("ascii")).hexdigest()

    def serialize_cache_value(self, value):
        """Pickle then compress value"""
        return zlib.compress(pickle.dumps(value))

    def desearialize_cache_value(self, value):
        """Decompress then depickle value"""
        return pickle.loads(zlib.decompress(value))

    def get_cache(self):
        """Get cached value for this particular form+user. Raises NotInCacheError if not cached
        value is found."""
        cache_key = self._get_cache_key()
        value = cache.get(cache_key)
        if value is None:
            raise NotInCacheError("Cache value for {} was not found".format(cache_key))
        self.cache_hit = True
        return self.desearialize_cache_value(value)

    def set_cache(self, value):
        """Sets cache for this particular form+user to 'value'. The value is serialized
        with QueryAction.serialize_cache_value. Note that large values may not fit in
        memcached."""
        cache.set(self._get_cache_key(), self.serialize_cache_value(value), timeout=5*60)

    def get_form_kwargs(self, **kwargs):
        return dict({
            "data": self.data, "project": self.project,
            "articlesets": self.articlesets, "user": self.user,
            "codingjobs": self.codingjobs
        }, **kwargs)

    def get_form_class(self):
        return self.form_class

    def get_task_handler(self):
        return self.task_handler

    def target_project(self, form):
        return self.project
        
    @cached
    def get_form(self) -> SelectionForm:
        form = self.form_class(**self.get_form_kwargs())
        form.fields["output_type"].choices = self.output_types

        if len(self.output_types) <= 1:
            form.fields["output_type"].widget.attrs["type"] = "hidden"

        return form

    def before_run(self, form):
        self.check_permission(form)
        
    def run(self, form):
        """Needs to be overriden by subclass. Return value must be at least
        json serialisable, preferably bytes.

        @param form: cleaned form
        @type form: (subclass of) QueryActionForm"""
        raise NotImplementedError

    def run_delayed(self):
        """
        Put this task in celery queue. Returns a task handler.

        @raises: django.core.exceptions.ValidationError if form invalid
        @rtype: QueryActionHandler
        """
        form = self.get_form()

        if not form.is_valid():
            raise ValidationError(form._errors)
        self.check_permission(form)
            
        return self.get_task_handler().call(
            target_class=self.__class__, user=self.user,
            project=self.project, arguments={
                "user": self.user, "project": self.project,
                "data": self.data, "articlesets": self.articlesets,
                "codingjobs": self.codingjobs
            }
        )

    def check_permission(self, form):
        project = self.target_project(form)
        actual = project.get_role_id(self.user)
        if actual < self.required_role:
            raise PermissionDenied("User has no permission to perform {self.__class__.__name__}"
                                   " on project {project}".format(**locals()))
            

{'fraction_non_alphanumeric': 0.10934612118487437, 'fraction_numerical': 0.0011988610819721264, 'mean_word_length': 2.5302415799682594, 'pattern_counts': {'":': 20, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1666861', 'n_tokens_mistral': 5368, 'n_tokens_neox': 5161, 'n_words': 1414}