<commit_before>#!/usr/bin/python
# ##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from __future__ import unicode_literals, print_function, absolute_import
import datetime
import logging

from django import forms
from django.forms import ModelChoiceField, BooleanField
from django.utils.datastructures import MultiValueDict
from django.db.models import Q

from amcat.scripts.forms import ModelMultipleChoiceFieldWithIdLabel
from amcat.models import CodingJob, CodingSchemaField, CodingSchema, Project, Codebook, Language
from amcat.models import Article, Sentence
from amcat.scripts.script import Script
from amcat.tools.table import table3
from amcat.scripts.output.xlsx import table_to_xlsx
from amcat.scripts.output.csv_output import table_to_csv
from amcat.tools.progress import NullMonitor

log = logging.getLogger(__name__)

import csv
import collections
import itertools
import json
import base64

from cStringIO import StringIO

FIELD_LABEL = "{label} {schemafield.label}"

AGGREGATABLE_FIELDS = {"medium"}

CODING_LEVEL_ARTICLE, CODING_LEVEL_SENTENCE, CODING_LEVEL_BOTH = range(3)
CODING_LEVELS = [
    (CODING_LEVEL_ARTICLE, "Article Codings"),
    (CODING_LEVEL_SENTENCE, "Sentence Codings"),
    (CODING_LEVEL_BOTH, "Article and Sentence Codings"),
]

ExportFormat = collections.namedtuple('ExportFormat', ["label", "function", "mimetype"])

EXPORT_FORMATS = (
    ExportFormat(label="csv", function=table_to_csv, mimetype="text/csv"),
    ExportFormat(label="xlsx", function=table_to_xlsx, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
    ExportFormat(label="json", function=lambda t: json.dumps(list(t.to_list())), mimetype=None),
)

_MetaField = collections.namedtuple("MetaField", ["object", "attr", "label"])

_METAFIELDS = [
    _MetaField("article", "id", "Article ID"),
    _MetaField("article", "headline", "Headline"),
    _MetaField("article", "byline", "Byline"),
    _MetaField("article", "medium", "Medium"),
    _MetaField("article", "medium_id", "Medium ID"),
    _MetaField("article", "date", "Date"),
    _MetaField("job", "id", "Codingjob ID"),
    _MetaField("job", "name", "Codingjob Name"),
    _MetaField("job", "coder", "Coder"),
    _MetaField("sentence", "id", "Sentence ID"),
    _MetaField("sentence", "parnr", "Paragraph"),
    _MetaField("sentence", "sentnr", "Sentence nr"),
    _MetaField("sentence", "sentence", "Sentence"),
    _MetaField("subsentence", "rangefrom", "Words from"),
    _MetaField("subsentence", "rangeto", "Words to"),
    _MetaField("subsentence", "subsentence", "Words coded"),
    _MetaField("coded_article", "comments", "Comments"),
    _MetaField("coded_article", "status", "Status"),
]


class CodingjobListForm(forms.Form):
    codingjobs = ModelMultipleChoiceFieldWithIdLabel(queryset=CodingJob.objects.all(), required=True)
    export_level = forms.ChoiceField(label="Level of codings to export", choices=CODING_LEVELS,
                                     initial=CODING_LEVEL_ARTICLE)

    def __init__(self, data=None, files=None, **kwargs):
        """
        Offers a form with a list of codingjobs. Raises a KeyError if keyword-
        argument project is not given.

        @param project: Restrict list of codingjobs to this project
        @type project: models.Project"""
        self.project = kwargs.pop("project", None)

        super(CodingjobListForm, self).__init__(data, files, **kwargs)
        if self.project:
            if isinstance(self.project, int):
                self.project = Project.objects.get(id=self.project)
            self.fields["codingjobs"].queryset = self.project.codingjob_set.all()
        self.data = self.data or MultiValueDict()


class CodingJobResultsForm(CodingjobListForm):
    """
    This is a dynamically rendered form, which consists of a static part (general
    options for exporting results) and a dynamic part. The dynamic part consists of
    field options being generated for each of the fields in the union of all fields
    in all codingjobs, depending on their type.
    """
    include_duplicates = forms.BooleanField(initial=False, required=False)
    export_format = forms.ChoiceField(tuple((c.label, c.label) for c in EXPORT_FORMATS))
    date_format = forms.CharField(initial="%Y-%m-%d %H-%M-%S", required=False)

    def __init__(self, data=None, files=None, **kwargs):
        """

        @param project: Restrict list of codingjobs to this project
        @type project: models.Project
        """
        codingjobs = kwargs.pop("codingjobs", None)
        export_level = kwargs.pop("export_level", None)
        super(CodingJobResultsForm, self).__init__(data, files, **kwargs)
        if codingjobs is None:  # is this necessary?
            data = self.data.getlist("codingjobs", codingjobs)
            codingjobs = self.fields["codingjobs"].clean(data)
        if export_level is None:
            export_level = int(self.fields["export_level"].clean(self.data['export_level']))

        # Hide fields from step (1)
        self.fields["codingjobs"].widget = forms.MultipleHiddenInput()
        self.fields["export_level"].widget = forms.HiddenInput()

        subsentences = CodingSchema.objects.filter(codingjobs_unit__in=codingjobs, subsentences=True).exists()


        # Add meta fields
        for field in _METAFIELDS:
            if field.object == "sentence" and export_level == CODING_LEVEL_ARTICLE: continue
            if field.object == "subsentence" and (export_level == CODING_LEVEL_ARTICLE or not subsentences): continue

            self.fields["meta_{field.object}_{field.attr}".format(**locals())] = forms.BooleanField(
                initial=True, required=False, label="Include {field.label}".format(**locals()))

        # Insert dynamic fields based on schemafields
        self.schemafields = _get_schemafields(codingjobs, export_level)
        self.fields.update(self.get_form_fields(self.schemafields))

    def get_form_fields(self, schemafields):
        """Returns a dict with all the fields needed to export this codingjob"""
        return dict(itertools.chain(*(self._get_form_fields(f) for f in schemafields)))

    def _get_form_fields(self, schemafield):
        """
        To prevent name collisions, this method also requires all schemafields, to check
        whether the current schemafield has a label which collides with a label of an
        other schemafield.
        """
        include_field = forms.BooleanField(
            label=FIELD_LABEL.format(label="Include", **locals()), initial=True, required=False,
        )

        # Show 'include this field' checkbox (for every field)
        prefix = _get_field_prefix(schemafield)
        yield ("{prefix}_included".format(**locals()), include_field)

        # Include field-specific form fields
        for id, field in schemafield.serialiser.get_export_fields():
            field.label = FIELD_LABEL.format(label="Export " + field.label, **locals())
            yield "{prefix}_{id}".format(**locals()), field

        # Include aggregation options for some metadata fields
        prefix = "aggregation"
        for field_name in AGGREGATABLE_FIELDS:
            # Aggregate field
            label = "Aggregate {field_name}, codebook".format(field_name=field_name)
            form_field = ModelChoiceField(queryset=Codebook.objects.all(), label=label, required=False)
            yield "{prefix}_{field_name}".format(**locals()), form_field

            # Aggregate language field
            label = "Aggregate {field_name}, language".format(field_name=field_name)
            form_field = ModelChoiceField(queryset=Language.objects.all(), label=label, required=True, initial=Language.objects.all()[0].id)
            yield "{prefix}_{field_name}_language".format(**locals()), form_field

            # Aggregate leave empty
            help_text = "If value is not found in codebook, leave field empty."
            label = "Aggregate {field_name}, include not found".format(field_name=field_name)
            form_field = BooleanField(initial=True, label=label, help_text=help_text)
            yield "{prefix}_{field_name}_default".format(**locals()), form_field


def _get_field_prefix(schemafield):
    return "schemafield_{schemafield.codingschema_id}_{schemafield.fieldnr}".format(**locals())


def _get_schemafields(codingjobs, level):
    unitfilter = Q(codingschema__codingjobs_unit__in=codingjobs)
    articlefilter = Q(codingschema__codingjobs_article__in=codingjobs)

    # Get fields based on given codingjobs and unit_codings setting
    fields = CodingSchemaField.objects.all()
    if level == CODING_LEVEL_ARTICLE:
        fields = fields.filter(articlefilter)<commit_msg>Allow unchecked 'leave empty' checkbox for aggregated codingjob export columns<commit_after>#!/usr/bin/python
# ##########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

from __future__ import unicode_literals, print_function, absolute_import
import datetime
import logging

from django import forms
from django.forms import ModelChoiceField, BooleanField
from django.utils.datastructures import MultiValueDict
from django.db.models import Q

from amcat.scripts.forms import ModelMultipleChoiceFieldWithIdLabel
from amcat.models import CodingJob, CodingSchemaField, CodingSchema, Project, Codebook, Language
from amcat.models import Article, Sentence
from amcat.scripts.script import Script
from amcat.tools.table import table3
from amcat.scripts.output.xlsx import table_to_xlsx
from amcat.scripts.output.csv_output import table_to_csv
from amcat.tools.progress import NullMonitor

log = logging.getLogger(__name__)

import csv
import collections
import itertools
import json
import base64

from cStringIO import StringIO

FIELD_LABEL = "{label} {schemafield.label}"

AGGREGATABLE_FIELDS = {"medium"}

CODING_LEVEL_ARTICLE, CODING_LEVEL_SENTENCE, CODING_LEVEL_BOTH = range(3)
CODING_LEVELS = [
    (CODING_LEVEL_ARTICLE, "Article Codings"),
    (CODING_LEVEL_SENTENCE, "Sentence Codings"),
    (CODING_LEVEL_BOTH, "Article and Sentence Codings"),
]

ExportFormat = collections.namedtuple('ExportFormat', ["label", "function", "mimetype"])

EXPORT_FORMATS = (
    ExportFormat(label="csv", function=table_to_csv, mimetype="text/csv"),
    ExportFormat(label="xlsx", function=table_to_xlsx, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
    ExportFormat(label="json", function=lambda t: json.dumps(list(t.to_list())), mimetype=None),
)

_MetaField = collections.namedtuple("MetaField", ["object", "attr", "label"])

_METAFIELDS = [
    _MetaField("article", "id", "Article ID"),
    _MetaField("article", "headline", "Headline"),
    _MetaField("article", "byline", "Byline"),
    _MetaField("article", "medium", "Medium"),
    _MetaField("article", "medium_id", "Medium ID"),
    _MetaField("article", "date", "Date"),
    _MetaField("job", "id", "Codingjob ID"),
    _MetaField("job", "name", "Codingjob Name"),
    _MetaField("job", "coder", "Coder"),
    _MetaField("sentence", "id", "Sentence ID"),
    _MetaField("sentence", "parnr", "Paragraph"),
    _MetaField("sentence", "sentnr", "Sentence nr"),
    _MetaField("sentence", "sentence", "Sentence"),
    _MetaField("subsentence", "rangefrom", "Words from"),
    _MetaField("subsentence", "rangeto", "Words to"),
    _MetaField("subsentence", "subsentence", "Words coded"),
    _MetaField("coded_article", "comments", "Comments"),
    _MetaField("coded_article", "status", "Status"),
]


class CodingjobListForm(forms.Form):
    codingjobs = ModelMultipleChoiceFieldWithIdLabel(queryset=CodingJob.objects.all(), required=True)
    export_level = forms.ChoiceField(label="Level of codings to export", choices=CODING_LEVELS,
                                     initial=CODING_LEVEL_ARTICLE)

    def __init__(self, data=None, files=None, **kwargs):
        """
        Offers a form with a list of codingjobs. Raises a KeyError if keyword-
        argument project is not given.

        @param project: Restrict list of codingjobs to this project
        @type project: models.Project"""
        self.project = kwargs.pop("project", None)

        super(CodingjobListForm, self).__init__(data, files, **kwargs)
        if self.project:
            if isinstance(self.project, int):
                self.project = Project.objects.get(id=self.project)
            self.fields["codingjobs"].queryset = self.project.codingjob_set.all()
        self.data = self.data or MultiValueDict()


class CodingJobResultsForm(CodingjobListForm):
    """
    This is a dynamically rendered form, which consists of a static part (general
    options for exporting results) and a dynamic part. The dynamic part consists of
    field options being generated for each of the fields in the union of all fields
    in all codingjobs, depending on their type.
    """
    include_duplicates = forms.BooleanField(initial=False, required=False)
    export_format = forms.ChoiceField(tuple((c.label, c.label) for c in EXPORT_FORMATS))
    date_format = forms.CharField(initial="%Y-%m-%d %H-%M-%S", required=False)

    def __init__(self, data=None, files=None, **kwargs):
        """

        @param project: Restrict list of codingjobs to this project
        @type project: models.Project
        """
        codingjobs = kwargs.pop("codingjobs", None)
        export_level = kwargs.pop("export_level", None)
        super(CodingJobResultsForm, self).__init__(data, files, **kwargs)
        if codingjobs is None:  # is this necessary?
            data = self.data.getlist("codingjobs", codingjobs)
            codingjobs = self.fields["codingjobs"].clean(data)
        if export_level is None:
            export_level = int(self.fields["export_level"].clean(self.data['export_level']))

        # Hide fields from step (1)
        self.fields["codingjobs"].widget = forms.MultipleHiddenInput()
        self.fields["export_level"].widget = forms.HiddenInput()

        subsentences = CodingSchema.objects.filter(codingjobs_unit__in=codingjobs, subsentences=True).exists()


        # Add meta fields
        for field in _METAFIELDS:
            if field.object == "sentence" and export_level == CODING_LEVEL_ARTICLE: continue
            if field.object == "subsentence" and (export_level == CODING_LEVEL_ARTICLE or not subsentences): continue

            self.fields["meta_{field.object}_{field.attr}".format(**locals())] = forms.BooleanField(
                initial=True, required=False, label="Include {field.label}".format(**locals()))

        # Insert dynamic fields based on schemafields
        self.schemafields = _get_schemafields(codingjobs, export_level)
        self.fields.update(self.get_form_fields(self.schemafields))

    def get_form_fields(self, schemafields):
        """Returns a dict with all the fields needed to export this codingjob"""
        return dict(itertools.chain(*(self._get_form_fields(f) for f in schemafields)))

    def _get_form_fields(self, schemafield):
        """
        To prevent name collisions, this method also requires all schemafields, to check
        whether the current schemafield has a label which collides with a label of an
        other schemafield.
        """
        include_field = forms.BooleanField(
            label=FIELD_LABEL.format(label="Include", **locals()), initial=True, required=False,
        )

        # Show 'include this field' checkbox (for every field)
        prefix = _get_field_prefix(schemafield)
        yield ("{prefix}_included".format(**locals()), include_field)

        # Include field-specific form fields
        for id, field in schemafield.serialiser.get_export_fields():
            field.label = FIELD_LABEL.format(label="Export " + field.label, **locals())
            yield "{prefix}_{id}".format(**locals()), field

        # Include aggregation options for some metadata fields
        prefix = "aggregation"
        for field_name in AGGREGATABLE_FIELDS:
            # Aggregate field
            label = "Aggregate {field_name}, codebook".format(field_name=field_name)
            form_field = ModelChoiceField(queryset=Codebook.objects.all(), label=label, required=False)
            yield "{prefix}_{field_name}".format(**locals()), form_field

            # Aggregate language field
            label = "Aggregate {field_name}, language".format(field_name=field_name)
            form_field = ModelChoiceField(queryset=Language.objects.all(), label=label, required=True, initial=Language.objects.all()[0].id)
            yield "{prefix}_{field_name}_language".format(**locals()), form_field

            # Aggregate leave empty
            help_text = "If value is not found in codebook, leave field empty."
            label = "Aggregate {field_name}, include not found".format(field_name=field_name)
            form_field = BooleanField(initial=True, label=label, required=False, help_text=help_text)
            yield "{prefix}_{field_name}_default".format(**locals()), form_field


def _get_field_prefix(schemafield):
    return "schemafield_{schemafield.codingschema_id}_{schemafield.fieldnr}".format(**locals())


def _get_schemafields(codingjobs, level):
    unitfilter = Q(codingschema__codingjobs_unit__in=codingjobs)
    articlefilter = Q(codingschema__codingjobs_article__in=codingjobs)

    # Get fields based on given codingjobs and unit_codings setting
    fields = CodingSchemaField.objects.all()
    if level == CODING_LEVEL_ARTICLE:
        fields = fields.filter(articlefilter)
{'fraction_non_alphanumeric': 0.11986659365822092, 'fraction_numerical': 0.0006968988003384937, 'mean_word_length': 3.284495628065686, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5891759', 'n_tokens_mistral': 5630, 'n_tokens_neox': 5362, 'n_words': 1441}