<commit_before># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
This module defines the :class:`AccessStatistics` model. It stores statistics
about the accessiblity status of papers related to any model
(currently :class:`.Researcher`, :class:`.Journal`, :class:`.Publisher`,
:class:`.Department`, :class:`.Institution` and :class:`.PaperWorld`).

As all these modules are spread in different apps and link to :class:`AccessStatistics`,
this model has been separated from the rest for dependency reasons.

The different states a paper can have are defined in :py:obj:`COMBINED_STATUS_CHOICES`
(where a human-readable description is given), :py:obj:`STATUS_QUERYSET_FILTER`
(where the corresponding QuerySet filters are defined) and in :py:func:`combined_status_for_instance`
which computes the status of a particular paper.
"""

from __future__ import unicode_literals

from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

#: Paper status (combined because it takes into account
#: both the publisher policy and the full text availability).
#:
#: If these states are changed, then the filters :py:obj:`STATUS_QUERYSET_FILTER`
#: have to be updated, as well as :py:func:`combined_status_for_instance`.
COMBINED_STATUS_CHOICES = [
    ('oa', _('Available from the publisher')),
    ('ok', _('Available from the author')),
    ('couldbe', _('Could be shared by the authors')),
    ('unk', _('Unknown/unclear sharing policy')),
    ('closed', _('Publisher forbids sharing')),
    ]

#: Helptext displayed when a paper logo is hovered
STATUS_CHOICES_HELPTEXT = {
    'oa': _('This paper is made freely available by the publisher.'),
    'ok': _('This paper is available in a repository.'),
    'couldbe': _('This paper was not found in any repository, but could be made available legally by the author.'),
    'unk': _('This paper was not found in any repository; the policy of its publisher is unknown or unclear.'),
    'closed': _('Distributing this paper is prohibited by the publisher'),
    }

#: Availability status choices
PDF_STATUS_CHOICES = [('OK', _('Available')),
                      ('NOK', _('Unavailable'))]

#: Filters associated to each combined status choice
#: These filters, once applied to a queryset of Papers,
#: select the papers in the given state.
STATUS_QUERYSET_FILTER = {
    'oa': lambda q: q.filter(oa_status='OA'),
    'ok': lambda q: q.filter(Q(pdf_url__isnull=False) & ~Q(oa_status='OA')),
    'couldbe': lambda q: q.filter(pdf_url__isnull=True, oa_status='OK'),
    'unk': lambda q: q.filter(pdf_url__isnull=True, oa_status='UNK'),
    'closed': lambda q: q.filter(pdf_url__isnull=True, oa_status='NOK'),
    }


def combined_status_for_instance(paper):
    """
    Computes the current combined status of a given paper.
    This is defined here so that the function can be easily updated when
    we change :py:obj:`STATUS_QUERYSET_FILTER` or :py:obj:`COMBINED_STATUS_CHOICES`.
    """
    if paper.oa_status == 'OA':
        return 'oa'
    elif paper.pdf_url:
        return 'ok'
    else:
        if paper.oa_status == 'OK':
            return 'couldbe'
        elif paper.oa_status == 'NOK':
            return 'closed'
    return 'unk'


class BareAccessStatistics(object):
    """
    A bare summary of the status of a set of publications
    """
    @classmethod
    def new(cls):
        """
        Creates an empty object (all counters set to 0)
        """
        s = cls()
        s.clear()
        return s

    def clear(self):
        """
        Sets all counters to zero
        """
        self.num_oa = 0
        self.num_ok = 0
        self.num_couldbe = 0
        self.num_unk = 0
        self.num_closed = 0
        self.num_tot = 0

    def add(self, other):
        """
        Adds the stats of `other` to self (sum of access statistics)
        """
        self.num_oa += other.num_oa
        self.num_ok += other.num_ok
        self.num_couldbe += other.num_couldbe
        self.num_unk += other.num_unk
        self.num_closed += other.num_closed
        self.num_tot += other.num_tot

    @classmethod
    def from_queryset(cls, qs):
        """
        This method fetches all the queryset.

        :param qs: a queryset of :class:`~papers.models.Paper` objects
        :returns: an object of the current class,
                  with the status of all papers in this queryset.
        """
        stats = cls.new()
        for paper in qs:
            stats.num_tot += 1
            attrname = 'num_'+paper.combined_status
            setattr(stats, attrname, getattr(stats, attrname) + 1)
        return stats

    @classmethod
    def from_dict(cls, d):
        stats = cls.new()
        for status, count in d.iteritems():
            stats.num_tot += count
            setattr(stats, 'num_'+status, count)
        return stats

    @classmethod
    def from_search_queryset(cls, qs):
        qs = qs.aggregations({
         "status": {"terms": {"field": "combined_status_exact"}},
        })
        aggregations = qs.get_aggregation_results()
        status = aggregations.get('status', {'buckets':[]})
        buckets = {
            bucket['key']: bucket['doc_count']
            for bucket in status['buckets']
        }
        return cls.from_dict(buckets)

    def check_values(self):<commit_msg>Fix an issue with search results<commit_after># -*- encoding: utf-8 -*-

# Dissemin: open access policy enforcement tool
# Copyright (C) 2014 <NAME>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#

"""
This module defines the :class:`AccessStatistics` model. It stores statistics
about the accessiblity status of papers related to any model
(currently :class:`.Researcher`, :class:`.Journal`, :class:`.Publisher`,
:class:`.Department`, :class:`.Institution` and :class:`.PaperWorld`).

As all these modules are spread in different apps and link to :class:`AccessStatistics`,
this model has been separated from the rest for dependency reasons.

The different states a paper can have are defined in :py:obj:`COMBINED_STATUS_CHOICES`
(where a human-readable description is given), :py:obj:`STATUS_QUERYSET_FILTER`
(where the corresponding QuerySet filters are defined) and in :py:func:`combined_status_for_instance`
which computes the status of a particular paper.
"""

from __future__ import unicode_literals

from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _

#: Paper status (combined because it takes into account
#: both the publisher policy and the full text availability).
#:
#: If these states are changed, then the filters :py:obj:`STATUS_QUERYSET_FILTER`
#: have to be updated, as well as :py:func:`combined_status_for_instance`.
COMBINED_STATUS_CHOICES = [
    ('oa', _('Available from the publisher')),
    ('ok', _('Available from the author')),
    ('couldbe', _('Could be shared by the authors')),
    ('unk', _('Unknown/unclear sharing policy')),
    ('closed', _('Publisher forbids sharing')),
    ]

#: Helptext displayed when a paper logo is hovered
STATUS_CHOICES_HELPTEXT = {
    'oa': _('This paper is made freely available by the publisher.'),
    'ok': _('This paper is available in a repository.'),
    'couldbe': _('This paper was not found in any repository, but could be made available legally by the author.'),
    'unk': _('This paper was not found in any repository; the policy of its publisher is unknown or unclear.'),
    'closed': _('Distributing this paper is prohibited by the publisher'),
    }

#: Availability status choices
PDF_STATUS_CHOICES = [('OK', _('Available')),
                      ('NOK', _('Unavailable'))]

#: Filters associated to each combined status choice
#: These filters, once applied to a queryset of Papers,
#: select the papers in the given state.
STATUS_QUERYSET_FILTER = {
    'oa': lambda q: q.filter(oa_status='OA'),
    'ok': lambda q: q.filter(Q(pdf_url__isnull=False) & ~Q(oa_status='OA')),
    'couldbe': lambda q: q.filter(pdf_url__isnull=True, oa_status='OK'),
    'unk': lambda q: q.filter(pdf_url__isnull=True, oa_status='UNK'),
    'closed': lambda q: q.filter(pdf_url__isnull=True, oa_status='NOK'),
    }


def combined_status_for_instance(paper):
    """
    Computes the current combined status of a given paper.
    This is defined here so that the function can be easily updated when
    we change :py:obj:`STATUS_QUERYSET_FILTER` or :py:obj:`COMBINED_STATUS_CHOICES`.
    """
    if paper.oa_status == 'OA':
        return 'oa'
    elif paper.pdf_url:
        return 'ok'
    else:
        if paper.oa_status == 'OK':
            return 'couldbe'
        elif paper.oa_status == 'NOK':
            return 'closed'
    return 'unk'


class BareAccessStatistics(object):
    """
    A bare summary of the status of a set of publications
    """
    @classmethod
    def new(cls):
        """
        Creates an empty object (all counters set to 0)
        """
        s = cls()
        s.clear()
        return s

    def clear(self):
        """
        Sets all counters to zero
        """
        self.num_oa = 0
        self.num_ok = 0
        self.num_couldbe = 0
        self.num_unk = 0
        self.num_closed = 0
        self.num_tot = 0

    def add(self, other):
        """
        Adds the stats of `other` to self (sum of access statistics)
        """
        self.num_oa += other.num_oa
        self.num_ok += other.num_ok
        self.num_couldbe += other.num_couldbe
        self.num_unk += other.num_unk
        self.num_closed += other.num_closed
        self.num_tot += other.num_tot

    @classmethod
    def from_queryset(cls, qs):
        """
        This method fetches all the queryset.

        :param qs: a queryset of :class:`~papers.models.Paper` objects
        :returns: an object of the current class,
                  with the status of all papers in this queryset.
        """
        stats = cls.new()
        for paper in qs:
            stats.num_tot += 1
            attrname = 'num_'+paper.combined_status
            setattr(stats, attrname, getattr(stats, attrname) + 1)
        return stats

    @classmethod
    def from_dict(cls, d):
        stats = cls.new()
        for status, count in d.iteritems():
            stats.num_tot += count
            setattr(stats, 'num_'+status, count)
        return stats

    @classmethod
    def from_search_queryset(cls, qs):
        qs = qs.aggregations({
         "status": {"terms": {"field": "combined_status_exact"}},
        })
        aggregations = qs.get_aggregation_results() or {}
        status = aggregations.get('status', {'buckets':[]})
        buckets = {
            bucket['key']: bucket['doc_count']
            for bucket in status['buckets']
        }
        return cls.from_dict(buckets)

    def check_values(self):
{'fraction_non_alphanumeric': 0.10552886572754662, 'fraction_numerical': 0.004234182884129957, 'mean_word_length': 3.0401315789473684, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1162878', 'n_tokens_mistral': 3658, 'n_tokens_neox': 3519, 'n_words': 1287}