<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014, 2015 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
import flask
from superdesk import get_resource_service
import superdesk
from superdesk.errors import SuperdeskApiError
from superdesk.metadata.item import not_analyzed, ITEM_STATE, PUBLISH_STATES
from superdesk.metadata.utils import aggregations, get_elastic_highlight_query
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.utc import utcnow

from bson.objectid import ObjectId
from eve.utils import ParsedRequest, config
from flask import current_app as app, request

from apps.archive.archive import SOURCE as ARCHIVE
from apps.archive.common import handle_existing_data, item_schema
from superdesk.publish.publish_queue import PUBLISHED_IN_PACKAGE
from apps.content import push_content_notification
from flask_babel import _

logger = logging.getLogger(__name__)

PUBLISHED = "published"
LAST_PUBLISHED_VERSION = "last_published_version"
QUEUE_STATE = "queue_state"
ERROR_MESSAGE = "error_message"
queue_states = ["pending", "in_progress", "queued", "queued_not_transmitted", "error"]
PUBLISH_STATE = namedtuple("PUBLISH_STATE", ["PENDING", "IN_PROGRESS", "QUEUED", "QUEUED_NOT_TRANSMITTED", "ERROR"])(
    *queue_states
)

published_item_fields = {
    "item_id": {"type": "string", "mapping": not_analyzed},
    "publish_state": {"type": "string"},
    # last_published_version field is set to true for last published version of the item in the published collection
    # and for the older version is set to false. This field is used to display the last version of the digital copy
    # in the published view.
    LAST_PUBLISHED_VERSION: {"type": "boolean", "default": True},
    QUEUE_STATE: {
        "type": "string",
        "default": "pending",
        "allowed": queue_states,
    },
    ERROR_MESSAGE: {"type": "string"},
    "is_take_item": {  # deprecated
        "type": "boolean",
        "default": False,
    },
    "digital_item_id": {"type": "string"},  # deprecated
    "publish_sequence_no": {"type": "integer", "readonly": True},
    "last_queue_event": {"type": "datetime"},
    # item is moved to legal. True if moved else false.
    "moved_to_legal": {"type": "boolean", "default": False},
    # if item is published as part of a package this will
    # be set to package id so transmitter can handle such
    # items differently
    PUBLISHED_IN_PACKAGE: {
        "type": "string",
    },
    "_current_version": {  # must be set explicitly, there is no versioning
        "type": "integer",
    },
}


def get_content_filter():
    """Filter out content of stages not visible to current user (if any).

    :return:
    """
    user = getattr(flask.g, "user", None)
    if user:
        if "invisible_stages" in user:
            stages = user.get("invisible_stages")
        else:
            stages = get_resource_service("users").get_invisible_stages_ids(user.get("_id"))

        if stages:
            return {"bool": {"must_not": {"terms": {"task.stage": stages}}}}


class PublishedItemResource(Resource):
    datasource = {
        "search_backend": "elastic",
        "aggregations": aggregations,
        "es_highlight": get_elastic_highlight_query,
        "default_sort": [("_updated", -1)],
        "elastic_filter_callback": get_content_filter,
        "projection": {
            "old_version": 0,
            "last_version": 0,
        },
    }

    schema = item_schema(published_item_fields)
    etag_ignore_fields = [config.ID_FIELD, "highlights", "item_id", LAST_PUBLISHED_VERSION, "moved_to_legal"]

    privileges = {"POST": "publish_queue", "PATCH": "publish_queue"}
    item_methods = ["GET", "PATCH"]
    additional_lookup = {"url": 'regex("[\w,.:-]+")', "field": "item_id"}


class PublishedItemService(BaseService):
    """
    PublishedItemService class is the base class for ArchivedService.
    """

    SEQ_KEY_NAME = "published_item_sequence_no"

    def on_fetched(self, docs):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items(docs[config.ITEMS])

    def on_fetched_item(self, doc):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items([doc])

    def on_create(self, docs):
        """Runs on create.

        An article can be published multiple times in its lifetime. So, it's necessary to preserve the _id which comes
        from archive collection. Also, sets the expiry on the published item and removes the lock information.
        """

        for doc in docs:<commit_msg>add mongo indexes for published items (#2275)<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014, 2015 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from collections import namedtuple
import json
import logging
import flask
from superdesk import get_resource_service
import superdesk
from superdesk.errors import SuperdeskApiError
from superdesk.metadata.item import not_analyzed, ITEM_STATE, PUBLISH_STATES
from superdesk.metadata.utils import aggregations, get_elastic_highlight_query
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.utc import utcnow

from bson.objectid import ObjectId
from eve.utils import ParsedRequest, config
from flask import current_app as app, request

from apps.archive.archive import SOURCE as ARCHIVE
from apps.archive.common import handle_existing_data, item_schema
from superdesk.publish.publish_queue import PUBLISHED_IN_PACKAGE
from apps.content import push_content_notification
from flask_babel import _

logger = logging.getLogger(__name__)

PUBLISHED = "published"
LAST_PUBLISHED_VERSION = "last_published_version"
QUEUE_STATE = "queue_state"
ERROR_MESSAGE = "error_message"
queue_states = ["pending", "in_progress", "queued", "queued_not_transmitted", "error"]
PUBLISH_STATE = namedtuple("PUBLISH_STATE", ["PENDING", "IN_PROGRESS", "QUEUED", "QUEUED_NOT_TRANSMITTED", "ERROR"])(
    *queue_states
)

published_item_fields = {
    "item_id": {"type": "string", "mapping": not_analyzed},
    "publish_state": {"type": "string"},
    # last_published_version field is set to true for last published version of the item in the published collection
    # and for the older version is set to false. This field is used to display the last version of the digital copy
    # in the published view.
    LAST_PUBLISHED_VERSION: {"type": "boolean", "default": True},
    QUEUE_STATE: {
        "type": "string",
        "default": "pending",
        "allowed": queue_states,
    },
    ERROR_MESSAGE: {"type": "string"},
    "is_take_item": {  # deprecated
        "type": "boolean",
        "default": False,
    },
    "digital_item_id": {"type": "string"},  # deprecated
    "publish_sequence_no": {"type": "integer", "readonly": True},
    "last_queue_event": {"type": "datetime"},
    # item is moved to legal. True if moved else false.
    "moved_to_legal": {"type": "boolean", "default": False},
    # if item is published as part of a package this will
    # be set to package id so transmitter can handle such
    # items differently
    PUBLISHED_IN_PACKAGE: {
        "type": "string",
    },
    "_current_version": {  # must be set explicitly, there is no versioning
        "type": "integer",
    },
}


def get_content_filter():
    """Filter out content of stages not visible to current user (if any).

    :return:
    """
    user = getattr(flask.g, "user", None)
    if user:
        if "invisible_stages" in user:
            stages = user.get("invisible_stages")
        else:
            stages = get_resource_service("users").get_invisible_stages_ids(user.get("_id"))

        if stages:
            return {"bool": {"must_not": {"terms": {"task.stage": stages}}}}


class PublishedItemResource(Resource):
    datasource = {
        "search_backend": "elastic",
        "aggregations": aggregations,
        "es_highlight": get_elastic_highlight_query,
        "default_sort": [("_updated", -1)],
        "elastic_filter_callback": get_content_filter,
        "projection": {
            "old_version": 0,
            "last_version": 0,
        },
    }

    schema = item_schema(published_item_fields)
    etag_ignore_fields = [config.ID_FIELD, "highlights", "item_id", LAST_PUBLISHED_VERSION, "moved_to_legal"]

    privileges = {"POST": "publish_queue", "PATCH": "publish_queue"}
    item_methods = ["GET", "PATCH"]
    additional_lookup = {"url": 'regex("[\w,.:-]+")', "field": "item_id"}

    mongo_indexes = {
        "guid_1": ([("guid", 1)], {"background": True}),
        "item_id_1": ([("item_id", 1)], {"background": True}),
    }


class PublishedItemService(BaseService):
    """
    PublishedItemService class is the base class for ArchivedService.
    """

    SEQ_KEY_NAME = "published_item_sequence_no"

    def on_fetched(self, docs):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items(docs[config.ITEMS])

    def on_fetched_item(self, doc):
        """
        Overriding this to enhance the published article with the one in archive collection
        """

        self.enhance_with_archive_items([doc])

    def on_create(self, docs):
        """Runs on create.

        An article can be published multiple times in its lifetime. So, it's necessary to preserve the _id which comes
        from archive collection. Also, sets the expiry on the published item and removes the lock information.
        """

        for doc in docs:
{'fraction_non_alphanumeric': 0.11000883999607111, 'fraction_numerical': 0.003928887142716825, 'mean_word_length': 3.674931129476584, 'pattern_counts': {'":': 90, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5875096', 'n_tokens_mistral': 3120, 'n_tokens_neox': 2935, 'n_words': 916}