<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import json
from eve.utils import ParsedRequest
from superdesk import get_resource_service
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.errors import SuperdeskApiError

logger = logging.getLogger(__name__)


class PublishFilterResource(Resource):
    schema = {
        'publish_filter': {
            'type': 'list',
            'schema': {
                'type': 'dict',
                'schema': {
                    'expression': {
                        'type': 'dict',
                        'schema': {
                            'fc': {
                                'type': 'list',
                                'schema': Resource.rel('filter_conditions', True)
                            },
                            'pf': {
                                'type': 'list',
                                'schema': Resource.rel('publish_filters', True)
                            }
                        }
                    }
                }
            }
        },
        'name': {
            'type': 'string',
            'nullable': False,
            'iunique': True
        },
        'is_global': {
            'type': 'boolean',
            'default': False
        }
    }

    additional_lookup = {
        'url': 'regex("[\w,.:-]+")',
        'field': 'name'
    }

    datasource = {'default_sort': [('_created', -1)]}
    privileges = {'POST': 'publish_filters',
                  'PATCH': 'publish_filters',
                  'DELETE': 'publish_filters'}


class PublishFilterService(BaseService):
    def get(self, req, lookup):
        if req is None:
            req = ParsedRequest()
        if req.args and req.args.get('is_global'):
            lookup = {'is_global': True}
        return self.backend.get(self.datasource, req=req, lookup=lookup)

    def update(self, id, updates, original):
        publish_filter = dict(original)
        publish_filter.update(updates)
        self._validate_no_circular_reference(publish_filter, publish_filter['_id'])
        super().update(id, updates, original)

    def delete(self, lookup):
        referenced_filters = self._get_publish_filters_by_publish_filter(lookup.get('_id'))
        if referenced_filters.count() > 0:
            references = ','.join([pf['name'] for pf in referenced_filters])
            raise SuperdeskApiError.badRequestError('Publish filter has been referenced in {}'.format(references))
        return super().delete(lookup)

    def _get_publish_filters_by_publish_filter(self, publish_filter_id):
        lookup = {'publish_filter.expression.pf': {'$in': [publish_filter_id]}}
        publish_filters = get_resource_service('publish_filters').get(req=None, lookup=lookup)
        return publish_filters

    def _get_publish_filters_by_filter_condition(self, filter_condition_id):
        lookup = {'publish_filter.expression.fc': {'$in': [filter_condition_id]}}
        publish_filters = super().get(req=None, lookup=lookup)
        all_publish_filters = self._get_referenced_publish_filters(list(publish_filters), None)
        return all_publish_filters

    def _get_referenced_publish_filters(self, publish_filters, pf_list):
        if not pf_list:
            pf_list = []

        for pf in publish_filters:
            pf_list.append(pf)
            references = list(self._get_publish_filters_by_publish_filter(pf['_id']))
            if references and len(references) > 0:
                return self._get_referenced_publish_filters(references, pf_list)
        return pf_list

    def _validate_no_circular_reference(self, publish_filter, filter_id):
        for expression in publish_filter.get('publish_filter', []):
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    if f == filter_id:
                        raise SuperdeskApiError.badRequestError('Circular dependency error in publish filters:{}'
                                                                .format(current_filter['name']))
                    self._validate_no_circular_reference(current_filter, filter_id)

    def build_mongo_query(self, doc):
        filter_condition_service = get_resource_service('filter_conditions')
        expressions = []
        for expression in doc.get('publish_filter', []):
            filter_conditions = []
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    current_filter = filter_condition_service.find_one(req=None, _id=f)
                    mongo_query = filter_condition_service.get_mongo_query(current_filter)
                    filter_conditions.append(mongo_query)
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    mongo_query = self.build_mongo_query(current_filter)
                    filter_conditions.append(mongo_query)

            if len(filter_conditions) > 1:
                expressions.append({'$and': filter_conditions})
            else:
                expressions.extend(filter_conditions)

        if len(expressions) > 1:
            return {'$or': expressions}
        else:
            return expressions[0]

    def build_elastic_query(self, doc):
        return {'query': {'filtered': {'query': self._get_elastic_query(doc)}}}

    def build_elastic_not_filter(self, doc):
        return {'query': {'filtered': {'filter': {'not': {'filter': self._get_elastic_query(doc)}}}}}

    def _get_elastic_query(self, doc):
        expressions = {'should': []}
        filter_condition_service = get_resource_service('filter_conditions')
        for expression in doc.get('publish_filter', []):
            filter_conditions = {'must': []}
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    current_filter = filter_condition_service.find_one(req=None, _id=f)
                    elastic_query = filter_condition_service.get_elastic_query(current_filter)
                    filter_conditions['must'].append(elastic_query)
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    elastic_query = self._get_elastic_query(current_filter)
                    filter_conditions['must'].append(elastic_query)

            expressions['should'].append({'bool': filter_conditions})
        return {'bool': expressions}

    def does_match(self, publish_filter, article):
        filter_condition_service = get_resource_service('filter_conditions')
        expressions = []
        for expression in publish_filter.get('publish_filter', []):
            filter_conditions = []
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    filter_condition = filter_condition_service.find_one(req=None, _id=f)
                    filter_conditions.append(filter_condition_service.does_match(filter_condition, article))
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    filter_conditions.append(self.does_match(current_filter, article))

            expressions.append(all(filter_conditions))
        return any(expressions)


class PublishFilterTestResource(Resource):
    item_url = 'regex("[\w,.:_-]+")'
    endpoint_name = 'publish_filter_tests'
    schema = {
        'filter_id': {'type': 'string'},
        'article_id': {'type': 'string'},<commit_msg>[SD-2764] - Error in testing filters for the not-matching articles<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

import logging
import json
from eve.utils import ParsedRequest
from superdesk import get_resource_service
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.errors import SuperdeskApiError

logger = logging.getLogger(__name__)


class PublishFilterResource(Resource):
    schema = {
        'publish_filter': {
            'type': 'list',
            'schema': {
                'type': 'dict',
                'schema': {
                    'expression': {
                        'type': 'dict',
                        'schema': {
                            'fc': {
                                'type': 'list',
                                'schema': Resource.rel('filter_conditions', True)
                            },
                            'pf': {
                                'type': 'list',
                                'schema': Resource.rel('publish_filters', True)
                            }
                        }
                    }
                }
            }
        },
        'name': {
            'type': 'string',
            'nullable': False,
            'iunique': True
        },
        'is_global': {
            'type': 'boolean',
            'default': False
        }
    }

    additional_lookup = {
        'url': 'regex("[\w,.:-]+")',
        'field': 'name'
    }

    datasource = {'default_sort': [('_created', -1)]}
    privileges = {'POST': 'publish_filters',
                  'PATCH': 'publish_filters',
                  'DELETE': 'publish_filters'}


class PublishFilterService(BaseService):
    def get(self, req, lookup):
        if req is None:
            req = ParsedRequest()
        if req.args and req.args.get('is_global'):
            lookup = {'is_global': True}
        return self.backend.get(self.datasource, req=req, lookup=lookup)

    def update(self, id, updates, original):
        publish_filter = dict(original)
        publish_filter.update(updates)
        self._validate_no_circular_reference(publish_filter, publish_filter['_id'])
        super().update(id, updates, original)

    def delete(self, lookup):
        referenced_filters = self._get_publish_filters_by_publish_filter(lookup.get('_id'))
        if referenced_filters.count() > 0:
            references = ','.join([pf['name'] for pf in referenced_filters])
            raise SuperdeskApiError.badRequestError('Publish filter has been referenced in {}'.format(references))
        return super().delete(lookup)

    def _get_publish_filters_by_publish_filter(self, publish_filter_id):
        lookup = {'publish_filter.expression.pf': {'$in': [publish_filter_id]}}
        publish_filters = get_resource_service('publish_filters').get(req=None, lookup=lookup)
        return publish_filters

    def _get_publish_filters_by_filter_condition(self, filter_condition_id):
        lookup = {'publish_filter.expression.fc': {'$in': [filter_condition_id]}}
        publish_filters = super().get(req=None, lookup=lookup)
        all_publish_filters = self._get_referenced_publish_filters(list(publish_filters), None)
        return all_publish_filters

    def _get_referenced_publish_filters(self, publish_filters, pf_list):
        if not pf_list:
            pf_list = []

        for pf in publish_filters:
            pf_list.append(pf)
            references = list(self._get_publish_filters_by_publish_filter(pf['_id']))
            if references and len(references) > 0:
                return self._get_referenced_publish_filters(references, pf_list)
        return pf_list

    def _validate_no_circular_reference(self, publish_filter, filter_id):
        for expression in publish_filter.get('publish_filter', []):
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    if f == filter_id:
                        raise SuperdeskApiError.badRequestError('Circular dependency error in publish filters:{}'
                                                                .format(current_filter['name']))
                    self._validate_no_circular_reference(current_filter, filter_id)

    def build_mongo_query(self, doc):
        filter_condition_service = get_resource_service('filter_conditions')
        expressions = []
        for expression in doc.get('publish_filter', []):
            filter_conditions = []
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    current_filter = filter_condition_service.find_one(req=None, _id=f)
                    mongo_query = filter_condition_service.get_mongo_query(current_filter)
                    filter_conditions.append(mongo_query)
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    mongo_query = self.build_mongo_query(current_filter)
                    filter_conditions.append(mongo_query)

            if len(filter_conditions) > 1:
                expressions.append({'$and': filter_conditions})
            else:
                expressions.extend(filter_conditions)

        if len(expressions) > 1:
            return {'$or': expressions}
        else:
            return expressions[0]

    def build_elastic_query(self, doc):
        return {'query': {'filtered': {'query': self._get_elastic_query(doc)}}}

    def build_elastic_not_filter(self, doc):
        return {'query': {'filtered': {'query': self._get_elastic_query(doc, matching=False)}}}

    def _get_elastic_query(self, doc, matching=True):
        expressions_list = []
        if matching:
            expressions = {'should': expressions_list}
        else:
            expressions = {'must_not': expressions_list}

        filter_condition_service = get_resource_service('filter_conditions')
        for expression in doc.get('publish_filter', []):
            filter_conditions = {'must': []}
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    current_filter = filter_condition_service.find_one(req=None, _id=f)
                    elastic_query = filter_condition_service.get_elastic_query(current_filter)
                    filter_conditions['must'].append(elastic_query)
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    elastic_query = self._get_elastic_query(current_filter)
                    filter_conditions['must'].append(elastic_query)

            expressions_list.append({'bool': filter_conditions})
        return {'bool': expressions}

    def does_match(self, publish_filter, article):
        filter_condition_service = get_resource_service('filter_conditions')
        expressions = []
        for expression in publish_filter.get('publish_filter', []):
            filter_conditions = []
            if 'fc' in expression.get('expression', {}):
                for f in expression['expression']['fc']:
                    filter_condition = filter_condition_service.find_one(req=None, _id=f)
                    filter_conditions.append(filter_condition_service.does_match(filter_condition, article))
            if 'pf' in expression.get('expression', {}):
                for f in expression['expression']['pf']:
                    current_filter = super().find_one(req=None, _id=f)
                    filter_conditions.append(self.does_match(current_filter, article))

            expressions.append(all(filter_conditions))
        return any(expressions)


class PublishFilterTestResource(Resource):
    item_url = 'regex("[\w,.:_-]+")'
    endpoint_name = 'publish_filter_tests'
    schema = {
        'filter_id': {'type': 'string'},
        'article_id': {'type': 'string'},
{'fraction_non_alphanumeric': 0.11412848426845316, 'fraction_numerical': 0.002033736092834071, 'mean_word_length': 2.2376065065840436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1884533', 'n_tokens_mistral': 4500, 'n_tokens_neox': 4336, 'n_words': 885}