<commit_before>from eve.io.base import DataLayer
from io import BytesIO
import urllib3
import json
import datetime
from superdesk.metadata.item import ITEM_TYPE, CONTENT_TYPE
from superdesk.utc import utc, utcnow
from eve_elastic.elastic import ElasticCursor
from superdesk.media.media_operations import process_file_from_stream, decode_metadata
from superdesk.media.renditions import generate_renditions, delete_file_on_error
from superdesk.errors import SuperdeskApiError
from superdesk.io.iptc import subject_codes
from flask import url_for
import urllib

import logging


urllib3.disable_warnings()

logger = logging.getLogger(__name__)


class AAPMMDatalayer(DataLayer):
    def __set_auth_cookie(self, app):
        if self._username is not None and self._password is not None:
            url = app.config['AAP_MM_SEARCH_URL'] + '/Users/login'
            values = {'username': self._username, 'password': self._password}
            r = self._http.urlopen('POST', url, headers={'Content-Type': 'application/json'}, body=json.dumps(values))
        else:
            url = app.config['AAP_MM_SEARCH_URL'] + '/Users/AnonymousToken'
            r = self._http.request('GET', url, redirect=False)

        self._headers = {'cookie': r.getheader('set-cookie'), 'Content-Type': 'application/json'}

    def set_credentials(self, user, password):
        if user != self._username and user != '' and password != self._password and password != '':
            self._username = user
            self._password = password
            self.__set_auth_cookie(self._app)

    def init_app(self, app):
        app.config.setdefault('AAP_MM_SEARCH_URL', 'https://one-api.aap.com.au/api/v3')
        app.config.setdefault('AAP_MM_CDN_URL', 'http://one-cdn.aap.com.au/Preview.mp4')
        self._app = app
        self._headers = None
        self._username = None
        self._password = None
        self._http = urllib3.PoolManager()

    def find(self, resource, req, lookup):
        """
        Called to execute a search against the AAP Mulitmedia API. It attempts to translate the search request
        passed in req to a suitable form for a search request against the API. It parses the response into a
        suitable ElasticCursor, the front end will never know.
        :param resource:
        :param req:
        :param lookup:
        :return:
        """
        if self._headers is None:
            self.__set_auth_cookie(self._app)

        url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/search'
        query_keywords = '*:*'
        if 'query' in req['query']['filtered']:
            query_keywords = req['query']['filtered']['query']['query_string']['query']
            query_keywords = query_keywords.replace('slugline:', 'objectname:')
            query_keywords = query_keywords.replace('description:', 'captionabstract:')

        fields = {}
        for filter in req.get('post_filter', {}).get('and', {}):
            # parse out the date range if possible
            if 'range' in filter:
                start = None
                end = None
                daterange = None
                if 'firstcreated' in filter.get('range', {}):
                    if 'gte' in filter['range']['firstcreated']:
                        start = filter['range']['firstcreated']['gte'][0:10]
                    if 'lte' in filter['range']['firstcreated']:
                        end = filter['range']['firstcreated']['lte'][0:10]
                # if there is a special start and no end it's one of the date buttons
                if start and not end:
                    if start == 'now-24H':
                        daterange = {'Dates': ['[NOW/HOUR-24HOURS TO NOW/HOUR]']}
                    if start == 'now-1w':
                        daterange = {'Dates': ['[NOW/DAY-7DAYS TO NOW/DAY]']}
                    if start == 'now-1M':
                        daterange = {'Dates': ['[NOW/DAY-1MONTH TO NOW/DAY]']}
                # we've got something but no daterange set above
                if (start or end) and not daterange:
                    daterange = {'DateRange': [{'Start': start, 'End': end}], 'DateCreatedFilter': 'true'}
                if daterange:
                    fields.update(daterange)

            if 'terms' in filter:
                if 'type' in filter.get('terms', {}):
                    fields.update({'MediaTypes': filter['terms']['type']})
                if 'credit' in filter.get('terms', {}):
                    fields.update({'Credits': filter['terms']['credit']})
                if 'anpa_category.name' in filter.get('terms', {}):
                    cat_list = []
                    for cat in filter['terms']['anpa_category.name']:
                        qcode = [key for key, value in subject_codes.items() if value == cat]
                        if qcode:
                            for code in qcode:
                                cat_list.append(code)
                        else:
                            cat_list.append(cat)
                    fields.update({'Categories': cat_list})

        size = int(req.get('size', '25')) if int(req.get('size', '25')) > 0 else 25
        query = {'Query': query_keywords, 'pageSize': str(size),
                 'pageNumber': str(int(req.get('from', '0')) // size + 1)}

        r = self._http.urlopen('POST', url + '?' + urllib.parse.urlencode(query),
                               body=json.dumps(fields), headers=self._headers)
        hits = self._parse_hits(json.loads(r.data.decode('UTF-8')))
        return ElasticCursor(docs=hits['docs'], hits={'hits': hits, 'aggregations': self._parse_aggregations(hits)})

    def _parse_doc(self, doc):
        new_doc = {}
        new_doc['_id'] = doc['AssetId']
        new_doc['guid'] = doc['AssetId']
        new_doc['headline'] = doc['Title']
        new_doc['description'] = doc['Description']
        new_doc['source'] = doc['Credit']
        if 'Source' in doc:
            new_doc['original_source'] = doc['Credit'] + '/' + str(doc.get('Source', ''))
        else:
            new_doc['original_source'] = doc['Credit']
        new_doc['versioncreated'] = self._datetime(doc['ModifiedDate'])<commit_msg>Rename variable in response to review<commit_after>from eve.io.base import DataLayer
from io import BytesIO
import urllib3
import json
import datetime
from superdesk.metadata.item import ITEM_TYPE, CONTENT_TYPE
from superdesk.utc import utc, utcnow
from eve_elastic.elastic import ElasticCursor
from superdesk.media.media_operations import process_file_from_stream, decode_metadata
from superdesk.media.renditions import generate_renditions, delete_file_on_error
from superdesk.errors import SuperdeskApiError
from superdesk.io.iptc import subject_codes
from flask import url_for
import urllib

import logging


urllib3.disable_warnings()

logger = logging.getLogger(__name__)


class AAPMMDatalayer(DataLayer):
    def __set_auth_cookie(self, app):
        if self._username is not None and self._password is not None:
            url = app.config['AAP_MM_SEARCH_URL'] + '/Users/login'
            values = {'username': self._username, 'password': self._password}
            r = self._http.urlopen('POST', url, headers={'Content-Type': 'application/json'}, body=json.dumps(values))
        else:
            url = app.config['AAP_MM_SEARCH_URL'] + '/Users/AnonymousToken'
            r = self._http.request('GET', url, redirect=False)

        self._headers = {'cookie': r.getheader('set-cookie'), 'Content-Type': 'application/json'}

    def set_credentials(self, user, password):
        if user != self._username and user != '' and password != self._password and password != '':
            self._username = user
            self._password = password
            self.__set_auth_cookie(self._app)

    def init_app(self, app):
        app.config.setdefault('AAP_MM_SEARCH_URL', 'https://one-api.aap.com.au/api/v3')
        app.config.setdefault('AAP_MM_CDN_URL', 'http://one-cdn.aap.com.au/Preview.mp4')
        self._app = app
        self._headers = None
        self._username = None
        self._password = <PASSWORD>
        self._http = urllib3.PoolManager()

    def find(self, resource, req, lookup):
        """
        Called to execute a search against the AAP Mulitmedia API. It attempts to translate the search request
        passed in req to a suitable form for a search request against the API. It parses the response into a
        suitable ElasticCursor, the front end will never know.
        :param resource:
        :param req:
        :param lookup:
        :return:
        """
        if self._headers is None:
            self.__set_auth_cookie(self._app)

        url = self._app.config['AAP_MM_SEARCH_URL'] + '/Assets/search'
        query_keywords = '*:*'
        if 'query' in req['query']['filtered']:
            query_keywords = req['query']['filtered']['query']['query_string']['query']
            query_keywords = query_keywords.replace('slugline:', 'objectname:')
            query_keywords = query_keywords.replace('description:', 'captionabstract:')

        fields = {}
        for criterion in req.get('post_filter', {}).get('and', {}):
            # parse out the date range if possible
            if 'range' in criterion:
                start = None
                end = None
                daterange = None
                if 'firstcreated' in criterion.get('range', {}):
                    if 'gte' in criterion['range']['firstcreated']:
                        start = criterion['range']['firstcreated']['gte'][0:10]
                    if 'lte' in criterion['range']['firstcreated']:
                        end = criterion['range']['firstcreated']['lte'][0:10]
                # if there is a special start and no end it's one of the date buttons
                if start and not end:
                    if start == 'now-24H':
                        daterange = {'Dates': ['[NOW/HOUR-24HOURS TO NOW/HOUR]']}
                    if start == 'now-1w':
                        daterange = {'Dates': ['[NOW/DAY-7DAYS TO NOW/DAY]']}
                    if start == 'now-1M':
                        daterange = {'Dates': ['[NOW/DAY-1MONTH TO NOW/DAY]']}
                # we've got something but no daterange set above
                if (start or end) and not daterange:
                    daterange = {'DateRange': [{'Start': start, 'End': end}], 'DateCreatedFilter': 'true'}
                if daterange:
                    fields.update(daterange)

            if 'terms' in criterion:
                if 'type' in criterion.get('terms', {}):
                    fields.update({'MediaTypes': criterion['terms']['type']})
                if 'credit' in criterion.get('terms', {}):
                    fields.update({'Credits': criterion['terms']['credit']})
                if 'anpa_category.name' in criterion.get('terms', {}):
                    cat_list = []
                    for cat in criterion['terms']['anpa_category.name']:
                        qcode = [key for key, value in subject_codes.items() if value == cat]
                        if qcode:
                            for code in qcode:
                                cat_list.append(code)
                        else:
                            cat_list.append(cat)
                    fields.update({'Categories': cat_list})

        size = int(req.get('size', '25')) if int(req.get('size', '25')) > 0 else 25
        query = {'Query': query_keywords, 'pageSize': str(size),
                 'pageNumber': str(int(req.get('from', '0')) // size + 1)}

        r = self._http.urlopen('POST', url + '?' + urllib.parse.urlencode(query),
                               body=json.dumps(fields), headers=self._headers)
        hits = self._parse_hits(json.loads(r.data.decode('UTF-8')))
        return ElasticCursor(docs=hits['docs'], hits={'hits': hits, 'aggregations': self._parse_aggregations(hits)})

    def _parse_doc(self, doc):
        new_doc = {}
        new_doc['_id'] = doc['AssetId']
        new_doc['guid'] = doc['AssetId']
        new_doc['headline'] = doc['Title']
        new_doc['description'] = doc['Description']
        new_doc['source'] = doc['Credit']
        if 'Source' in doc:
            new_doc['original_source'] = doc['Credit'] + '/' + str(doc.get('Source', ''))
        else:
            new_doc['original_source'] = doc['Credit']
        new_doc['versioncreated'] = self._datetime(doc['ModifiedDate'])
{'fraction_non_alphanumeric': 0.13212393642639267, 'fraction_numerical': 0.004655642960346765, 'mean_word_length': 2.1815628192032688, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4654372', 'n_tokens_mistral': 3674, 'n_tokens_neox': 3475, 'n_words': 858}