<commit_before>import json
import hmac
import logging
import tempfile
import urllib.parse
import uuid
from flask import abort
import requests
from bson import ObjectId
from hashlib import sha1
from eve.io.mongo import MongoJSONEncoder
from requests.exceptions import RequestException, ConnectionError, ConnectTimeout
from requests.packages.urllib3.exceptions import MaxRetryError
from superdesk import get_resource_service
from superdesk.metadata.item import ITEM_TYPE, CONTENT_TYPE
from apps.auth import SuperdeskTokenAuth
from .exceptions import APIConnectionError, DownloadError
from werkzeug.datastructures import FileStorage

logger = logging.getLogger('superdesk')


def generate_api_key():
    """Generate a new API Key."""
    random_uuid = uuid.uuid4()
    return hmac.new(random_uuid.bytes, digestmod=sha1).hexdigest()


def trailing_slash(url):
    """Add trailing slash to url."""
    if not url.endswith('/'):
        url = '{}/'.format(url)
    return url


def cast_to_object_id(doc, fields):
    """Cast provided document fields to ObjectId."""
    for field in fields:
        value = doc.get(field)
        if not value:
            continue

        if not ObjectId.is_valid(value):
            logger.warning('Field "{}" value "{}" is not a valid ObjectId')
            continue

        doc[field] = ObjectId(value)


def send_api_request(api_url, api_key, method='GET', args=None, data=None, json_loads=True, timeout=5):
    """Utility function to send http requests for consumer/producer api endpoints."""
    if data:
        data = json.dumps(data, cls=MongoJSONEncoder)

    session = requests.Session()
    session.trust_env = False
    adapter = requests.adapters.HTTPAdapter(max_retries=0)
    session.mount('http://', adapter)
    session.mount('https://', adapter)

    logger.debug('API {} request to {} with params={} and data={}'.format(method, api_url, args, data))
    headers = {
        'Content-Type': 'application/json'
    }
    if api_key:
        headers['Authorization'] = api_key

    try:
        response = requests.request(method, api_url, headers=headers, params=args, data=data, timeout=timeout)
    except (ConnectionError, ConnectTimeout, RequestException, MaxRetryError):
        raise APIConnectionError('Unable to connect to api_url "{}".'.format(api_url))

    logger.debug('API {} request to {} - response: {} {}'.format(
        method, api_url, response.status_code, response.content
    ))

    if not json_loads:
        return response
    else:
        return response.json()


def fetch_url(url, timeout=5):
    """Fetch url using python-requests and save data to temporary file."""
    try:
        response = requests.get(url, timeout=timeout)
    except (ConnectionError, RequestException, MaxRetryError):
        raise DownloadError('Unable to download url: "{}"'.format(url))
    fd = tempfile.NamedTemporaryFile()
    for chunk in response.iter_content(chunk_size=1024):
        if chunk:
            fd.write(chunk)
    fd.seek(0)
    return fd


def blueprint_superdesk_token_auth():
    auth = SuperdeskTokenAuth()
    authorized = auth.authorized(allowed_roles=[], resource='producers', method='GET')
    if not authorized:
        return abort(401, 'Authorization failed.')


def extract_post_items_data(original_doc):
    """Extract blog post items."""
    items_service = get_resource_service('items')
    user_service = get_resource_service('users')
    item_type = original_doc.get(ITEM_TYPE, '')
    if item_type != CONTENT_TYPE.COMPOSITE:
        raise NotImplementedError('Post item_type "{}" not supported.'.format(item_type))

    items = []
    needed_fields = ("avatar", "avatar_renditions", "byline",
                     "display_name", "email", "first_name",
                     "last_name", "picture_url", "sign_off",
                     "username", "_id", "_created", "_updated")

    for group in original_doc['groups']:
        if group['id'] == 'main':
            for ref in group['refs']:
                item = items_service.find_one(req=None, guid=ref['guid'])
                syndicated_creator = user_service.find_one(req=None, _id=item['original_creator'])
                text = item.get('text')
                item_type = item.get('item_type')
                group_type = item.get('group_type')
                meta = item.get('meta', {})
                data = {
                    'text': text,
                    'item_type': item_type,
                    'group_type': group_type,
                    'commenter': item.get('commenter'),
                    'syndicated_creator': {k: v for k, v in syndicated_creator.items() if k in needed_fields},
                    'meta': meta<commit_msg>fix: behave.<commit_after>import json
import hmac
import logging
import tempfile
import urllib.parse
import uuid
from flask import abort
import requests
from bson import ObjectId
from hashlib import sha1
from eve.io.mongo import MongoJSONEncoder
from requests.exceptions import RequestException, ConnectionError, ConnectTimeout
from requests.packages.urllib3.exceptions import MaxRetryError
from superdesk import get_resource_service
from superdesk.metadata.item import ITEM_TYPE, CONTENT_TYPE
from apps.auth import SuperdeskTokenAuth
from .exceptions import APIConnectionError, DownloadError
from werkzeug.datastructures import FileStorage

logger = logging.getLogger('superdesk')


def generate_api_key():
    """Generate a new API Key."""
    random_uuid = uuid.uuid4()
    return hmac.new(random_uuid.bytes, digestmod=sha1).hexdigest()


def trailing_slash(url):
    """Add trailing slash to url."""
    if not url.endswith('/'):
        url = '{}/'.format(url)
    return url


def cast_to_object_id(doc, fields):
    """Cast provided document fields to ObjectId."""
    for field in fields:
        value = doc.get(field)
        if not value:
            continue

        if not ObjectId.is_valid(value):
            logger.warning('Field "{}" value "{}" is not a valid ObjectId')
            continue

        doc[field] = ObjectId(value)


def send_api_request(api_url, api_key, method='GET', args=None, data=None, json_loads=True, timeout=5):
    """Utility function to send http requests for consumer/producer api endpoints."""
    if data:
        data = json.dumps(data, cls=MongoJSONEncoder)

    session = requests.Session()
    session.trust_env = False
    adapter = requests.adapters.HTTPAdapter(max_retries=0)
    session.mount('http://', adapter)
    session.mount('https://', adapter)

    logger.debug('API {} request to {} with params={} and data={}'.format(method, api_url, args, data))
    headers = {
        'Content-Type': 'application/json'
    }
    if api_key:
        headers['Authorization'] = api_key

    try:
        response = requests.request(method, api_url, headers=headers, params=args, data=data, timeout=timeout)
    except (ConnectionError, ConnectTimeout, RequestException, MaxRetryError):
        raise APIConnectionError('Unable to connect to api_url "{}".'.format(api_url))

    logger.debug('API {} request to {} - response: {} {}'.format(
        method, api_url, response.status_code, response.content
    ))

    if not json_loads:
        return response
    else:
        return response.json()


def fetch_url(url, timeout=5):
    """Fetch url using python-requests and save data to temporary file."""
    try:
        response = requests.get(url, timeout=timeout)
    except (ConnectionError, RequestException, MaxRetryError):
        raise DownloadError('Unable to download url: "{}"'.format(url))
    fd = tempfile.NamedTemporaryFile()
    for chunk in response.iter_content(chunk_size=1024):
        if chunk:
            fd.write(chunk)
    fd.seek(0)
    return fd


def blueprint_superdesk_token_auth():
    auth = SuperdeskTokenAuth()
    authorized = auth.authorized(allowed_roles=[], resource='producers', method='GET')
    if not authorized:
        return abort(401, 'Authorization failed.')


def extract_post_items_data(original_doc):
    """Extract blog post items."""
    items_service = get_resource_service('items')
    user_service = get_resource_service('users')
    item_type = original_doc.get(ITEM_TYPE, '')
    if item_type != CONTENT_TYPE.COMPOSITE:
        raise NotImplementedError('Post item_type "{}" not supported.'.format(item_type))

    items = []
    needed_fields = ("avatar", "avatar_renditions", "byline",
                     "display_name", "email", "first_name",
                     "last_name", "picture_url", "sign_off",
                     "username", "_id", "_created", "_updated")

    for group in original_doc['groups']:
        if group['id'] == 'main':
            for ref in group['refs']:
                item = items_service.find_one(req=None, guid=ref['guid'])
                syndicated_creator = user_service.find_one(req=None, _id=item['original_creator'])
                if syndicated_creator:
                    syndicated_obj = {k: v for k, v in syndicated_creator.items() if k in needed_fields}
                text = item.get('text')
                item_type = item.get('item_type')
                group_type = item.get('group_type')
                meta = item.get('meta', {})
                data = {
                    'text': text,
                    'item_type': item_type,
                    'group_type': group_type,
                    'commenter': item.get('commenter'),
                    'syndicated_creator': syndicated_obj,
                    'meta': meta
{'fraction_non_alphanumeric': 0.11492317406861713, 'fraction_numerical': 0.003157230056830141, 'mean_word_length': 2.9861577181208054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5612131', 'n_tokens_mistral': 2762, 'n_tokens_neox': 2678, 'n_words': 696}