<commit_before>"""Upload module"""
import logging
import superdesk
from eve.utils import config
from superdesk import SuperdeskError
from .resource import Resource
from .services import BaseService
from flask import url_for, Response, current_app as app, json
from superdesk.media.renditions import generate_renditions, delete_file_on_error
from superdesk.media.media_operations import download_file_from_url, process_file_from_stream, \
    crop_image, decode_metadata, download_file_from_encoded_str


bp = superdesk.Blueprint('upload_raw', __name__)
superdesk.blueprint(bp)
logger = logging.getLogger(__name__)


@bp.route('/upload/<path:media_id>/raw', methods=['GET'])
def get_upload_as_data_uri(media_id):
    media_file = app.media.get(media_id)
    if media_file:
        return Response(media_file.read(), mimetype=media_file.content_type)
    raise SuperdeskError(status_code=404, payload='File not found on media storage.')


def url_for_media(media_id):
    return url_for('upload_raw.get_upload_as_data_uri', media_id=media_id,
                   _external=True, _schema=superdesk.config.URL_PROTOCOL)


def init_app(app):
    endpoint_name = 'upload'
    service = UploadService(endpoint_name, backend=superdesk.get_backend())
    UploadResource(endpoint_name, app=app, service=service)


class UploadResource(Resource):
    schema = {
        'media': {'type': 'file'},
        'CropLeft': {'type': 'integer'},
        'CropRight': {'type': 'integer'},
        'CropTop': {'type': 'integer'},
        'CropBottom': {'type': 'integer'},
        'URL': {'type': 'string'},
        'mime_type': {'type': 'string'},
        'filemeta': {'type': 'dict'}
    }
    extra_response_fields = ['renditions']
    datasource = {
        'projection': {
            'mime_type': 1,
            'filemeta': 1,
            '_created': 1,
            '_updated': 1,
            '_etag': 1,
            'media': 1,
            'renditions': 1,
        }
    }
    item_methods = ['GET', 'DELETE']
    resource_methods = ['GET', 'POST']
    privileges = {'POST': 'archive', 'DELETE': 'archive'}


class UploadService(BaseService):

    def on_create(self, docs):
        for doc in docs:
            if doc.get('URL') and doc.get('media'):
                message = 'Uploading file by URL and file stream in the same time is not supported.'
                raise SuperdeskError(payload=message)

            content = None
            filename = None
            content_type = None
            if doc.get('media'):
                content = doc['media']
                filename = content.filename
                content_type = content.mimetype
            elif doc.get('URL'):
                content, filename, content_type = self.download_file(doc)

            self.store_file(doc, content, filename, content_type)

    def store_file(self, doc, content, filename, content_type):
        res = process_file_from_stream(content, filename=filename, content_type=content_type)
        file_name, content_type, metadata = res

        cropping_data = self.get_cropping_data(doc)
        _, out = crop_image(content, filename, cropping_data)
        metadata['length'] = json.dumps(len(out.getvalue()))

        try:
            logger.debug('Going to save media file with %s ' % file_name)
            out.seek(0)
            id = app.media.put(out, filename=file_name, content_type=content_type, metadata=metadata)
            doc['media'] = id
            doc['mime_type'] = content_type
            doc['filemeta'] = decode_metadata(metadata)
            inserted = [doc['media']]
            file_type = content_type.split('/')[0]

            rendition_spec = config.RENDITIONS['avatar']
            renditions = generate_renditions(out, doc['media'], inserted, file_type,
                                             content_type, rendition_spec, url_for_media)
            doc['renditions'] = renditions
        except Exception as io:
            logger.exception(io)
            for file_id in inserted:<commit_msg>file upload : check if getvalue attribute exists before calling it<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

"""Upload module"""
import logging
import superdesk
from eve.utils import config
from superdesk import SuperdeskError
from .resource import Resource
from .services import BaseService
from flask import url_for, Response, current_app as app, json
from superdesk.media.renditions import generate_renditions, delete_file_on_error
from superdesk.media.media_operations import download_file_from_url, process_file_from_stream, \
    crop_image, decode_metadata, download_file_from_encoded_str


bp = superdesk.Blueprint('upload_raw', __name__)
superdesk.blueprint(bp)
logger = logging.getLogger(__name__)


@bp.route('/upload/<path:media_id>/raw', methods=['GET'])
def get_upload_as_data_uri(media_id):
    media_file = app.media.get(media_id)
    if media_file:
        return Response(media_file.read(), mimetype=media_file.content_type)
    raise SuperdeskError(status_code=404, payload='File not found on media storage.')


def url_for_media(media_id):
    return url_for('upload_raw.get_upload_as_data_uri', media_id=media_id,
                   _external=True, _schema=superdesk.config.URL_PROTOCOL)


def init_app(app):
    endpoint_name = 'upload'
    service = UploadService(endpoint_name, backend=superdesk.get_backend())
    UploadResource(endpoint_name, app=app, service=service)


class UploadResource(Resource):
    schema = {
        'media': {'type': 'file'},
        'CropLeft': {'type': 'integer'},
        'CropRight': {'type': 'integer'},
        'CropTop': {'type': 'integer'},
        'CropBottom': {'type': 'integer'},
        'URL': {'type': 'string'},
        'mime_type': {'type': 'string'},
        'filemeta': {'type': 'dict'}
    }
    extra_response_fields = ['renditions']
    datasource = {
        'projection': {
            'mime_type': 1,
            'filemeta': 1,
            '_created': 1,
            '_updated': 1,
            '_etag': 1,
            'media': 1,
            'renditions': 1,
        }
    }
    item_methods = ['GET', 'DELETE']
    resource_methods = ['GET', 'POST']
    privileges = {'POST': 'archive', 'DELETE': 'archive'}


class UploadService(BaseService):

    def on_create(self, docs):
        for doc in docs:
            if doc.get('URL') and doc.get('media'):
                message = 'Uploading file by URL and file stream in the same time is not supported.'
                raise SuperdeskError(payload=message)

            content = None
            filename = None
            content_type = None
            if doc.get('media'):
                content = doc['media']
                filename = content.filename
                content_type = content.mimetype
            elif doc.get('URL'):
                content, filename, content_type = self.download_file(doc)

            self.store_file(doc, content, filename, content_type)

    def store_file(self, doc, content, filename, content_type):
        res = process_file_from_stream(content, filename=filename, content_type=content_type)
        file_name, content_type, metadata = res
        cropping_data = self.get_cropping_data(doc)
        _, out = crop_image(content, filename, cropping_data)
        # for heaviest files, `out` contains a BufferedRandom stream,
        # then it's not possible to retrieve the length and `getvalue()` is not available
        if hasattr(out, 'getvalue'):
            metadata['length'] = json.dumps(len(out.getvalue()))
        try:
            logger.debug('Going to save media file with %s ' % file_name)
            out.seek(0)
            id = app.media.put(out, filename=file_name, content_type=content_type, metadata=metadata)
            doc['media'] = id
            doc['mime_type'] = content_type
            doc['filemeta'] = decode_metadata(metadata)
            inserted = [doc['media']]
            file_type = content_type.split('/')[0]
            rendition_spec = config.RENDITIONS['avatar']
            renditions = generate_renditions(out, doc['media'], inserted, file_type,
                                             content_type, rendition_spec, url_for_media)
            doc['renditions'] = renditions
        except Exception as io:
            logger.exception(io)
            for file_id in inserted:
{'fraction_non_alphanumeric': 0.11458454386984311, 'fraction_numerical': 0.0038349796629866357, 'mean_word_length': 2.774561403508772, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1636419', 'n_tokens_mistral': 2526, 'n_tokens_neox': 2456, 'n_words': 598}