<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2013 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import datetime
import six
from bson.objectid import ObjectId

from girder import events
from girder.constants import SettingKey
from girder.utility import assetstore_utilities
from .model_base import Model, GirderException, ValidationException
from girder.utility.progress import noProgress


class Upload(Model):
    """
    This model stores temporary records for uploads that have been approved
    but are not yet complete, so that they can be uploaded in chunks of
    arbitrary size. The chunks must be uploaded in order.
    """
    def initialize(self):
        self.name = 'upload'

    def _getChunkSize(self, minSize=32 * 1024**2):
        """
        Return a chunk size to use in file uploads.  This is the maximum of
        the setting for minimum upload chunk size and the specified size.

        :param minSize: the minimum size to return.
        :return: chunk size to use for file uploads.
        """
        minChunkSize = self.model('setting').get(SettingKey.UPLOAD_MINIMUM_CHUNK_SIZE)
        return max(minChunkSize, minSize)

    def uploadFromFile(self, obj, size, name, parentType=None, parent=None,
                       user=None, mimeType=None, reference=None,
                       assetstore=None, attachParent=False):
        """
        This method wraps the entire upload process into a single function to
        facilitate "internal" uploads from a file-like object. Example:

        .. code-block:: python

            size = os.path.getsize(filename)

            with open(filename, 'rb') as f:
                self.model('upload').uploadFromFile(
                    f, size, filename, 'item', parentItem, user)

        :param obj: The object representing the content to upload.
        :type obj: file-like
        :param size: The total size of
        :param name: The name of the file to create.
        :type name: str
        :param parentType: The type of the parent: "folder" or "item".
        :type parentType: str
        :param parent: The parent (item or folder) to upload into.
        :type parent: dict
        :param user: The user who is creating the file.
        :type user: dict
        :param mimeType: MIME type of the file.
        :type mimeType: str
        :param reference: An optional reference string that will be sent to the
                          data.process event.
        :param assetstore: An optional assetstore to use to store the file.  If
            unspecified, the current assetstore is used.
        :type reference: str
        :param attachParent: if True, instead of creating an item within the
            parent or giving the file an itemId, set itemId to None and set
            attachedToType and attachedToId instead (using the values passed in
            parentType and parent).  This is intended for files that shouldn't
            appear as direct children of the parent, but are still associated
            with it.
        :type attach: boolean
        """
        upload = self.createUpload(
            user=user, name=name, parentType=parentType, parent=parent,
            size=size, mimeType=mimeType, reference=reference,
            assetstore=assetstore, attachParent=attachParent)
        # The greater of 32 MB or the the upload minimum chunk size.
        chunkSize = self._getChunkSize()

        while True:
            data = obj.read(chunkSize)
            if not data:
                break

            upload = self.handleChunk(upload, six.BytesIO(data))

        return upload

    def validate(self, doc):
        if doc['size'] < 0:
            raise ValidationException('File size must not be negative.')
        if doc['received'] > doc['size']:
            raise ValidationException('Received too many bytes.')

        doc['updated'] = datetime.datetime.utcnow()

        return doc

    def handleChunk(self, upload, chunk):
        """
        When a chunk is uploaded, this should be called to process the chunk.
        If this is the final chunk of the upload, this method will finalize
        the upload automatically.

        :param upload: The upload document to update.
        :type upload: dict
        :param chunk: The file object representing the chunk that was uploaded.
        :type chunk: file
        """
        assetstore = self.model('assetstore').load(upload['assetstoreId'])
        adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)

        upload = self.save(adapter.uploadChunk(upload, chunk))

        # If upload is finished, we finalize it
        if upload['received'] == upload['size']:
            return self.finalizeUpload(upload, assetstore)
        else:
            return upload

    def requestOffset(self, upload):
        """
        Requests the offset that should be used to resume uploading. This
        makes the request from the assetstore adapter.
        """
        assetstore = self.model('assetstore').load(upload['assetstoreId'])
        adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
        return adapter.requestOffset(upload)

    def finalizeUpload(self, upload, assetstore=None):
        """
        This should only be called manually in the case of creating an
        empty file, i.e. one that has no chunks.

        :param upload: The upload document.
        :type upload: dict
        :param assetstore: If known, the containing assetstore for the upload.
        :type assetstore: dict
        :returns: The file object that was created.
        """
        events.trigger('model.upload.finalize', upload)
        if assetstore is None:
            assetstore = self.model('assetstore').load(upload['assetstoreId'])

        if 'fileId' in upload:  # Updating an existing file's contents
            file = self.model('file').load(upload['fileId'], force=True)

            # Delete the previous file contents from the containing assetstore
            assetstore_utilities.getAssetstoreAdapter(
                self.model('assetstore').load(
                    file['assetstoreId'])).deleteFile(file)

            item = self.model('item').load(file['itemId'], force=True)
            self.model('file').propagateSizeChange(
                item, upload['size'] - file['size'])

            # Update file info
            file['creatorId'] = upload['userId']
            file['created'] = datetime.datetime.utcnow()
            file['assetstoreId'] = assetstore['_id']
            file['size'] = upload['size']
            # If the file was previously import, it is no longer.
            if file.get('imported'):
                file['imported'] = False

        else:  # Creating a new file record
            if upload.get('attachParent'):
                item = None
            elif upload['parentType'] == 'folder':
                # Create a new item with the name of the file.
                item = self.model('item').createItem(
                    name=upload['name'], creator={'_id': upload['userId']},
                    folder={'_id': upload['parentId']})
            elif upload['parentType'] == 'item':
                item = self.model('item').load(
                    id=upload['parentId'], force=True)
            else:
                item = None

            file = self.model('file').createFile(
                item=item, name=upload['name'], size=upload['size'],
                creator={'_id': upload['userId']}, assetstore=assetstore,
                mimeType=upload['mimeType'], saveFile=False)
            if upload.get('attachParent'):
                if upload['parentType'] and upload['parentId']:
                    file['attachedToType'] = upload['parentType']
                    file['attachedToId'] = upload['parentId']<commit_msg>Fix typo<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright 2013 Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import datetime
import six
from bson.objectid import ObjectId

from girder import events
from girder.constants import SettingKey
from girder.utility import assetstore_utilities
from .model_base import Model, GirderException, ValidationException
from girder.utility.progress import noProgress


class Upload(Model):
    """
    This model stores temporary records for uploads that have been approved
    but are not yet complete, so that they can be uploaded in chunks of
    arbitrary size. The chunks must be uploaded in order.
    """
    def initialize(self):
        self.name = 'upload'

    def _getChunkSize(self, minSize=32 * 1024**2):
        """
        Return a chunk size to use in file uploads.  This is the maximum of
        the setting for minimum upload chunk size and the specified size.

        :param minSize: the minimum size to return.
        :return: chunk size to use for file uploads.
        """
        minChunkSize = self.model('setting').get(SettingKey.UPLOAD_MINIMUM_CHUNK_SIZE)
        return max(minChunkSize, minSize)

    def uploadFromFile(self, obj, size, name, parentType=None, parent=None,
                       user=None, mimeType=None, reference=None,
                       assetstore=None, attachParent=False):
        """
        This method wraps the entire upload process into a single function to
        facilitate "internal" uploads from a file-like object. Example:

        .. code-block:: python

            size = os.path.getsize(filename)

            with open(filename, 'rb') as f:
                self.model('upload').uploadFromFile(
                    f, size, filename, 'item', parentItem, user)

        :param obj: The object representing the content to upload.
        :type obj: file-like
        :param size: The total size of
        :param name: The name of the file to create.
        :type name: str
        :param parentType: The type of the parent: "folder" or "item".
        :type parentType: str
        :param parent: The parent (item or folder) to upload into.
        :type parent: dict
        :param user: The user who is creating the file.
        :type user: dict
        :param mimeType: MIME type of the file.
        :type mimeType: str
        :param reference: An optional reference string that will be sent to the
                          data.process event.
        :param assetstore: An optional assetstore to use to store the file.  If
            unspecified, the current assetstore is used.
        :type reference: str
        :param attachParent: if True, instead of creating an item within the
            parent or giving the file an itemId, set itemId to None and set
            attachedToType and attachedToId instead (using the values passed in
            parentType and parent).  This is intended for files that shouldn't
            appear as direct children of the parent, but are still associated
            with it.
        :type attach: boolean
        """
        upload = self.createUpload(
            user=user, name=name, parentType=parentType, parent=parent,
            size=size, mimeType=mimeType, reference=reference,
            assetstore=assetstore, attachParent=attachParent)
        # The greater of 32 MB or the the upload minimum chunk size.
        chunkSize = self._getChunkSize()

        while True:
            data = obj.read(chunkSize)
            if not data:
                break

            upload = self.handleChunk(upload, six.BytesIO(data))

        return upload

    def validate(self, doc):
        if doc['size'] < 0:
            raise ValidationException('File size must not be negative.')
        if doc['received'] > doc['size']:
            raise ValidationException('Received too many bytes.')

        doc['updated'] = datetime.datetime.utcnow()

        return doc

    def handleChunk(self, upload, chunk):
        """
        When a chunk is uploaded, this should be called to process the chunk.
        If this is the final chunk of the upload, this method will finalize
        the upload automatically.

        :param upload: The upload document to update.
        :type upload: dict
        :param chunk: The file object representing the chunk that was uploaded.
        :type chunk: file
        """
        assetstore = self.model('assetstore').load(upload['assetstoreId'])
        adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)

        upload = self.save(adapter.uploadChunk(upload, chunk))

        # If upload is finished, we finalize it
        if upload['received'] == upload['size']:
            return self.finalizeUpload(upload, assetstore)
        else:
            return upload

    def requestOffset(self, upload):
        """
        Requests the offset that should be used to resume uploading. This
        makes the request from the assetstore adapter.
        """
        assetstore = self.model('assetstore').load(upload['assetstoreId'])
        adapter = assetstore_utilities.getAssetstoreAdapter(assetstore)
        return adapter.requestOffset(upload)

    def finalizeUpload(self, upload, assetstore=None):
        """
        This should only be called manually in the case of creating an
        empty file, i.e. one that has no chunks.

        :param upload: The upload document.
        :type upload: dict
        :param assetstore: If known, the containing assetstore for the upload.
        :type assetstore: dict
        :returns: The file object that was created.
        """
        events.trigger('model.upload.finalize', upload)
        if assetstore is None:
            assetstore = self.model('assetstore').load(upload['assetstoreId'])

        if 'fileId' in upload:  # Updating an existing file's contents
            file = self.model('file').load(upload['fileId'], force=True)

            # Delete the previous file contents from the containing assetstore
            assetstore_utilities.getAssetstoreAdapter(
                self.model('assetstore').load(
                    file['assetstoreId'])).deleteFile(file)

            item = self.model('item').load(file['itemId'], force=True)
            self.model('file').propagateSizeChange(
                item, upload['size'] - file['size'])

            # Update file info
            file['creatorId'] = upload['userId']
            file['created'] = datetime.datetime.utcnow()
            file['assetstoreId'] = assetstore['_id']
            file['size'] = upload['size']
            # If the file was previously imported, it is no longer.
            if file.get('imported'):
                file['imported'] = False

        else:  # Creating a new file record
            if upload.get('attachParent'):
                item = None
            elif upload['parentType'] == 'folder':
                # Create a new item with the name of the file.
                item = self.model('item').createItem(
                    name=upload['name'], creator={'_id': upload['userId']},
                    folder={'_id': upload['parentId']})
            elif upload['parentType'] == 'item':
                item = self.model('item').load(
                    id=upload['parentId'], force=True)
            else:
                item = None

            file = self.model('file').createFile(
                item=item, name=upload['name'], size=upload['size'],
                creator={'_id': upload['userId']}, assetstore=assetstore,
                mimeType=upload['mimeType'], saveFile=False)
            if upload.get('attachParent'):
                if upload['parentType'] and upload['parentId']:
                    file['attachedToType'] = upload['parentType']
                    file['attachedToId'] = upload['parentId']
{'fraction_non_alphanumeric': 0.10730848253595539, 'fraction_numerical': 0.00223070149691811, 'mean_word_length': 2.339082712661701, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6779017', 'n_tokens_mistral': 4303, 'n_tokens_neox': 4160, 'n_words': 1542}