<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#############################################################################

import cherrypy
import six
from six.moves import urllib

from girder.models.model_base import GirderException, ValidationException
from girder.utility.abstract_assetstore_adapter import AbstractAssetstoreAdapter

from .dbs import getDBConnectorClassFromDialect
from .query import dbFormatList, queryDatabase


dbInfoKey = 'databaseMetadata'


class DatabaseAssetstoreAdapter(AbstractAssetstoreAdapter):
    def __init__(self, assetstore):
        super(DatabaseAssetstoreAdapter, self).__init__(assetstore)

    @staticmethod
    def validateInfo(doc):
        """
        Ensures we have the necessary information to connect to a database.
        """
        # Ensure that the assetstore is marked read-only
        doc['readOnly'] = True
        info = doc.get('database', {})
        uri = info.get('uri')
        if not uri:
            raise ValidationException('Missing uri field.')
        dbtype = info.get('dbtype')
        dialect = uri.split('://', 1)[0] if '://' in uri else dbtype
        validatedDialect, validatedDbtype = getDBConnectorClassFromDialect(
            dialect, dbtype)
        if validatedDbtype is None:
            if dbtype:
                raise ValidationException(
                    'URI is not valid for dbtype %s.' % dbtype)
            raise ValidationException(
                'Either specify dbtype or include a dialect in the URI.')
        info['uri'] = uri.split('://', 1)[-1]
        if validatedDialect:
            info['uri'] = '%s://%s' % (validatedDialect, info['uri'])
        info['dbtype'] = validatedDbtype

    def initUpload(self, upload):
        raise NotImplementedError('Database assetstores are read only.')

    def finializeUpload(self, upload, file):
        raise NotImplementedError('Database assetstores are read only.')

    def deleteFile(self, file):
        if file.get(dbInfoKey, {}).get('imported'):
            return
        raise NotImplementedError('Database assetstores are read only.')

    def setContentHeaders(self, file, offset, endByte, contentDisposition=None):
        """
        Sets the Content-Length, Content-Disposition, Content-Type, and also
        the Content-Range header if this is a partial download.

        :param file: The file being downloaded.
        :param offset: The start byte of the download.
        :type offset: int
        :param endByte: The end byte of the download (non-inclusive).
        :type endByte: int
        :param contentDisposition: Content-Disposition response header
            disposition-type value, if None, Content-Disposition will
            be set to 'attachment; filename=$filename'.
        :type contentDisposition: str or None
        """
        cherrypy.response.headers['Content-Type'] = \
            file.get('mimeType') or 'application/octet-stream'
        if contentDisposition == 'inline':
            cherrypy.response.headers['Content-Disposition'] = \
                'inline; filename="%s"' % file['name']
        else:
            cherrypy.response.headers['Content-Disposition'] = \
                'attachment; filename="%s"' % file['name']
        cherrypy.response.headers['Accept-Ranges'] = 'bytes'
        if (endByte is not None and offset is not None and
                file['size'] is not None):
            cherrypy.response.headers['Content-Length'] = max(
                endByte - offset, 0)
            if (offset or endByte < file['size']) and file['size']:
                cherrypy.response.headers['Content-Range'] = \
                    'bytes %d-%d/%d' % (offset, endByte - 1, file['size'])

    def downloadFile(self, file, offset=0, headers=True, endByte=None,
                     contentDisposition=None, extraParameters=None, **kwargs):
        """
        This method is in charge of returning a value to the RESTful endpoint
        that can be used to download the file.  This can return a generator
        function that streams the file directly, or can modify the cherrypy
        request headers and perform a redirect and return None, for example.

        :param file: The file document being downloaded.
        :type file: dict
        :param offset: Offset in bytes to start the download at.
        :type offset: int
        :param headers: Flag for whether headers should be sent on the response.
        :type headers: bool
        :param endByte: Final byte to download. If ``None``, downloads to the
            end of the file.
        :type endByte: int or None
        :param contentDisposition: Value for Content-Disposition response
            header disposition-type value.
        :type contentDisposition: str or None
        :type extraParameters: str or None.  url encoded query string of
            parameters to add to the query.
        :returns: a function that returns a generator for the data.
        """
        dbinfo = {
            'type': self.assetstore['database']['dbtype'],
            'url': self.assetstore['database']['uri'],
            'table': file[dbInfoKey]['table'],
            'collection': file[dbInfoKey]['table']
        }
        params = {
            'sort': file[dbInfoKey].get('sort'),
            'fields': file[dbInfoKey].get('fields'),
            'filters': file[dbInfoKey].get('filters'),
            'format': file[dbInfoKey].get('format'),
            'offset': 0,
            'limit': file[dbInfoKey].get('limit'),
        }
        if extraParameters:
            params.update({key: value for (key, value) in
                           urllib.parse.parse_qsl(extraParameters)})
        resultFunc, mimeType = queryDatabase(file.get('_id'), dbinfo, params)
        file['mimeType'] = mimeType

        # We often have to compute the response length.  This also handles
        # partial range requests (though not very efficiently, as each
        # request will requery the database, which may not be consistent).
        file['size'] = None
        if offset or endByte is not None:
            totallen = 0
            skipped = 0
            output = six.BytesIO()
            for chunk in resultFunc():
                totallen += len(chunk)
                if skipped < offset:
                    if skipped + len(chunk) <= offset:
                        skipped += len(chunk)
                    else:
                        chunk = chunk[offset - skipped:]
                        skipped = offset
                if endByte and totallen > endByte:
                    chunk = chunk[:endByte - totallen]
                output.write(chunk)
            output.seek(0)
            file['size'] = totallen

            def bufferResults():
                while True:
                    chunk = output.read(65536)
                    if not chunk:
                        break
                    yield chunk

            resultFunc = bufferResults
            if endByte is None or endByte > file['size']:
                endByte = file['size']

        if headers:
            self.setContentHeaders(file, offset, endByte, contentDisposition)
            if endByte is not None and endByte - offset <= 0:
                return lambda: ''

        return resultFunc

    def copyFile(self, srcFile, destFile):
        """
        This method copies the necessary fields and data so that the
        destination file contains the same data as the source file.

        :param srcFile: The original File document.
        :type srcFile: dict
        :param destFile: The File which should have the data copied to it.
        :type destFile: dict
        :returns: A dict with the destination file.
        """
        return destFile

    def importData(self, parent, parentType, params, progress, user, **kwargs):
        """
        Import a list of tables or collections, each to a file within a
        distinct item.

        :param parent: The parent object to import into.  Must be a folder.
        :param parentType: The model type of the parent object.  Ignored, as it
            must be 'folder'.
        :param params: Additional parameters required for the import process:
            tables: a list of tables to add.  If there is already an item with
                    an exact table name, it is updated.
            sort: default sort parameter.  Used in plain downloads.
            fields: default fields parameter.  Used in plain downloads.
            filters: default filters parameter.  Used in plain downloads.
            format: default format parameter.  Used in plain downloads.
        :type params: dict
        :param progress: Object on which to record progress if possible.
        :type progress: :py:class:`girder.utility.progress.ProgressContext`
        :param user: The Girder user performing the import.
        :type user: dict or None
        """
        itemModel = self.model('item')
        fileModel = self.model('file')
        for table in params['tables']:
            progress.update(message='Importing %s' % table)
            # Create an item if needed
            item = itemModel.findOne({
                'folderId': parent['_id'],
                'name': table
            })
            if item is None:
                item = itemModel.createItem(
                    name=table, creator=user, folder=parent)
            # Create a file if needed
            file = fileModel.findOne({
                'name': table,
                'itemId': item['_id']
            })
            if file is None:
                file = fileModel.createFile(
                    creator=user, item=item, name=table, size=0,
                    assetstore=self.assetstore,
                    mimeType=dbFormatList.get(params.get('format')),
                    saveFile=False)
            if file.get(dbInfoKey) and not file[dbInfoKey].get('imported'):
                raise GirderException(
                    'A file for table %s is present but cannot be updated '
                    'because it wasn\'t imported.' % table)
            # Set or replace the database parameters for the file
            file[dbInfoKey] = {
                'imported': True,
                'table': table,
                'sort': params.get('sort'),
                'fields': params.get('fields'),
                'filters': params.get('filters'),
                'format': params.get('format'),
                'limit': params.get('limit'),
            }
            # Validate that we can perform queries by trying to download 1
            # record from the file
            downloadFunc = self.downloadFile(
                file.copy(), headers=False, extraParameters='limit=1')
            # Test the download without keeping it
            [None for chunk in downloadFunc()]
            try:
                if (params.get('limit') not in (None, '') and
                        int(params.get('limit')) <= 0):
                    raise ValueError()
            except ValueError:
                raise GirderException(
                    'limit must be empty or a positive integer')
            # Now save the new file
            fileModel.save(file)
<commit_msg>Support range requests.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#############################################################################

import cherrypy
import six
from six.moves import urllib

from girder.models.model_base import GirderException, ValidationException
from girder.utility.abstract_assetstore_adapter import AbstractAssetstoreAdapter

from .dbs import getDBConnectorClassFromDialect
from .query import dbFormatList, queryDatabase


dbInfoKey = 'databaseMetadata'


class DatabaseAssetstoreAdapter(AbstractAssetstoreAdapter):
    def __init__(self, assetstore):
        super(DatabaseAssetstoreAdapter, self).__init__(assetstore)

    @staticmethod
    def validateInfo(doc):
        """
        Ensures we have the necessary information to connect to a database.
        """
        # Ensure that the assetstore is marked read-only
        doc['readOnly'] = True
        info = doc.get('database', {})
        uri = info.get('uri')
        if not uri:
            raise ValidationException('Missing uri field.')
        dbtype = info.get('dbtype')
        dialect = uri.split('://', 1)[0] if '://' in uri else dbtype
        validatedDialect, validatedDbtype = getDBConnectorClassFromDialect(
            dialect, dbtype)
        if validatedDbtype is None:
            if dbtype:
                raise ValidationException(
                    'URI is not valid for dbtype %s.' % dbtype)
            raise ValidationException(
                'Either specify dbtype or include a dialect in the URI.')
        info['uri'] = uri.split('://', 1)[-1]
        if validatedDialect:
            info['uri'] = '%s://%s' % (validatedDialect, info['uri'])
        info['dbtype'] = validatedDbtype

    def initUpload(self, upload):
        raise NotImplementedError('Database assetstores are read only.')

    def finializeUpload(self, upload, file):
        raise NotImplementedError('Database assetstores are read only.')

    def deleteFile(self, file):
        if file.get(dbInfoKey, {}).get('imported'):
            return
        raise NotImplementedError('Database assetstores are read only.')

    def setContentHeaders(self, file, offset, endByte, contentDisposition=None):
        """
        Sets the Content-Length, Content-Disposition, Content-Type, and also
        the Content-Range header if this is a partial download.

        :param file: The file being downloaded.
        :param offset: The start byte of the download.
        :type offset: int
        :param endByte: The end byte of the download (non-inclusive).
        :type endByte: int
        :param contentDisposition: Content-Disposition response header
            disposition-type value, if None, Content-Disposition will
            be set to 'attachment; filename=$filename'.
        :type contentDisposition: str or None
        """
        cherrypy.response.headers['Content-Type'] = \
            file.get('mimeType') or 'application/octet-stream'
        if contentDisposition == 'inline':
            cherrypy.response.headers['Content-Disposition'] = \
                'inline; filename="%s"' % file['name']
        else:
            cherrypy.response.headers['Content-Disposition'] = \
                'attachment; filename="%s"' % file['name']
        cherrypy.response.headers['Accept-Ranges'] = 'bytes'
        if (endByte is not None and offset is not None and
                file['size'] is not None):
            cherrypy.response.headers['Content-Length'] = max(
                endByte - offset, 0)
            if (offset or endByte < file['size']) and file['size']:
                cherrypy.response.headers['Content-Range'] = \
                    'bytes %d-%d/%d' % (offset, endByte - 1, file['size'])

    def _getDownloadSize(self, file, resultFunc, offset, endByte):
        """
        Given a file and an output generator function, generate the output to
        determine its total length, buffering the part that is needed for the
        final output.  Adjust the file's size and the endByte value
        accordingly.

        :param file: the file used for the original generation.  Its size value
                     is updated.
        :param resultFunc: a function that produces a generator for the output.
        :param offset: offset within the file information to output.
        :param endByte: the maximum index to output (the output is
                        data[offset:endByte]).
        :returns: a new function that produces a generator for the output.
        :returns: a new value for endByte.
        """
        totallen = 0
        skipped = 0
        output = six.BytesIO()
        for chunk in resultFunc():
            totallen += len(chunk)
            if skipped < offset:
                if skipped + len(chunk) <= offset:
                    skipped += len(chunk)
                    continue
                else:
                    chunk = chunk[offset - skipped:]
                    skipped = offset
            if endByte and totallen > endByte:
                chunk = chunk[:endByte - totallen]
            output.write(chunk)
        output.seek(0)
        file['size'] = totallen

        def bufferResults():
            while True:
                chunk = output.read(65536)
                if not chunk:
                    break
                yield chunk

        if endByte is None or endByte > file['size']:
            endByte = file['size']
        return bufferResults, endByte

    def downloadFile(self, file, offset=0, headers=True, endByte=None,
                     contentDisposition=None, extraParameters=None, **kwargs):
        """
        This method is in charge of returning a value to the RESTful endpoint
        that can be used to download the file.  This can return a generator
        function that streams the file directly, or can modify the cherrypy
        request headers and perform a redirect and return None, for example.

        :param file: The file document being downloaded.
        :type file: dict
        :param offset: Offset in bytes to start the download at.
        :type offset: int
        :param headers: Flag for whether headers should be sent on the response.
        :type headers: bool
        :param endByte: Final byte to download. If ``None``, downloads to the
            end of the file.
        :type endByte: int or None
        :param contentDisposition: Value for Content-Disposition response
            header disposition-type value.
        :type contentDisposition: str or None
        :type extraParameters: str or None.  url encoded query string of
            parameters to add to the query.
        :returns: a function that returns a generator for the data.
        """
        dbinfo = {
            'type': self.assetstore['database']['dbtype'],
            'url': self.assetstore['database']['uri'],
            'table': file[dbInfoKey]['table'],
            'collection': file[dbInfoKey]['table']
        }
        params = {
            'sort': file[dbInfoKey].get('sort'),
            'fields': file[dbInfoKey].get('fields'),
            'filters': file[dbInfoKey].get('filters'),
            'format': file[dbInfoKey].get('format'),
            'offset': 0,
            'limit': file[dbInfoKey].get('limit'),
        }
        if extraParameters:
            params.update({key: value for (key, value) in
                           urllib.parse.parse_qsl(extraParameters)})
        resultFunc, mimeType = queryDatabase(file.get('_id'), dbinfo, params)
        file['mimeType'] = mimeType

        # If we have been asked for headers, recheck if we should have a range
        # request
        if headers and cherrypy.request.headers.get('Range'):
            rangeHeader = cherrypy.lib.httputil.get_ranges(
                cherrypy.request.headers.get('Range'), six.MAXSIZE)
            if rangeHeader and len(rangeHeader):
                # Currently we only support a single range.
                offset, endByte = rangeHeader[0]

        # We often have to compute the response length.  This also handles
        # partial range requests (though not very efficiently, as each
        # request will requery the database, which may not be consistent).
        file['size'] = None
        if offset or endByte is not None:
            resultFunc, endByte = self._getDownloadSize(
                file, resultFunc, offset, endByte)

        if headers:
            self.setContentHeaders(file, offset, endByte, contentDisposition)
            if endByte is not None and endByte - offset <= 0:
                return lambda: ''

        return resultFunc

    def copyFile(self, srcFile, destFile):
        """
        This method copies the necessary fields and data so that the
        destination file contains the same data as the source file.

        :param srcFile: The original File document.
        :type srcFile: dict
        :param destFile: The File which should have the data copied to it.
        :type destFile: dict
        :returns: A dict with the destination file.
        """
        return destFile

    def importData(self, parent, parentType, params, progress, user, **kwargs):
        """
        Import a list of tables or collections, each to a file within a
        distinct item.

        :param parent: The parent object to import into.  Must be a folder.
        :param parentType: The model type of the parent object.  Ignored, as it
            must be 'folder'.
        :param params: Additional parameters required for the import process:
            tables: a list of tables to add.  If there is already an item with
                    an exact table name, it is updated.
            sort: default sort parameter.  Used in plain downloads.
            fields: default fields parameter.  Used in plain downloads.
            filters: default filters parameter.  Used in plain downloads.
            format: default format parameter.  Used in plain downloads.
        :type params: dict
        :param progress: Object on which to record progress if possible.
        :type progress: :py:class:`girder.utility.progress.ProgressContext`
        :param user: The Girder user performing the import.
        :type user: dict or None
        """
        itemModel = self.model('item')
        fileModel = self.model('file')
        for table in params['tables']:
            progress.update(message='Importing %s' % table)
            # Create an item if needed
            item = itemModel.findOne({
                'folderId': parent['_id'],
                'name': table
            })
            if item is None:
                item = itemModel.createItem(
                    name=table, creator=user, folder=parent)
            # Create a file if needed
            file = fileModel.findOne({
                'name': table,
                'itemId': item['_id']
            })
            if file is None:
                file = fileModel.createFile(
                    creator=user, item=item, name=table, size=0,
                    assetstore=self.assetstore,
                    mimeType=dbFormatList.get(params.get('format')),
                    saveFile=False)
            if file.get(dbInfoKey) and not file[dbInfoKey].get('imported'):
                raise GirderException(
                    'A file for table %s is present but cannot be updated '
                    'because it wasn\'t imported.' % table)
            # Set or replace the database parameters for the file
            file[dbInfoKey] = {
                'imported': True,
                'table': table,
                'sort': params.get('sort'),
                'fields': params.get('fields'),
                'filters': params.get('filters'),
                'format': params.get('format'),
                'limit': params.get('limit'),
            }
            # Validate that we can perform queries by trying to download 1
            # record from the file
            downloadFunc = self.downloadFile(
                file.copy(), headers=False, extraParameters='limit=1')
            # Test the download without keeping it
            [None for chunk in downloadFunc()]
            try:
                if (params.get('limit') not in (None, '') and
                        int(params.get('limit')) <= 0):
                    raise ValueError()
            except ValueError:
                raise GirderException(
                    'limit must be empty or a positive integer')
            # Now save the new file
            fileModel.save(file)

{'fraction_non_alphanumeric': 0.10068613371629169, 'fraction_numerical': 0.0021142492420615925, 'mean_word_length': 2.0062357596834155, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728186', 'n_tokens_mistral': 6282, 'n_tokens_neox': 6064, 'n_words': 2229}