<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
import json

from bson import ObjectId
from bson.errors import InvalidId
import geojson
import six

from girder.api import access
from girder.api.rest import RestException, loadmodel, setRawResponse, \
    setResponseHeader
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType
from girder.models.model_base import GirderException
from girder.utility import mail_utils, ziputil

from .base import IsicResource
from ..utility import querylang


class ImageResource(IsicResource):
    def __init__(self,):
        super(ImageResource, self).__init__()
        self.resourceName = 'image'

        self.route('GET', (), self.find)
        self.route('GET', ('download',), self.downloadMultiple)
        self.route('GET', ('histogram',), self.getHistogram)
        self.route('GET', (':id',), self.getImage)
        self.route('GET', (':id', 'thumbnail'), self.thumbnail)
        self.route('GET', (':id', 'download'), self.download)
        self.route('GET', (':id', 'superpixels'), self.getSuperpixels)

        self.route('POST', (':id', 'segment'), self.doSegmentation)

    @describeRoute(
        Description('Return a list of lesion images.')
        .pagingParams(defaultSort='name')
        .param('datasetId', 'The ID of the dataset to use.', required=False)
        .param('name', 'Find an image with a specific name.',
               required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar '
                         '(causing "datasetId" and "name" to be ignored).',
               required=False)
        .errorResponse()
    )
    @access.cookie
    @access.public
    def find(self, params):
        Image = self.model('image', 'isic_archive')

        user = self.getCurrentUser()
        limit, offset, sort = self.getPagingParameters(params, 'name')

        if 'filter' in params:
            query = querylang.astToMongo(json.loads(params['filter']))
        else:
            query = {}
            if 'datasetId' in params:
                try:
                    query.update({'folderId': ObjectId(params['datasetId'])})
                except InvalidId:
                    raise RestException(
                        'Invalid "folderId" ObjectId: %s' % params['datasetId'])
            if 'name' in params:
                query.update({'name': params['name']})

        return [
            {
                field: image[field]
                for field in
                Image.summaryFields
            }
            for image in
            Image.filterResultsByPermission(
                # TODO: exclude additional fields from the cursor
                Image.find(query, sort=sort, fields={'meta': 0}),
                user=user, level=AccessType.READ, limit=limit, offset=offset)
        ]

    @describeRoute(
        Description('Download multiple images as a ZIP file.')
        .param('datasetId', 'The ID of the dataset to download.',
               required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar '
                         '(causing "datasetId" to be ignored).',
               required=False)
        .errorResponse()
    )
    @access.public
    def downloadMultiple(self, params):
        Dataset = self.model('dataset', 'isic_archive')
        File = self.model('file')
        Image = self.model('image', 'isic_archive')

        if 'filter' in params:
            query = querylang.astToMongo(json.loads(params['filter']))
        else:
            query = {}
            if 'datasetId' in params:
                try:
                    query.update({'folderId': ObjectId(params['datasetId'])})
                except InvalidId:
                    raise RestException(
                        'Invalid "folderId" ObjectId: %s' % params['datasetId'])

        user = self.getCurrentUser()
        downloadFileName = 'ISIC-images.zip'

        def stream():
            datasetCache = dict()
            zipGenerator = ziputil.ZipGenerator(downloadFileName)

            for image in Image.filterResultsByPermission(
                    # TODO: exclude additional fields from the cursor
                    Image.find(query, sort=[('name', 1)], fields={'meta': 0}),
                    user=user, level=AccessType.READ, limit=0, offset=0):
                datasetId = image['folderId']
                if datasetId not in datasetCache:
                    datasetCache[datasetId] = Dataset.load(
                        datasetId, force=True, exc=True)
                dataset = datasetCache[datasetId]
                imageFile = Image.originalFile(image)
                imageFileGenerator = File.download(imageFile, headers=False)
                for data in zipGenerator.addFile(
                        imageFileGenerator,
                        path=os.path.join(dataset['name'], image['name'])):
                    yield data

            for dataset in six.viewvalues(datasetCache):
                licenseText = mail_utils.renderTemplate(
                    'license_%s.mako' % dataset['meta']['license'])
                attributionText = mail_utils.renderTemplate(
                    'attribution_%s.mako' % dataset['meta']['license'],
                    {
                        'work': dataset['name'],
                        'author':
                            dataset['meta']['attribution']
                            if not dataset['meta']['anonymous']
                            else 'Anonymous'
                    })
                for data in zipGenerator.addFile(
                        lambda: [licenseText],
                        path=os.path.join(dataset['name'], 'license.txt')):
                    yield data
                for data in zipGenerator.addFile(
                        lambda: [attributionText],
                        path=os.path.join(dataset['name'], 'attribution.txt')):
                    yield data

            yield zipGenerator.footer()

        setResponseHeader('Content-Type', 'application/zip')
        setResponseHeader(
            'Content-Disposition',
            'attachment; filename="%s"' % downloadFileName)
        return stream

    @describeRoute(
        Description('Return histograms of image metadata.')
        .param('filter',
               'Get the histogram after the results of this filter. ' +
               'TODO: describe our filter grammar (an AST tree).',
               required=False)
        .errorResponse()
    )
    @access.cookie
    @access.public
    def getHistogram(self, params):<commit_msg>Fix file naming in ZIP file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import os
import json

from bson import ObjectId
from bson.errors import InvalidId
import geojson
import six

from girder.api import access
from girder.api.rest import RestException, loadmodel, setRawResponse, \
    setResponseHeader
from girder.api.describe import Description, describeRoute
from girder.constants import AccessType
from girder.models.model_base import GirderException
from girder.utility import mail_utils, ziputil

from .base import IsicResource
from ..utility import querylang


class ImageResource(IsicResource):
    def __init__(self,):
        super(ImageResource, self).__init__()
        self.resourceName = 'image'

        self.route('GET', (), self.find)
        self.route('GET', ('download',), self.downloadMultiple)
        self.route('GET', ('histogram',), self.getHistogram)
        self.route('GET', (':id',), self.getImage)
        self.route('GET', (':id', 'thumbnail'), self.thumbnail)
        self.route('GET', (':id', 'download'), self.download)
        self.route('GET', (':id', 'superpixels'), self.getSuperpixels)

        self.route('POST', (':id', 'segment'), self.doSegmentation)

    @describeRoute(
        Description('Return a list of lesion images.')
        .pagingParams(defaultSort='name')
        .param('datasetId', 'The ID of the dataset to use.', required=False)
        .param('name', 'Find an image with a specific name.',
               required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar '
                         '(causing "datasetId" and "name" to be ignored).',
               required=False)
        .errorResponse()
    )
    @access.cookie
    @access.public
    def find(self, params):
        Image = self.model('image', 'isic_archive')

        user = self.getCurrentUser()
        limit, offset, sort = self.getPagingParameters(params, 'name')

        if 'filter' in params:
            query = querylang.astToMongo(json.loads(params['filter']))
        else:
            query = {}
            if 'datasetId' in params:
                try:
                    query.update({'folderId': ObjectId(params['datasetId'])})
                except InvalidId:
                    raise RestException(
                        'Invalid "folderId" ObjectId: %s' % params['datasetId'])
            if 'name' in params:
                query.update({'name': params['name']})

        return [
            {
                field: image[field]
                for field in
                Image.summaryFields
            }
            for image in
            Image.filterResultsByPermission(
                # TODO: exclude additional fields from the cursor
                Image.find(query, sort=sort, fields={'meta': 0}),
                user=user, level=AccessType.READ, limit=limit, offset=offset)
        ]

    @describeRoute(
        Description('Download multiple images as a ZIP file.')
        .param('datasetId', 'The ID of the dataset to download.',
               required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar '
                         '(causing "datasetId" to be ignored).',
               required=False)
        .errorResponse()
    )
    @access.public
    def downloadMultiple(self, params):
        Dataset = self.model('dataset', 'isic_archive')
        File = self.model('file')
        Image = self.model('image', 'isic_archive')

        if 'filter' in params:
            query = querylang.astToMongo(json.loads(params['filter']))
        else:
            query = {}
            if 'datasetId' in params:
                try:
                    query.update({'folderId': ObjectId(params['datasetId'])})
                except InvalidId:
                    raise RestException(
                        'Invalid "folderId" ObjectId: %s' % params['datasetId'])

        user = self.getCurrentUser()
        downloadFileName = 'ISIC-images'

        def stream():
            datasetCache = dict()
            zipGenerator = ziputil.ZipGenerator(downloadFileName)

            for image in Image.filterResultsByPermission(
                    # TODO: exclude additional fields from the cursor
                    Image.find(query, sort=[('name', 1)], fields={'meta': 0}),
                    user=user, level=AccessType.READ, limit=0, offset=0):
                datasetId = image['folderId']
                if datasetId not in datasetCache:
                    datasetCache[datasetId] = Dataset.load(
                        datasetId, force=True, exc=True)
                dataset = datasetCache[datasetId]
                imageFile = Image.originalFile(image)
                imageFileGenerator = File.download(imageFile, headers=False)
                for data in zipGenerator.addFile(
                        imageFileGenerator,
                        path=os.path.join(dataset['name'], imageFile['name'])):
                    yield data

            for dataset in six.viewvalues(datasetCache):
                licenseText = mail_utils.renderTemplate(
                    'license_%s.mako' % dataset['meta']['license'])
                attributionText = mail_utils.renderTemplate(
                    'attribution_%s.mako' % dataset['meta']['license'],
                    {
                        'work': dataset['name'],
                        'author':
                            dataset['meta']['attribution']
                            if not dataset['meta']['anonymous']
                            else 'Anonymous'
                    })
                for data in zipGenerator.addFile(
                        lambda: [licenseText],
                        path=os.path.join(dataset['name'], 'license.txt')):
                    yield data
                for data in zipGenerator.addFile(
                        lambda: [attributionText],
                        path=os.path.join(dataset['name'], 'attribution.txt')):
                    yield data

            yield zipGenerator.footer()

        setResponseHeader('Content-Type', 'application/zip')
        setResponseHeader(
            'Content-Disposition',
            'attachment; filename="%s.zip"' % downloadFileName)
        return stream

    @describeRoute(
        Description('Return histograms of image metadata.')
        .param('filter',
               'Get the histogram after the results of this filter. ' +
               'TODO: describe our filter grammar (an AST tree).',
               required=False)
        .errorResponse()
    )
    @access.cookie
    @access.public
    def getHistogram(self, params):
{'fraction_non_alphanumeric': 0.1281288053037478, 'fraction_numerical': 0.0013529968881071574, 'mean_word_length': 1.990088996763754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '876055', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3593, 'n_words': 996}