<commit_before>import json
import os

from bson import ObjectId
from bson.errors import InvalidId
import geojson

from girder.api import access
from girder.api.describe import autoDescribeRoute, describeRoute, Description
from girder.api.rest import loadmodel, setRawResponse, setResponseHeader
from girder.constants import AccessType
from girder.exceptions import GirderException, RestException, ValidationException
from girder.models.file import File
from girder.utility import mail_utils, ziputil
from girder_large_image.models.image_item import ImageItem
from large_image.exceptions import TileGeneralException

from .base import IsicResource
from ..models.dataset import Dataset
from ..models.image import Image
from ..models.segmentation import Segmentation
from ..models.user import User
from ..utility import querylang


class ImageResource(IsicResource):
    def __init__(self,):
        super(ImageResource, self).__init__()
        self.resourceName = 'image'

        self.route('GET', (), self.find)
        self.route('GET', ('download',), self.downloadMultiple)
        self.route('GET', ('histogram',), self.getHistogram)
        self.route('GET', (':id',), self.getImage)
        self.route('GET', (':id', 'thumbnail'), self.thumbnail)
        self.route('GET', (':id', 'tiles'), self.getTileInfo)
        self.route('GET', (':id', 'tiles', ':z', ':x', ':y'), self.getTile)
        self.route('GET', (':id', 'download'), self.download)
        self.route('GET', (':id', 'superpixels'), self.getSuperpixels)

        self.route('POST', (':id', 'segment'), self.doSegmentation)
        self.route('POST', (':id', 'metadata'), self.applyMetadata)

    def _parseFilter(self, filterParam):
        if isinstance(filterParam, str):
            try:
                filterParam = json.loads(filterParam)
            except ValueError as e:
                raise ValidationException(f'Could not decode JSON: {str(e)}', 'filter')
        try:
            return querylang.astToMongo(filterParam)
        except (TypeError, ValueError, InvalidId) as e:
            raise ValidationException(f'Could not parse filter: {str(e)}', 'filter')

    def _parseImageIds(self, imageIdsParam):
        if isinstance(imageIdsParam, str):
            try:
                imageIdsParam = json.loads(imageIdsParam)
            except ValueError as e:
                raise ValidationException(f'Could not decode JSON: {str(e)}', 'imageIds')
        if not isinstance(imageIdsParam, list):
            raise ValidationException('"imageIds" must be a JSON-encoded array.', 'imageIds')
        if len(imageIdsParam) > 300:
            raise ValidationException('"imageIds" may not contain more than 300 IDs.', 'imageIds')
        imageIds = []
        for imageId in imageIdsParam:
            if not isinstance(imageId, str):
                raise ValidationException('Each of "imageIds" must be a string.', 'imageIds')
            try:
                imageId = ObjectId(imageId)
            except InvalidId:
                raise ValidationException('Invalid ObjectId.', 'imageIds')
            imageIds.append(imageId)
        # TODO: We could validate that the user has access to all of the requested IDs, and
        # raise a 403 error if now. Currently, we will silently exclude images that the user doesn't
        # have permission to READ.

        query = {
            '_id': {'$in': imageIds}
        }
        return query

    @describeRoute(
        Description('Return a list of lesion images.')
        .pagingParams(defaultSort='name')
        .param('detail', 'Display the full information for each image, instead of a summary.',
               required=False, dataType='boolean', default=False)
        .param('name', 'Find an image with a specific name.', required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar.',
               required=False)
        .param('imageIds',
               'A JSON array of up to 300 image IDs, which will be exclusively included.',
               required=False)
        .errorResponse()
    )
    @access.public(cookie=True)
    def find(self, params):
        user = self.getCurrentUser()
        detail = self.boolParam('detail', params, default=False)
        limit, offset, sort = self.getPagingParameters(params, 'name')

        if sum(option in params for option in {'name', 'filter', 'imageIds'}) > 1:
            raise RestException('Only one of "name", "filter", or "imageIds" may be specified.')

        if 'name' in params:
            query = {'name': params['name']}
        elif 'filter' in params:
            query = self._parseFilter(params['filter'])
        elif 'imageIds' in params:
            query = self._parseImageIds(params['imageIds'])
        else:
            query = {}
        # Exclude images that haven't passed QC review (though they can still be accessed by more
        # direct endpoints if the user has permission)
        query.update({
            'meta.reviewed.accepted': True
        })

        filterFunc = Image().filter if detail else Image().filterSummary
        return [
            filterFunc(image, user)
            for image in
            Image().filterResultsByPermission(
                Image().find(query, sort=sort),
                user=user, level=AccessType.READ, limit=limit, offset=offset)
        ]

    def _imagesZipGenerator(self, downloadFileName, images, include):
        datasetCache = {}
        zipGenerator = ziputil.ZipGenerator(downloadFileName)

        for image in images:
            datasetId = image['meta']['datasetId']
            if datasetId not in datasetCache:
                datasetCache[datasetId] = Dataset().load(datasetId, force=True, exc=True)
            dataset = datasetCache[datasetId]

            if include in {'all', 'images'}:
                imageFile = Image().originalFile(image)
                imageFileGenerator = File().download(imageFile, headers=False)
                for data in zipGenerator.addFile(
                        imageFileGenerator,
                        path=os.path.join(dataset['name'], imageFile['name'])):
                    yield data
            if include in {'all', 'metadata'}:
                def metadataGenerator():
                    # TODO: Consider replacing this with Image().filter
                    yield json.dumps({
                        '_id': str(image['_id']),
                        'name': image['name'],
                        'meta': {
                            'acquisition': image['meta']['acquisition'],
                            'clinical': image['meta']['clinical']
                        }
                    })
                for data in zipGenerator.addFile(
                        metadataGenerator,
                        path=os.path.join(dataset['name'], f'{image["name"]}.json')):
                    yield data

        for dataset in datasetCache.values():
            licenseText = mail_utils.renderTemplate(
                f'license_{dataset["license"]}.mako')
            attributionText = mail_utils.renderTemplate(
                f'attribution_{dataset["license"]}.mako',
                {
                    'work': dataset['name'],
                    'author': dataset['attribution']
                })
            for data in zipGenerator.addFile(
                    lambda: [licenseText],
                    path=os.path.join(dataset['name'], 'LICENSE.txt')):
                yield data
            for data in zipGenerator.addFile(
                    lambda: [attributionText],
                    path=os.path.join(dataset['name'], 'ATTRIBUTION.txt')):<commit_msg>Output metadata in image download as CSV<commit_after>import csv
import io
import json
import os

from bson import ObjectId
from bson.errors import InvalidId
import geojson

from girder.api import access
from girder.api.describe import autoDescribeRoute, describeRoute, Description
from girder.api.rest import loadmodel, setRawResponse, setResponseHeader
from girder.constants import AccessType
from girder.exceptions import GirderException, RestException, ValidationException
from girder.models.file import File
from girder.utility import mail_utils, ziputil
from girder_large_image.models.image_item import ImageItem
from large_image.exceptions import TileGeneralException

from .base import IsicResource
from ..models.dataset import Dataset
from ..models.image import Image
from ..models.segmentation import Segmentation
from ..models.user import User
from ..utility import querylang


class ImageResource(IsicResource):
    def __init__(self,):
        super(ImageResource, self).__init__()
        self.resourceName = 'image'

        self.route('GET', (), self.find)
        self.route('GET', ('download',), self.downloadMultiple)
        self.route('GET', ('histogram',), self.getHistogram)
        self.route('GET', (':id',), self.getImage)
        self.route('GET', (':id', 'thumbnail'), self.thumbnail)
        self.route('GET', (':id', 'tiles'), self.getTileInfo)
        self.route('GET', (':id', 'tiles', ':z', ':x', ':y'), self.getTile)
        self.route('GET', (':id', 'download'), self.download)
        self.route('GET', (':id', 'superpixels'), self.getSuperpixels)

        self.route('POST', (':id', 'segment'), self.doSegmentation)
        self.route('POST', (':id', 'metadata'), self.applyMetadata)

    def _parseFilter(self, filterParam):
        if isinstance(filterParam, str):
            try:
                filterParam = json.loads(filterParam)
            except ValueError as e:
                raise ValidationException(f'Could not decode JSON: {str(e)}', 'filter')
        try:
            return querylang.astToMongo(filterParam)
        except (TypeError, ValueError, InvalidId) as e:
            raise ValidationException(f'Could not parse filter: {str(e)}', 'filter')

    def _parseImageIds(self, imageIdsParam):
        if isinstance(imageIdsParam, str):
            try:
                imageIdsParam = json.loads(imageIdsParam)
            except ValueError as e:
                raise ValidationException(f'Could not decode JSON: {str(e)}', 'imageIds')
        if not isinstance(imageIdsParam, list):
            raise ValidationException('"imageIds" must be a JSON-encoded array.', 'imageIds')
        if len(imageIdsParam) > 300:
            raise ValidationException('"imageIds" may not contain more than 300 IDs.', 'imageIds')
        imageIds = []
        for imageId in imageIdsParam:
            if not isinstance(imageId, str):
                raise ValidationException('Each of "imageIds" must be a string.', 'imageIds')
            try:
                imageId = ObjectId(imageId)
            except InvalidId:
                raise ValidationException('Invalid ObjectId.', 'imageIds')
            imageIds.append(imageId)
        # TODO: We could validate that the user has access to all of the requested IDs, and
        # raise a 403 error if now. Currently, we will silently exclude images that the user doesn't
        # have permission to READ.

        query = {
            '_id': {'$in': imageIds}
        }
        return query

    @describeRoute(
        Description('Return a list of lesion images.')
        .pagingParams(defaultSort='name')
        .param('detail', 'Display the full information for each image, instead of a summary.',
               required=False, dataType='boolean', default=False)
        .param('name', 'Find an image with a specific name.', required=False)
        .param('filter', 'Filter the images by a PegJS-specified grammar.',
               required=False)
        .param('imageIds',
               'A JSON array of up to 300 image IDs, which will be exclusively included.',
               required=False)
        .errorResponse()
    )
    @access.public(cookie=True)
    def find(self, params):
        user = self.getCurrentUser()
        detail = self.boolParam('detail', params, default=False)
        limit, offset, sort = self.getPagingParameters(params, 'name')

        if sum(option in params for option in {'name', 'filter', 'imageIds'}) > 1:
            raise RestException('Only one of "name", "filter", or "imageIds" may be specified.')

        if 'name' in params:
            query = {'name': params['name']}
        elif 'filter' in params:
            query = self._parseFilter(params['filter'])
        elif 'imageIds' in params:
            query = self._parseImageIds(params['imageIds'])
        else:
            query = {}
        # Exclude images that haven't passed QC review (though they can still be accessed by more
        # direct endpoints if the user has permission)
        query.update({
            'meta.reviewed.accepted': True
        })

        filterFunc = Image().filter if detail else Image().filterSummary
        return [
            filterFunc(image, user)
            for image in
            Image().filterResultsByPermission(
                Image().find(query, sort=sort),
                user=user, level=AccessType.READ, limit=limit, offset=offset)
        ]

    def _imagesZipGenerator(self, downloadFileName, images, include):  # noqa C901
        datasetCache = {}
        zipGenerator = ziputil.ZipGenerator(downloadFileName)

        if include in {'all', 'metadata'}:
            metadataFieldnames = [
                '_id',
                'name'
            ]
            for k in sorted([
                'age_approx', 'anatom_site_general', 'benign_malignant',
                'clin_size_long_diam_mm', 'diagnosis', 'diagnosis_confirm_type',
                'family_hx_mm', 'lesion_id', 'mel_class', 'mel_mitotic_index', 'mel_thick_mm',
                'mel_type', 'mel_ulcer', 'melanocytic', 'nevus_type', 'patient_id',
                'personal_hx_mm', 'sex',
            ]):
                metadataFieldnames.append(f'meta.clinical.{k}')
            for k in sorted([
                'acquisition_day', 'blurry', 'color_tint', 'dermoscopic_type', 'hairy',
                'image_type', 'marker_pen', 'pixelsX', 'pixelsY',
            ]):
                metadataFieldnames.append(f'meta.acquisition.{k}')

            csvStream = io.StringIO()
            csvWriter = csv.DictWriter(csvStream, metadataFieldnames)
            csvWriter.writeheader()

        for image in images:
            datasetId = image['meta']['datasetId']
            if datasetId not in datasetCache:
                datasetCache[datasetId] = Dataset().load(datasetId, force=True, exc=True)
            dataset = datasetCache[datasetId]

            if include in {'all', 'images'}:
                imageFile = Image().originalFile(image)
                imageFileGenerator = File().download(imageFile, headers=False)
                for data in zipGenerator.addFile(
                        imageFileGenerator,
                        path=os.path.join(dataset['name'], imageFile['name'])):
                    yield data
            if include in {'all', 'metadata'}:
                flattenedMetadata = {
                    '_id': str(image['_id']),
                    'name': image['name']
                }
                for metaType in ['acquisition', 'clinical']:
                    for k, v in image['meta'][metaType].items():
                        flattenedMetadata[f'meta.{metaType}.{k}'] = v
                csvWriter.writerow(flattenedMetadata)

        if include in {'all', 'metadata'}:
            csvStream.seek(0)
            csvStreamChunks = iter(csvStream.readline, '')
            for data in zipGenerator.addFile(
                # Girder expects a callable function instead of an actual generator
                lambda: csvStreamChunks,
                path='metadata.csv'
            ):
                yield data

        for dataset in datasetCache.values():
            licenseText = mail_utils.renderTemplate(
                f'license_{dataset["license"]}.mako')
            attributionText = mail_utils.renderTemplate(
                f'attribution_{dataset["license"]}.mako',
                {
                    'work': dataset['name'],
                    'author': dataset['attribution']
                })
            for data in zipGenerator.addFile(
                    lambda: [licenseText],
                    path=os.path.join(dataset['name'], 'LICENSE.txt')):
                yield data
            for data in zipGenerator.addFile(
                    lambda: [attributionText],
                    path=os.path.join(dataset['name'], 'ATTRIBUTION.txt')):
{'fraction_non_alphanumeric': 0.11286121039558974, 'fraction_numerical': 0.0018174107954201247, 'mean_word_length': 2.193036750483559, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2195296', 'n_tokens_mistral': 4388, 'n_tokens_neox': 4165, 'n_words': 1181}