<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import itertools
import math
import six
from six import BytesIO
from six.moves import range

from .base import FileTileSource, TileSourceException
from ..cache_util import LruCacheMetaclass, methodcache
from .tiff_reader import TiledTiffDirectory, TiffException, \
    InvalidOperationTiffException, IOTiffException, ValidationTiffException

try:
    import girder
    from girder import logger
    from .base import GirderTileSource, TILE_FORMAT_PIL
except ImportError:
    girder = None
    import logging as logger

try:
    import PIL.Image
except ImportError:
    PIL = None


@six.add_metaclass(LruCacheMetaclass)
class TiffFileTileSource(FileTileSource):
    """
    Provides tile access to TIFF files.
    """
    cacheName = 'tilesource'
    name = 'tifffile'

    def __init__(self, item, **kwargs):
        super(TiffFileTileSource, self).__init__(item, **kwargs)

        largeImagePath = self._getLargeImagePath()
        lastException = None

        alldir = []
        for directoryNum in itertools.count():
            try:
                td = TiledTiffDirectory(largeImagePath, directoryNum)
            except ValidationTiffException as lastException:
                continue
            except TiffException as lastException:
                break
            if not td.tileWidth or not td.tileHeight:
                continue
            level = int(math.ceil(math.log(max(
                float(td.imageWidth) / td.tileWidth,
                float(td.imageHeight) / td.tileHeight)) / math.log(2)))
            alldir.append((td.tileWidth * td.tileHeight, level, td))
        if not len(alldir):
            msg = 'File %s didn\'t meet requirements for tile source: %s' % (
                largeImagePath, lastException)
            logger.debug(msg)
            raise TileSourceException(msg)
        alldir.sort()
        highest = alldir[-1][-1]
        directories = {}
        for dir in alldir:
            td = dir[-1]<commit_msg>Add more comments to the tif directory processing.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

###############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
###############################################################################

import itertools
import math
import six
from six import BytesIO
from six.moves import range

from .base import FileTileSource, TileSourceException
from ..cache_util import LruCacheMetaclass, methodcache
from .tiff_reader import TiledTiffDirectory, TiffException, \
    InvalidOperationTiffException, IOTiffException, ValidationTiffException

try:
    import girder
    from girder import logger
    from .base import GirderTileSource, TILE_FORMAT_PIL
except ImportError:
    girder = None
    import logging as logger

try:
    import PIL.Image
except ImportError:
    PIL = None


@six.add_metaclass(LruCacheMetaclass)
class TiffFileTileSource(FileTileSource):
    """
    Provides tile access to TIFF files.
    """
    cacheName = 'tilesource'
    name = 'tifffile'

    def __init__(self, item, **kwargs):
        super(TiffFileTileSource, self).__init__(item, **kwargs)

        largeImagePath = self._getLargeImagePath()
        lastException = None

        # Query all know directories in the tif file.  Only keep track of
        # directories that contain tiled images.
        alldir = []
        for directoryNum in itertools.count():
            try:
                td = TiledTiffDirectory(largeImagePath, directoryNum)
            except ValidationTiffException as lastException:
                continue
            except TiffException as lastException:
                break
            if not td.tileWidth or not td.tileHeight:
                continue
            # Calculate the tile level, where 0 is a single tile, 1 is up to a
            # set of 2x2 tiles, 2 is 4x4, etc.
            level = int(math.ceil(math.log(max(
                float(td.imageWidth) / td.tileWidth,
                float(td.imageHeight) / td.tileHeight)) / math.log(2)))
            # Store information for sorting with the directory.
            alldir.append((td.tileWidth * td.tileHeight, level, td))
        # If there are no tiled images, raise an exception.
        if not len(alldir):
            msg = 'File %s didn\'t meet requirements for tile source: %s' % (
                largeImagePath, lastException)
            logger.debug(msg)
            raise TileSourceException(msg)
        # Sort the known directories by image area (width * height).  Given
        # equal area, sort by the level.
        alldir.sort()
        # The highest resolution image is our preferred image
        highest = alldir[-1][-1]
        directories = {}
        # Discard any images that use a different tiling scheme than our
        # preferred image
        for dir in alldir:
            td = dir[-1]
{'fraction_non_alphanumeric': 0.12226512226512226, 'fraction_numerical': 0.004021879021879022, 'mean_word_length': 2.733933933933934, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '208429', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1633, 'n_words': 565}