<commit_before># -*- coding: utf-8 -*-

#############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#############################################################################

import json
import mapnik
import math
import PIL.Image
import palettable
import pyproj
import six
import struct
import threading
from operator import attrgetter
from osgeo import gdal
from osgeo import gdalconst
from osgeo import osr
from pkg_resources import DistributionNotFound, get_distribution

from large_image import config
from large_image.cache_util import LruCacheMetaclass, methodcache, CacheProperties
from large_image.constants import SourcePriority, TileInputUnits
from large_image.exceptions import TileSourceException
from large_image.tilesource import FileTileSource, TILE_FORMAT_PIL


try:
    __version__ = get_distribution(__name__).version
except DistributionNotFound:
    # package is not installed
    pass


mapnik.logger.set_severity(mapnik.severity_type.Debug)


TileInputUnits['projection'] = 'projection'
TileInputUnits['proj'] = 'projection'
TileInputUnits['wgs84'] = 'proj4:EPSG:4326'
TileInputUnits['4326'] = 'proj4:EPSG:4326'

# Inform the tile source cache about the potential size of this tile source
CacheProperties['tilesource']['itemExpectedSize'] = max(
    CacheProperties['tilesource']['itemExpectedSize'],
    100 * 1024 ** 2)

# Used to cache pixel size for projections
ProjUnitsAcrossLevel0 = {}
ProjUnitsAcrossLevel0_MaxSize = 100

InitPrefix = '+init='


@six.add_metaclass(LruCacheMetaclass)
class MapnikFileTileSource(FileTileSource):
    """
    Provides tile access to geospatial files.
    """

    cacheName = 'tilesource'
    name = 'mapnikfile'
    extensions = {
        None: SourcePriority.MEDIUM,
        'nc': SourcePriority.PREFERRED,  # netcdf
        # National Imagery Transmission Format
        'ntf': SourcePriority.PREFERRED,
        'nitf': SourcePriority.PREFERRED,
        'tif': SourcePriority.LOW,
        'tiff': SourcePriority.LOW,
        'vrt': SourcePriority.PREFERRED,
    }
    mimeTypes = {
        None: SourcePriority.FALLBACK,
        'image/geotiff': SourcePriority.PREFERRED,
        'image/tiff': SourcePriority.LOW,
        'image/x-tiff': SourcePriority.LOW,
    }

    def __init__(self, path, projection=None, style=None, unitsPerPixel=None, **kwargs):
        """
        Initialize the tile class.  See the base class for other available
        parameters.

        :param path: a filesystem path for the tile source.
        :param projection: None to use pixel space, otherwise a proj4
            projection string or a case-insensitive string of the form
            'EPSG:<epsg number>'.  If a string and case-insensitively prefixed
            with 'proj4:', that prefix is removed.  For instance,
            'proj4:EPSG:3857', 'PROJ4:+init=epsg:3857', and '+init=epsg:3857',
            and 'EPSG:3857' are all equivilant.
        :param style: if None, use the default style for the file.  Otherwise,
            this is a string with a json-encoded dictionary.  The style is
            ignored if it does not contain 'band' or 'bands'.  The style can
            contain the following keys:
                band: either -1 for the default band(s), a 1-based value for
                    the band to use for styling, or a string that matches the
                    interpretation of the band ('red', 'green', 'blue', gray',
                    etc.).
                min: the value of the band to map to the first palette value.
                    Defaults to 0.  'auto' to use 0 if the reported minimum and
                    maximum of the band are between [0, 255] or use the
                    reported minimum otherwise.  'min' or 'max' to always uses
                    the reported minimum or maximum.
                max: the value of the band to map to the last palette value.
                    Defaults to 255.  'auto' to use 0 if the reported minimum
                    and maximum of the band are between [0, 255] or use the
                    reported maximum otherwise.  'min' or 'max' to always uses
                    the reported minimum or maximum.
                scheme: one of the mapnik.COLORIZER_xxx values.  Case
                    insensitive.  Possible values are at least 'discrete',
                    'linear', and 'exact'.  If palette is unspecified and the
                    band was specified as a string, this defaults to 'linear'.
                    Otherwise it defaults to 'discrete'.
                palette: either a list of two or more color strings, a string
                    with a dotted class name from the python palettable
                    package, or 'colortable' to use the band's color table
                    (palette).  Color strings must be parsable by mapnik's
                    Color class.  Many css strings work.  If scheme is
                    unspecified and the band is one of 'red', 'green', 'blue',
                    gray', or 'alpha', this defaults to an appropriate band
                    pair.  Otherwise, this defaults to the band's color table
                    (palette) if it has one and 'cmocean.diverging.Curl_10' if
                    it does not.
                nodata: the value to use for missing data.  'auto' to use the
                    band reported value, if any.  null or unset to not use a
                    nodata value.
                composite: this is a string containing one of the mapnik
                    CompositeOp properties.  It defaults to 'lighten'.
            Alternately, the style object can contain a single key of 'bands',
            which has a value which is a list of style dictionaries as above,
            excepting that each must have a band that is not -1.  Bands are
            composited in the order listed.
        :param unitsPerPixel: The size of a pixel at the 0 tile size.  Ignored
            if the projection is None.  For projections, None uses the default,
            which is the distance between (-180,0) and (180,0) in EPSG:4326
            converted to the projection divided by the tile size.  Proj4
            projections that are not latlong (is_geographic is False) must
            specify unitsPerPixel.
        """
        super(MapnikFileTileSource, self).__init__(path, **kwargs)
        self._bounds = {}
        self._path = self._getLargeImagePath()
        self.dataset = gdal.Open(self._path, gdalconst.GA_ReadOnly)
        self._getDatasetLock = threading.RLock()
        self.tileSize = 256
        self.tileWidth = self.tileSize
        self.tileHeight = self.tileSize
        self._projection = projection
        if projection and projection.lower().startswith('epsg:'):
            projection = InitPrefix + projection.lower()
        if projection and not isinstance(projection, six.binary_type):
            projection = projection.encode('utf8')
        self.projection = projection
        self._jsonstyle = style
        if style:
            try:
                self.style = json.loads(style)
                if not isinstance(self.style, dict):
                    raise TypeError
            except TypeError:
                raise TileSourceException('Style is not a valid json object.')

        try:
            with self._getDatasetLock:
                self.sourceSizeX = self.sizeX = self.dataset.RasterXSize
                self.sourceSizeY = self.sizeY = self.dataset.RasterYSize
        except AttributeError:
            raise TileSourceException('File cannot be opened via Mapnik.')<commit_msg>Guard against GDAL throwing a runtime error on open.<commit_after># -*- coding: utf-8 -*-

#############################################################################
#  Copyright Kitware Inc.
#
#  Licensed under the Apache License, Version 2.0 ( the "License" );
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#############################################################################

import json
import mapnik
import math
import PIL.Image
import palettable
import pyproj
import six
import struct
import threading
from operator import attrgetter
from osgeo import gdal
from osgeo import gdalconst
from osgeo import osr
from pkg_resources import DistributionNotFound, get_distribution

from large_image import config
from large_image.cache_util import LruCacheMetaclass, methodcache, CacheProperties
from large_image.constants import SourcePriority, TileInputUnits
from large_image.exceptions import TileSourceException
from large_image.tilesource import FileTileSource, TILE_FORMAT_PIL


try:
    __version__ = get_distribution(__name__).version
except DistributionNotFound:
    # package is not installed
    pass


mapnik.logger.set_severity(mapnik.severity_type.Debug)


TileInputUnits['projection'] = 'projection'
TileInputUnits['proj'] = 'projection'
TileInputUnits['wgs84'] = 'proj4:EPSG:4326'
TileInputUnits['4326'] = 'proj4:EPSG:4326'

# Inform the tile source cache about the potential size of this tile source
CacheProperties['tilesource']['itemExpectedSize'] = max(
    CacheProperties['tilesource']['itemExpectedSize'],
    100 * 1024 ** 2)

# Used to cache pixel size for projections
ProjUnitsAcrossLevel0 = {}
ProjUnitsAcrossLevel0_MaxSize = 100

InitPrefix = '+init='


@six.add_metaclass(LruCacheMetaclass)
class MapnikFileTileSource(FileTileSource):
    """
    Provides tile access to geospatial files.
    """

    cacheName = 'tilesource'
    name = 'mapnikfile'
    extensions = {
        None: SourcePriority.MEDIUM,
        'nc': SourcePriority.PREFERRED,  # netcdf
        # National Imagery Transmission Format
        'ntf': SourcePriority.PREFERRED,
        'nitf': SourcePriority.PREFERRED,
        'tif': SourcePriority.LOW,
        'tiff': SourcePriority.LOW,
        'vrt': SourcePriority.PREFERRED,
    }
    mimeTypes = {
        None: SourcePriority.FALLBACK,
        'image/geotiff': SourcePriority.PREFERRED,
        'image/tiff': SourcePriority.LOW,
        'image/x-tiff': SourcePriority.LOW,
    }

    def __init__(self, path, projection=None, style=None, unitsPerPixel=None, **kwargs):  # noqa
        """
        Initialize the tile class.  See the base class for other available
        parameters.

        :param path: a filesystem path for the tile source.
        :param projection: None to use pixel space, otherwise a proj4
            projection string or a case-insensitive string of the form
            'EPSG:<epsg number>'.  If a string and case-insensitively prefixed
            with 'proj4:', that prefix is removed.  For instance,
            'proj4:EPSG:3857', 'PROJ4:+init=epsg:3857', and '+init=epsg:3857',
            and 'EPSG:3857' are all equivilant.
        :param style: if None, use the default style for the file.  Otherwise,
            this is a string with a json-encoded dictionary.  The style is
            ignored if it does not contain 'band' or 'bands'.  The style can
            contain the following keys:
                band: either -1 for the default band(s), a 1-based value for
                    the band to use for styling, or a string that matches the
                    interpretation of the band ('red', 'green', 'blue', gray',
                    etc.).
                min: the value of the band to map to the first palette value.
                    Defaults to 0.  'auto' to use 0 if the reported minimum and
                    maximum of the band are between [0, 255] or use the
                    reported minimum otherwise.  'min' or 'max' to always uses
                    the reported minimum or maximum.
                max: the value of the band to map to the last palette value.
                    Defaults to 255.  'auto' to use 0 if the reported minimum
                    and maximum of the band are between [0, 255] or use the
                    reported maximum otherwise.  'min' or 'max' to always uses
                    the reported minimum or maximum.
                scheme: one of the mapnik.COLORIZER_xxx values.  Case
                    insensitive.  Possible values are at least 'discrete',
                    'linear', and 'exact'.  If palette is unspecified and the
                    band was specified as a string, this defaults to 'linear'.
                    Otherwise it defaults to 'discrete'.
                palette: either a list of two or more color strings, a string
                    with a dotted class name from the python palettable
                    package, or 'colortable' to use the band's color table
                    (palette).  Color strings must be parsable by mapnik's
                    Color class.  Many css strings work.  If scheme is
                    unspecified and the band is one of 'red', 'green', 'blue',
                    gray', or 'alpha', this defaults to an appropriate band
                    pair.  Otherwise, this defaults to the band's color table
                    (palette) if it has one and 'cmocean.diverging.Curl_10' if
                    it does not.
                nodata: the value to use for missing data.  'auto' to use the
                    band reported value, if any.  null or unset to not use a
                    nodata value.
                composite: this is a string containing one of the mapnik
                    CompositeOp properties.  It defaults to 'lighten'.
            Alternately, the style object can contain a single key of 'bands',
            which has a value which is a list of style dictionaries as above,
            excepting that each must have a band that is not -1.  Bands are
            composited in the order listed.
        :param unitsPerPixel: The size of a pixel at the 0 tile size.  Ignored
            if the projection is None.  For projections, None uses the default,
            which is the distance between (-180,0) and (180,0) in EPSG:4326
            converted to the projection divided by the tile size.  Proj4
            projections that are not latlong (is_geographic is False) must
            specify unitsPerPixel.
        """
        super(MapnikFileTileSource, self).__init__(path, **kwargs)
        self._bounds = {}
        self._path = self._getLargeImagePath()
        try:
            self.dataset = gdal.Open(self._path, gdalconst.GA_ReadOnly)
        except RuntimeError:
            raise TileSourceException('File cannot be opened via GDAL')
        self._getDatasetLock = threading.RLock()
        self.tileSize = 256
        self.tileWidth = self.tileSize
        self.tileHeight = self.tileSize
        self._projection = projection
        if projection and projection.lower().startswith('epsg:'):
            projection = InitPrefix + projection.lower()
        if projection and not isinstance(projection, six.binary_type):
            projection = projection.encode('utf8')
        self.projection = projection
        self._jsonstyle = style
        if style:
            try:
                self.style = json.loads(style)
                if not isinstance(self.style, dict):
                    raise TypeError
            except TypeError:
                raise TileSourceException('Style is not a valid json object.')

        try:
            with self._getDatasetLock:
                self.sourceSizeX = self.sizeX = self.dataset.RasterXSize
                self.sourceSizeY = self.sizeY = self.dataset.RasterYSize
        except AttributeError:
            raise TileSourceException('File cannot be opened via Mapnik.')
{'fraction_non_alphanumeric': 0.08587843967342002, 'fraction_numerical': 0.011006954944058059, 'mean_word_length': 2.335888642323986, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4038422', 'n_tokens_mistral': 4416, 'n_tokens_neox': 4194, 'n_words': 1665}