<commit_before>import os

import gdal
import mapnik
import osr

from config import to_file, to_path
from layer import Layer


def create(layer_dict):
    layer = _from_dict(layer_dict)
    layer._update_file()
    _build_overviews(layer)
    return layer


def band_count(raster_file):
    return _open_readonly(raster_file).RasterCount


def read_nodata(raster_file):
    ds = _open_readonly(raster_file)
    return ds.GetRasterBand(1).GetNoDataValue()


def band_info(raster_file, band_index, nodata):
    ds = _open_writable(raster_file)
    band = ds.GetRasterBand(band_index)
    file_nodata = band.GetNoDataValue()
    try:
        if nodata:
            if nodata != file_nodata:
                band.SetNoDataValue(nodata)
        else:
            band.DeleteNoDataValue()

        (min, max, mean, std_dev) = band.ComputeStatistics(False)
        histogram = band.GetHistogram(min, max, 256, approx_ok=0)

    finally:
        if file_nodata:
            band.SetNoDataValue(file_nodata)
        elif band.GetNoDataValue():
            band.DeleteNoDataValue()
        if nodata != file_nodata:
            _remove_stats(ds)  # Stats doesn't match the file's nodata value, delete them
    return {
        'min': min,
        'max': max,
        'mean': mean,
        'stdDev': std_dev,
        'histogram': histogram
    }


def _open_writable(raster_file):
    for retry in range(1, 3):
        ds = gdal.OpenShared(raster_file, gdal.GA_Update)
        if ds:
            return ds
    raise Exception('Unable to open file: ' + str(raster_file))


def _open_readonly(raster_file):
    for retry in range(1, 3):
        ds = gdal.OpenShared(raster_file)
        if ds:
            return ds
    raise Exception('Unable to open file: ' + str(raster_file))


def _remove_stats(ds):
    files_to_delete = [
        related_file
        for related_file in ds.GetFileList()
        if related_file.endswith('.aux.xml')]
    for file_to_delete in files_to_delete:
        _remove_file(file_to_delete)


def _remove_file(file_to_delete):
    try:
        os.remove(file_to_delete)
    except:
        pass


def _build_overviews(layer):
    ds = _open_readonly(layer.file)
    bands_without_index = [
        index
        for index in range(1, band_count(layer.file))
        if not ds.GetRasterBand(index).GetOverviewCount()
    ]
    if bands_without_index:
        ds.BuildOverviews(resampling='average', overviewlist=[2, 4, 8])


def _from_dict(layer_dict):
    nodata = layer_dict.get('nodata', None)
    if nodata:
        nodata = float(nodata)
    return RasterLayer(
        id=layer_dict['id'],
        file=to_file(layer_dict['path']),
        nodata=nodata,
        bands=_Band.from_dict(layer_dict['bands'])
    )


# noinspection PyUnresolvedReferences
class RasterLayer(Layer):
    """Represents a map layer."""

    concurrent = False

    def __init__(self, id, file, nodata, bands):
        """Creates a raster layer.

        :param file: Path to raster image.
        :type file: str

        :param bands: Iterable of gwv.Band instances to include.
        :type bands: iterable
        """
        super(RasterLayer, self).__init__()
        self.id = id
        self.file = file
        self.nodata = nodata
        # self._add_overview(bands)
        self.band_layers = [
            _BandLayer(file, band, nodata)
            for band in bands]

    def append_to(self, map):
        """Appends Mapnik layers and styles for the bands in this layer.

        :param map: The mapnik Map.
        :type map: mapnik.Map
        """
        for band_layer in self.band_layers:
            band_layer.append_to(map)

    def to_dict(self):
        return {
            'id': self.id,
            'type': 'raster',
            'path': to_path(self.file),
            'nodata': self.nodata,
            'bands': [band_layer.band.to_dict() for band_layer in self.band_layers],
            'bounds': self.bounds()
        }

    def update(self, layer_dict):
        layer = self.to_dict()
        layer.update(layer_dict)
        updated_layer = _from_dict(layer)
        updated_layer._update_file()
        return updated_layer

    def features(self, lat, lng):
        return [
            self.band_value(band_layer, lat, lng)
            for band_layer in self.band_layers
        ]

    def band_value(self, band_layer, lat, lng):
        features = self.layer_features(band_layer.band.index - 1, lat, lng)
        if features:
            return features[0]['value']
        return None

    def _update_file(self):
        ds = _open_writable(self.file)
        for band_index in range(1, band_count(self.file)):
            band = ds.GetRasterBand(band_index)
            if self.nodata:
                band.SetNoDataValue(self.nodata)
            else:
                band.DeleteNoDataValue()

    def _flatten(self, items):
        if items == []:
            return items
        if isinstance(items[0], list):
            return self._flatten(items[0]) + self._flatten(items[1:])
        return items[:1] + self._flatten(items[1:])


class _BandLayer(object):
    def __init__(self, file, band, nodata):
        self.file = file
        self.band = band
        self.style = self.create_style()
        self.mapnik_datasource = mapnik.Gdal(
            file=file,
            band=band.index,
            nodata=nodata,<commit_msg>Passed index of layer instead of band number when getting raster features.<commit_after>import os

import gdal
import mapnik
import osr

from config import to_file, to_path
from layer import Layer


def create(layer_dict):
    layer = _from_dict(layer_dict)
    layer._update_file()
    _build_overviews(layer)
    return layer


def band_count(raster_file):
    return _open_readonly(raster_file).RasterCount


def read_nodata(raster_file):
    ds = _open_readonly(raster_file)
    return ds.GetRasterBand(1).GetNoDataValue()


def band_info(raster_file, band_index, nodata):
    ds = _open_writable(raster_file)
    band = ds.GetRasterBand(band_index)
    file_nodata = band.GetNoDataValue()
    try:
        if nodata:
            if nodata != file_nodata:
                band.SetNoDataValue(nodata)
        else:
            band.DeleteNoDataValue()

        (min, max, mean, std_dev) = band.ComputeStatistics(False)
        histogram = band.GetHistogram(min, max, 256, approx_ok=0)

    finally:
        if file_nodata:
            band.SetNoDataValue(file_nodata)
        elif band.GetNoDataValue():
            band.DeleteNoDataValue()
        if nodata != file_nodata:
            _remove_stats(ds)  # Stats doesn't match the file's nodata value, delete them
    return {
        'min': min,
        'max': max,
        'mean': mean,
        'stdDev': std_dev,
        'histogram': histogram
    }


def _open_writable(raster_file):
    for retry in range(1, 3):
        ds = gdal.OpenShared(raster_file, gdal.GA_Update)
        if ds:
            return ds
    raise Exception('Unable to open file: ' + str(raster_file))


def _open_readonly(raster_file):
    for retry in range(1, 3):
        ds = gdal.OpenShared(raster_file)
        if ds:
            return ds
    raise Exception('Unable to open file: ' + str(raster_file))


def _remove_stats(ds):
    files_to_delete = [
        related_file
        for related_file in ds.GetFileList()
        if related_file.endswith('.aux.xml')]
    for file_to_delete in files_to_delete:
        _remove_file(file_to_delete)


def _remove_file(file_to_delete):
    try:
        os.remove(file_to_delete)
    except:
        pass


def _build_overviews(layer):
    ds = _open_readonly(layer.file)
    bands_without_index = [
        index
        for index in range(1, band_count(layer.file))
        if not ds.GetRasterBand(index).GetOverviewCount()
    ]
    if bands_without_index:
        ds.BuildOverviews(resampling='average', overviewlist=[2, 4, 8])


def _from_dict(layer_dict):
    nodata = layer_dict.get('nodata', None)
    if nodata:
        nodata = float(nodata)
    return RasterLayer(
        id=layer_dict['id'],
        file=to_file(layer_dict['path']),
        nodata=nodata,
        bands=_Band.from_dict(layer_dict['bands'])
    )


# noinspection PyUnresolvedReferences
class RasterLayer(Layer):
    """Represents a map layer."""

    concurrent = False

    def __init__(self, id, file, nodata, bands):
        """Creates a raster layer.

        :param file: Path to raster image.
        :type file: str

        :param bands: Iterable of gwv.Band instances to include.
        :type bands: iterable
        """
        super(RasterLayer, self).__init__()
        self.id = id
        self.file = file
        self.nodata = nodata
        # self._add_overview(bands)
        self.band_layers = [
            _BandLayer(file, band, nodata)
            for band in bands]

    def append_to(self, map):
        """Appends Mapnik layers and styles for the bands in this layer.

        :param map: The mapnik Map.
        :type map: mapnik.Map
        """
        for band_layer in self.band_layers:
            band_layer.append_to(map)

    def to_dict(self):
        return {
            'id': self.id,
            'type': 'raster',
            'path': to_path(self.file),
            'nodata': self.nodata,
            'bands': [band_layer.band.to_dict() for band_layer in self.band_layers],
            'bounds': self.bounds()
        }

    def update(self, layer_dict):
        layer = self.to_dict()
        layer.update(layer_dict)
        updated_layer = _from_dict(layer)
        updated_layer._update_file()
        return updated_layer

    def features(self, lat, lng):

        return [
            self.band_value(index, lat, lng)
            for index, band_layer in enumerate(self.band_layers)
        ]

    def band_value(self, band_index, lat, lng):
        features = self.layer_features(band_index, lat, lng)
        if features:
            return features[0]['value']
        return None

    def _update_file(self):
        ds = _open_writable(self.file)
        for band_index in range(1, band_count(self.file)):
            band = ds.GetRasterBand(band_index)
            if self.nodata:
                band.SetNoDataValue(self.nodata)
            else:
                band.DeleteNoDataValue()

    def _flatten(self, items):
        if items == []:
            return items
        if isinstance(items[0], list):
            return self._flatten(items[0]) + self._flatten(items[1:])
        return items[:1] + self._flatten(items[1:])


class _BandLayer(object):
    def __init__(self, file, band, nodata):
        self.file = file
        self.band = band
        self.style = self.create_style()
        self.mapnik_datasource = mapnik.Gdal(
            file=file,
            band=band.index,
            nodata=nodata,
{'fraction_non_alphanumeric': 0.09974069272087424, 'fraction_numerical': 0.0037969994443415446, 'mean_word_length': 2.2783849423193687, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4882545', 'n_tokens_mistral': 3478, 'n_tokens_neox': 3378, 'n_words': 774}