<commit_before>from collections import Counter

import numpy

from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Polygon
from django.db import connection

from .const import WEB_MERCATOR_SRID
from .formulas import FormulaParser, RasterAlgebraParser
from .rasterize import rasterize
from .tiler import tile_index_range


CLIPPED_VALUE_COUNT_SQL = """
WITH tiles_for_agg AS (
    SELECT ST_ValueCount(ST_Clip(ST_Transform(rast, {geom_srid}), ST_GeomFromEWKT('{geom_ewkt}'))) AS vcresult
    FROM raster_rastertile
    WHERE ST_Intersects(rast, ST_Transform(ST_GeomFromEWKT('{geom_ewkt}'), {rast_srid}))
    AND rasterlayer_id = {rasterlayer_id}
    AND tilez = {zoom}
)
SELECT (vcresult).value, SUM((vcresult).count) AS count
FROM tiles_for_agg
GROUP BY (vcresult).value
"""

GLOBAL_VALUE_COUNT_SQL = """
WITH tiles_for_agg AS (
    SELECT ST_ValueCount(rast) AS vcresult
    FROM raster_rastertile
    WHERE rasterlayer_id = {rasterlayer_id}
    AND tilez = {zoom}
)
SELECT (vcresult).value, SUM((vcresult).count) AS count
FROM tiles_for_agg
GROUP BY (vcresult).value
"""

MINSIZE_SQL = """
SELECT
    ST_ScaleX(ST_Transform(rast, {srid})) AS scalex,
    ST_ScaleY(ST_Transform(rast, {srid})) AS scaley
FROM raster_rastertile
WHERE rasterlayer_id = {rasterlayer_id}
AND tilez = {zoom}
LIMIT 1
"""

MAX_ZOOM_SQL = """
SELECT MAX(tilez)
FROM raster_rastertile
WHERE rasterlayer_id={rasterlayer_id}
"""


class RasterAggregationException(Exception):
    pass


def aggregator(layer_dict, zoom=None, geom=None, formula=None, acres=True, grouping='auto'):
    """
    Compute aggregate statistics for a layers dictionary, potentially for
    an algebra expression and clipped by a geometry.

    The grouping parameter specifies how to group the pixel values for the
    aggregation count.

    Allowed are the following options:

    * 'auto' (the default). The output will be grouped by unique values if all
      input rasters are discrete, otherwise a numpy histogram is used.
    * 'discrete' groups the data will be grouped by unique values
    * 'continuous' groups the data in a numpy histogram
    * If an integer value is passed to the argument, it is interpreted as a
      legend_id. The data will be grouped using the legend expressions. For
      For instance, use grouping=23 for grouping the output with legend 23.
    """
    from .models import RasterLayer, RasterTile, Legend

    algebra_parser = RasterAlgebraParser()

    # Get layers
    layers = RasterLayer.objects.filter(id__in=layer_dict.values())

    # Compute zoom if not provided
    if zoom is None:
        zoom = min(layers.values_list('metadata__max_zoom', flat=True))

    # Compute tilerange for this area and the given zoom level
    if geom:
        # Transform geom to web mercator
        if geom.srid != WEB_MERCATOR_SRID:
            geom.transform(WEB_MERCATOR_SRID)

        # Clip against max extent for limiting nr of tiles.
        # This is important for requests on large areas for small rasters.
        max_extent = MultiPolygon([Polygon.from_bbox(lyr.extent()) for lyr in layers]).envelope
        max_extent = geom.intersection(max_extent)<commit_msg>Added empty return if for aggregation there is no overlay<commit_after>from collections import Counter

import numpy

from django.contrib.gis.geos import GEOSGeometry, MultiPolygon, Polygon
from django.db import connection

from .const import WEB_MERCATOR_SRID
from .formulas import FormulaParser, RasterAlgebraParser
from .rasterize import rasterize
from .tiler import tile_index_range


CLIPPED_VALUE_COUNT_SQL = """
WITH tiles_for_agg AS (
    SELECT ST_ValueCount(ST_Clip(ST_Transform(rast, {geom_srid}), ST_GeomFromEWKT('{geom_ewkt}'))) AS vcresult
    FROM raster_rastertile
    WHERE ST_Intersects(rast, ST_Transform(ST_GeomFromEWKT('{geom_ewkt}'), {rast_srid}))
    AND rasterlayer_id = {rasterlayer_id}
    AND tilez = {zoom}
)
SELECT (vcresult).value, SUM((vcresult).count) AS count
FROM tiles_for_agg
GROUP BY (vcresult).value
"""

GLOBAL_VALUE_COUNT_SQL = """
WITH tiles_for_agg AS (
    SELECT ST_ValueCount(rast) AS vcresult
    FROM raster_rastertile
    WHERE rasterlayer_id = {rasterlayer_id}
    AND tilez = {zoom}
)
SELECT (vcresult).value, SUM((vcresult).count) AS count
FROM tiles_for_agg
GROUP BY (vcresult).value
"""

MINSIZE_SQL = """
SELECT
    ST_ScaleX(ST_Transform(rast, {srid})) AS scalex,
    ST_ScaleY(ST_Transform(rast, {srid})) AS scaley
FROM raster_rastertile
WHERE rasterlayer_id = {rasterlayer_id}
AND tilez = {zoom}
LIMIT 1
"""

MAX_ZOOM_SQL = """
SELECT MAX(tilez)
FROM raster_rastertile
WHERE rasterlayer_id={rasterlayer_id}
"""


class RasterAggregationException(Exception):
    pass


def aggregator(layer_dict, zoom=None, geom=None, formula=None, acres=True, grouping='auto'):
    """
    Compute aggregate statistics for a layers dictionary, potentially for
    an algebra expression and clipped by a geometry.

    The grouping parameter specifies how to group the pixel values for the
    aggregation count.

    Allowed are the following options:

    * 'auto' (the default). The output will be grouped by unique values if all
      input rasters are discrete, otherwise a numpy histogram is used.
    * 'discrete' groups the data will be grouped by unique values
    * 'continuous' groups the data in a numpy histogram
    * If an integer value is passed to the argument, it is interpreted as a
      legend_id. The data will be grouped using the legend expressions. For
      For instance, use grouping=23 for grouping the output with legend 23.
    """
    from .models import RasterLayer, RasterTile, Legend

    algebra_parser = RasterAlgebraParser()

    # Get layers
    layers = RasterLayer.objects.filter(id__in=layer_dict.values())

    # Compute zoom if not provided
    if zoom is None:
        zoom = min(layers.values_list('metadata__max_zoom', flat=True))

    # Compute tilerange for this area and the given zoom level
    if geom:
        # Transform geom to web mercator
        if geom.srid != WEB_MERCATOR_SRID:
            geom.transform(WEB_MERCATOR_SRID)

        # Clip against max extent for limiting nr of tiles.
        # This is important for requests on large areas for small rasters.
        max_extent = MultiPolygon([Polygon.from_bbox(lyr.extent()) for lyr in layers]).envelope
        max_extent = geom.intersection(max_extent)

        # Abort if there is no spatial overlay
        if max_extent.empty:
            return {}
{'fraction_non_alphanumeric': 0.07716097108396475, 'fraction_numerical': 0.0015463120457708365, 'mean_word_length': 4.029548989113531, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '73704', 'n_tokens_mistral': 2067, 'n_tokens_neox': 1953, 'n_words': 646}