<commit_before>package com.opendatasoft.elasticsearch.search.aggregations.bucket.geohashclustering;

import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.common.util.LongObjectPagedHashMap;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.search.aggregations.*;
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
import org.elasticsearch.search.aggregations.support.AggregationContext;
import org.elasticsearch.search.aggregations.support.ValuesSource;

import java.io.IOException;
import java.util.*;

public class GeoHashClusteringAggregator extends BucketsAggregator {

    private static final int INITIAL_CAPACITY = 50; // TODO sizing

    private final ValuesSource.GeoPoint valuesSource;
    private LongHash bucketOrds;
    private LongObjectPagedHashMap<ClusterCollector> clusterCollectors;
    private GeoPointValues geoValues;
    private int zoom;
    private int distance;

    public static class ClusterCollector {
        private List<GeoPoint> geoPoints;
        private double latMin;
        private double latMax;
        private double lonMin;
        private double lonMax;


        public ClusterCollector(GeoPoint geoPoint) {
            geoPoints = new ArrayList<GeoPoint>();
            geoPoints.add(geoPoint);
            latMin = latMax = geoPoint.getLat();
            lonMin = lonMax = geoPoint.getLon();
        }

        public void addPoint(GeoPoint point) {
            geoPoints.add(point);
            latMin = Math.min(point.getLat(), latMin);
            latMax = Math.max(point.getLat(), latMax);
            lonMin = Math.min(point.getLon(), lonMin);
            lonMax = Math.max(point.getLon(), lonMax);
        }

        public double getLatMin() {
            return latMin;
        }

        public double getLatMax() {
            return latMax;
        }

        public double getLonMin() {
            return lonMin;
        }

        public double getLonMax() {
            return lonMax;
        }
    }

    public GeoHashClusteringAggregator(String name, AggregatorFactories factories, ValuesSource.GeoPoint valuesSource,
                                       AggregationContext aggregationContext, Aggregator parent,
                                       int zoom, int distance) {
        super(name, BucketAggregationMode.PER_BUCKET, factories, INITIAL_CAPACITY, aggregationContext, parent);
        this.valuesSource = valuesSource;
        bucketOrds = new LongHash(INITIAL_CAPACITY, aggregationContext.bigArrays());
        clusterCollectors = new LongObjectPagedHashMap<ClusterCollector>(INITIAL_CAPACITY, aggregationContext.bigArrays());
        this.zoom = zoom;
        this.distance = distance;
    }

    @Override
    public boolean shouldCollect() {
        return true;
    }

    @Override
    public void setNextReader(AtomicReaderContext reader) {
        geoValues = valuesSource.geoPointValues();
    }

    @Override
    public void collect(int doc, long owningBucketOrdinal) throws IOException {
        assert owningBucketOrdinal == 0;

        final int valuesCount2 = geoValues.setDocument(doc);


        for (int i = 0; i<valuesCount2; i++) {

            final GeoPoint geoPoint = geoValues.nextValue();
            double meterByPixel = GeoClusterUtils.getMeterByPixel(zoom, geoPoint.getLat());
            int pointPrecision = GeoUtils.geoHashLevelsForPrecision(distance * meterByPixel);
            pointPrecision = pointPrecision > 1 ? pointPrecision - 1: pointPrecision ;
            final long clusterHashLong = GeoHashUtils.encodeAsLong(geoPoint.getLat(), geoPoint.getLon(), pointPrecision);
            ClusterCollector collec;
            long bucketOrdinal2 = bucketOrds.add(clusterHashLong);
            if (bucketOrdinal2 < 0) { // already seen
                bucketOrdinal2 = - 1 - bucketOrdinal2;
                collec = clusterCollectors.get(clusterHashLong);
                collec.addPoint(geoPoint);
                collectExistingBucket(doc, bucketOrdinal2);
            } else {
                collec = new ClusterCollector(geoPoint);
                collectBucket(doc, bucketOrdinal2);
            }

            clusterCollectors.put(clusterHashLong, collec);
        }
    }

    // private impl that stores a bucket ord. This allows for computing the aggregations lazily.
    static class OrdinalBucket extends InternalGeoHashClustering.Bucket {

        public OrdinalBucket() {
            super(0, null, 0, null);<commit_msg>Cleaning unused variable<commit_after>package com.opendatasoft.elasticsearch.search.aggregations.bucket.geohashclustering;

import org.apache.lucene.index.AtomicReaderContext;
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.lease.Releasables;
import org.elasticsearch.common.util.LongHash;
import org.elasticsearch.common.util.LongObjectPagedHashMap;
import org.elasticsearch.index.fielddata.GeoPointValues;
import org.elasticsearch.search.aggregations.*;
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
import org.elasticsearch.search.aggregations.support.AggregationContext;
import org.elasticsearch.search.aggregations.support.ValuesSource;

import java.io.IOException;
import java.util.*;

public class GeoHashClusteringAggregator extends BucketsAggregator {

    private static final int INITIAL_CAPACITY = 50; // TODO sizing

    private final ValuesSource.GeoPoint valuesSource;
    private LongHash bucketOrds;
    private LongObjectPagedHashMap<ClusterCollector> clusterCollectors;
    private GeoPointValues geoValues;
    private int zoom;
    private int distance;

    public static class ClusterCollector {
//        private List<GeoPoint> geoPoints;
        private double latMin;
        private double latMax;
        private double lonMin;
        private double lonMax;


        public ClusterCollector(GeoPoint geoPoint) {
//            geoPoints = new ArrayList<GeoPoint>();
//            geoPoints.add(geoPoint);
            latMin = latMax = geoPoint.getLat();
            lonMin = lonMax = geoPoint.getLon();
        }

        public void addPoint(GeoPoint point) {
//            geoPoints.add(point);
            latMin = Math.min(point.getLat(), latMin);
            latMax = Math.max(point.getLat(), latMax);
            lonMin = Math.min(point.getLon(), lonMin);
            lonMax = Math.max(point.getLon(), lonMax);
        }

        public double getLatMin() {
            return latMin;
        }

        public double getLatMax() {
            return latMax;
        }

        public double getLonMin() {
            return lonMin;
        }

        public double getLonMax() {
            return lonMax;
        }
    }

    public GeoHashClusteringAggregator(String name, AggregatorFactories factories, ValuesSource.GeoPoint valuesSource,
                                       AggregationContext aggregationContext, Aggregator parent,
                                       int zoom, int distance) {
        super(name, BucketAggregationMode.PER_BUCKET, factories, INITIAL_CAPACITY, aggregationContext, parent);
        this.valuesSource = valuesSource;
        bucketOrds = new LongHash(INITIAL_CAPACITY, aggregationContext.bigArrays());
        clusterCollectors = new LongObjectPagedHashMap<ClusterCollector>(INITIAL_CAPACITY, aggregationContext.bigArrays());
        this.zoom = zoom;
        this.distance = distance;
    }

    @Override
    public boolean shouldCollect() {
        return true;
    }

    @Override
    public void setNextReader(AtomicReaderContext reader) {
        geoValues = valuesSource.geoPointValues();
    }

    @Override
    public void collect(int doc, long owningBucketOrdinal) throws IOException {
        assert owningBucketOrdinal == 0;

        final int valuesCount = geoValues.setDocument(doc);


        for (int i = 0; i<valuesCount; i++) {

            final GeoPoint geoPoint = geoValues.nextValue();
            double meterByPixel = GeoClusterUtils.getMeterByPixel(zoom, geoPoint.getLat());
            int pointPrecision = GeoUtils.geoHashLevelsForPrecision(distance * meterByPixel);
            pointPrecision = pointPrecision > 1 ? pointPrecision - 1: pointPrecision ;
            final long clusterHashLong = GeoHashUtils.encodeAsLong(geoPoint.getLat(), geoPoint.getLon(), pointPrecision);
            ClusterCollector collec;
            long bucketOrdinal2 = bucketOrds.add(clusterHashLong);
            if (bucketOrdinal2 < 0) { // already seen
                bucketOrdinal2 = - 1 - bucketOrdinal2;
                collec = clusterCollectors.get(clusterHashLong);
                collec.addPoint(geoPoint);
                collectExistingBucket(doc, bucketOrdinal2);
            } else {
                collec = new ClusterCollector(geoPoint);
                collectBucket(doc, bucketOrdinal2);
            }

            clusterCollectors.put(clusterHashLong, collec);
        }
    }

    // private impl that stores a bucket ord. This allows for computing the aggregations lazily.
    static class OrdinalBucket extends InternalGeoHashClustering.Bucket {

        public OrdinalBucket() {
            super(0, null, 0, null);
{'fraction_non_alphanumeric': 0.08052846807172066, 'fraction_numerical': 0.0035650623885918, 'mean_word_length': 2.7185185185185183, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5137454', 'n_tokens_mistral': 2646, 'n_tokens_neox': 2507, 'n_words': 553}