<commit_before>// Package geoindex provides in memory geoindex implementation. It works by splitting the earth surface
// into grid with fixed size cells and storing data in each cell. The data can be points, count of points,
// and expiring points/counts. Has Range and K-Nearest queries.
package geoindex

import (
	"math"
	"sort"
)

// A geoindex that stores points.
type PointsIndex struct {
	index           *geoIndex
	currentPosition map[string]Point
}

// NewPointsIndex creates new PointsIndex that maintains the points in each cell.
func NewPointsIndex(resolution Meters) *PointsIndex {
	newSet := func() interface{} {
		return newSet()
	}

	return &PointsIndex{newGeoIndex(resolution, newSet), make(map[string]Point)}
}

// NewExpiringPointsIndex creates new PointIndex that expires the points in each cell after expiration minutes.
func NewExpiringPointsIndex(resolution Meters, expiration Minutes) *PointsIndex {
	currentPosition := make(map[string]Point)

	newExpiringSet := func() interface{} {
		set := newExpiringSet(expiration)

		set.OnExpire(func(id string, value interface{}) {
			point := value.(Point)
			delete(currentPosition, point.Id())
		})

		return set
	}

	return &PointsIndex{newGeoIndex(resolution, newExpiringSet), currentPosition}
}

// Get gets a point from the index given an id.
func (points *PointsIndex) Get(id string) Point {
	if point, ok := points.currentPosition[id]; ok {
		// first it gets the set of the currentPosition and then gets the point from the set
		// this is done so it triggers expiration on expiringSet, and returns nil if a point has expired<commit_msg>Add a NewExpiringPointsIndexWithCallback to be notified when a point is deleted from the Set<commit_after>// Package geoindex provides in memory geoindex implementation. It works by splitting the earth surface
// into grid with fixed size cells and storing data in each cell. The data can be points, count of points,
// and expiring points/counts. Has Range and K-Nearest queries.
package geoindex

import (
	"math"
	"sort"
)

// A geoindex that stores points.
type PointsIndex struct {
	index           *geoIndex
	currentPosition map[string]Point
}

// NewPointsIndex creates new PointsIndex that maintains the points in each cell.
func NewPointsIndex(resolution Meters) *PointsIndex {
	newSet := func() interface{} {
		return newSet()
	}

	return &PointsIndex{newGeoIndex(resolution, newSet), make(map[string]Point)}
}

// NewExpiringPointsIndex creates new PointIndex that expires the points in each cell after expiration minutes.
// the callback will be called when the point gets deleted
func NewExpiringPointsIndexWithCallback(resolution Meters, expiration Minutes, callback func(point Point)) *PointsIndex {
	currentPosition := make(map[string]Point)

	newExpiringSet := func() interface{} {
		set := newExpiringSet(expiration)

		set.OnExpire(func(id string, value interface{}) {
			point := value.(Point)
			delete(currentPosition, point.Id())
			callback(point)
		})

		return set
	}

	return &PointsIndex{newGeoIndex(resolution, newExpiringSet), currentPosition}
}

// NewExpiringPointsIndex creates new PointIndex that expires the points in each cell after expiration minutes.
func NewExpiringPointsIndex(resolution Meters, expiration Minutes) *PointsIndex {
	return NewExpiringPointsIndexWithCallback(resolution, expiration, func(point Point) {})
}

// Get gets a point from the index given an id.
func (points *PointsIndex) Get(id string) Point {
	if point, ok := points.currentPosition[id]; ok {
		// first it gets the set of the currentPosition and then gets the point from the set
		// this is done so it triggers expiration on expiringSet, and returns nil if a point has expired
{'fraction_non_alphanumeric': 0.08034510649770828, 'fraction_numerical': 0.0, 'mean_word_length': 4.690184049079755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '179406', 'n_tokens_mistral': 1065, 'n_tokens_neox': 989, 'n_words': 409}