<commit_before>package limit

import (
	"errors"
	"imposm3/geom/geojson"
	"imposm3/geom/geos"
	"imposm3/logging"
	"math"
	"os"
	"strings"
	"sync"
)

var log = logging.NewLogger("limiter")

// Tile bbox into multiple sub-boxes, each of `width` size.

// >>> list(tile_bbox((-1, 1, 0.49, 1.51), 0.5)) #doctest: +NORMALIZE_WHITESPACE
// [(-1.0, 1.0, -0.5, 1.5),
//  (-1.0, 1.5, -0.5, 2.0),
//  (-0.5, 1.0, 0.0, 1.5),
//  (-0.5, 1.5, 0.0, 2.0),
//  (0.0, 1.0, 0.5, 1.5),
//  (0.0, 1.5, 0.5, 2.0)]
func tileBounds(bounds geos.Bounds, width float64) []geos.Bounds {
	var results []geos.Bounds
	minX := math.Floor(bounds.MinX/width) * width
	minY := math.Floor(bounds.MinY/width) * width
	maxX := math.Ceil(bounds.MaxX/width) * width
	maxY := math.Ceil(bounds.MaxY/width) * width

	xSteps := math.Ceil((maxX - minX) / width)
	ySteps := math.Ceil((maxY - minY) / width)

	for x := 0; x < int(xSteps); x++ {
		for y := 0; y < int(ySteps); y++ {
			results = append(results, geos.Bounds{
				minX + float64(x)*width,
				minY + float64(y)*width,
				minX + float64(x+1)*width,
				minY + float64(y+1)*width,
			})
		}
	}
	return results
}

func splitParams(bounds geos.Bounds, maxGrids int, minGridWidth float64) (float64, float64) {
	width := bounds.MaxX - bounds.MinX
	height := bounds.MaxY - bounds.MinY

	gridWidthX := minGridWidth
	gridWidthY := minGridWidth
	if width/gridWidthX > float64(maxGrids) {
		gridWidthX = width / float64(maxGrids)
	}
	if height/gridWidthY > float64(maxGrids) {
		gridWidthY = height / float64(maxGrids)
	}

	gridWidth := math.Max(gridWidthX, gridWidthY)
	currentWidth := gridWidth

	for currentWidth <= width/2 {
		currentWidth *= 2
	}
	return gridWidth, currentWidth
}

func SplitPolygonAtAutoGrid(g *geos.Geos, geom *geos.Geom) ([]*geos.Geom, error) {
	geomBounds := geom.Bounds()
	if geomBounds == geos.NilBounds {
		return nil, errors.New("couldn't create bounds for geom")
	}
	gridWidth, currentGridWidth := splitParams(geomBounds, 64, 20000.0)
	return SplitPolygonAtGrid(g, geom, gridWidth, currentGridWidth)
}

func SplitPolygonAtGrid(g *geos.Geos, geom *geos.Geom, gridWidth, currentGridWidth float64) ([]*geos.Geom, error) {
	var result []*geos.Geom
	geomBounds := geom.Bounds()
	if geomBounds == geos.NilBounds {
		return nil, errors.New("couldn't create bounds for geom")
	}
	for _, bounds := range tileBounds(geom.Bounds(), currentGridWidth) {
		clipGeom := g.BoundsPolygon(bounds)
		if clipGeom == nil {
			return nil, errors.New("couldn't create bounds polygon")
		}
		part := g.Intersection(geom, clipGeom)
		if part == nil {
			return nil, errors.New("couldn't create intersection")
		}
		if !g.IsEmpty(part) && strings.HasSuffix(g.Type(part), "Polygon") {
			if gridWidth >= currentGridWidth {
				result = append(result, part)
			} else {
				moreParts, err := SplitPolygonAtGrid(g, part, gridWidth, currentGridWidth/2.0)
				if err != nil {
					return nil, err
				}
				result = append(result, moreParts...)
			}
		}
	}
	return result, nil
}

type Limiter struct {
	// for quick intersections of small geometries
	index *geos.Index
	// for direct intersections of large geometries
	geom *geos.Geom
	// for quick contains checks
	geomPrep   *geos.PreparedGeom
	geomPrepMu *sync.Mutex

	// bufferedXxx for diff elements
	// (we keep more data at the boundary to be able to build
	// geometries that intersects the limitto geometry)

	// for quick coarse contains checks
	bufferedBbox geos.Bounds
	// for quick contains checks
	bufferedPrep   *geos.PreparedGeom
	bufferedPrepMu *sync.Mutex
}

func NewFromGeoJson(source string) (*Limiter, error) {
	return NewFromGeoJsonWithBuffered(source, 0.0)
}

func NewFromGeoJsonWithBuffered(source string, buffer float64) (*Limiter, error) {

	f, err := os.Open(source)
	if err != nil {
		return nil, err
	}

	gj, err := geojson.NewGeoJson(f)
	if err != nil {
		return nil, err
	}
	geoms, err := gj.Geoms()
	if err != nil {
		return nil, err
	}

	g := geos.NewGeos()
	defer g.Finish()<commit_msg>close geojson file handle<commit_after>package limit

import (
	"errors"
	"imposm3/geom/geojson"
	"imposm3/geom/geos"
	"imposm3/logging"
	"math"
	"os"
	"strings"
	"sync"
)

var log = logging.NewLogger("limiter")

// Tile bbox into multiple sub-boxes, each of `width` size.

// >>> list(tile_bbox((-1, 1, 0.49, 1.51), 0.5)) #doctest: +NORMALIZE_WHITESPACE
// [(-1.0, 1.0, -0.5, 1.5),
//  (-1.0, 1.5, -0.5, 2.0),
//  (-0.5, 1.0, 0.0, 1.5),
//  (-0.5, 1.5, 0.0, 2.0),
//  (0.0, 1.0, 0.5, 1.5),
//  (0.0, 1.5, 0.5, 2.0)]
func tileBounds(bounds geos.Bounds, width float64) []geos.Bounds {
	var results []geos.Bounds
	minX := math.Floor(bounds.MinX/width) * width
	minY := math.Floor(bounds.MinY/width) * width
	maxX := math.Ceil(bounds.MaxX/width) * width
	maxY := math.Ceil(bounds.MaxY/width) * width

	xSteps := math.Ceil((maxX - minX) / width)
	ySteps := math.Ceil((maxY - minY) / width)

	for x := 0; x < int(xSteps); x++ {
		for y := 0; y < int(ySteps); y++ {
			results = append(results, geos.Bounds{
				minX + float64(x)*width,
				minY + float64(y)*width,
				minX + float64(x+1)*width,
				minY + float64(y+1)*width,
			})
		}
	}
	return results
}

func splitParams(bounds geos.Bounds, maxGrids int, minGridWidth float64) (float64, float64) {
	width := bounds.MaxX - bounds.MinX
	height := bounds.MaxY - bounds.MinY

	gridWidthX := minGridWidth
	gridWidthY := minGridWidth
	if width/gridWidthX > float64(maxGrids) {
		gridWidthX = width / float64(maxGrids)
	}
	if height/gridWidthY > float64(maxGrids) {
		gridWidthY = height / float64(maxGrids)
	}

	gridWidth := math.Max(gridWidthX, gridWidthY)
	currentWidth := gridWidth

	for currentWidth <= width/2 {
		currentWidth *= 2
	}
	return gridWidth, currentWidth
}

func SplitPolygonAtAutoGrid(g *geos.Geos, geom *geos.Geom) ([]*geos.Geom, error) {
	geomBounds := geom.Bounds()
	if geomBounds == geos.NilBounds {
		return nil, errors.New("couldn't create bounds for geom")
	}
	gridWidth, currentGridWidth := splitParams(geomBounds, 64, 20000.0)
	return SplitPolygonAtGrid(g, geom, gridWidth, currentGridWidth)
}

func SplitPolygonAtGrid(g *geos.Geos, geom *geos.Geom, gridWidth, currentGridWidth float64) ([]*geos.Geom, error) {
	var result []*geos.Geom
	geomBounds := geom.Bounds()
	if geomBounds == geos.NilBounds {
		return nil, errors.New("couldn't create bounds for geom")
	}
	for _, bounds := range tileBounds(geom.Bounds(), currentGridWidth) {
		clipGeom := g.BoundsPolygon(bounds)
		if clipGeom == nil {
			return nil, errors.New("couldn't create bounds polygon")
		}
		part := g.Intersection(geom, clipGeom)
		if part == nil {
			return nil, errors.New("couldn't create intersection")
		}
		if !g.IsEmpty(part) && strings.HasSuffix(g.Type(part), "Polygon") {
			if gridWidth >= currentGridWidth {
				result = append(result, part)
			} else {
				moreParts, err := SplitPolygonAtGrid(g, part, gridWidth, currentGridWidth/2.0)
				if err != nil {
					return nil, err
				}
				result = append(result, moreParts...)
			}
		}
	}
	return result, nil
}

type Limiter struct {
	// for quick intersections of small geometries
	index *geos.Index
	// for direct intersections of large geometries
	geom *geos.Geom
	// for quick contains checks
	geomPrep   *geos.PreparedGeom
	geomPrepMu *sync.Mutex

	// bufferedXxx for diff elements
	// (we keep more data at the boundary to be able to build
	// geometries that intersects the limitto geometry)

	// for quick coarse contains checks
	bufferedBbox geos.Bounds
	// for quick contains checks
	bufferedPrep   *geos.PreparedGeom
	bufferedPrepMu *sync.Mutex
}

func NewFromGeoJson(source string) (*Limiter, error) {
	return NewFromGeoJsonWithBuffered(source, 0.0)
}

func NewFromGeoJsonWithBuffered(source string, buffer float64) (*Limiter, error) {

	f, err := os.Open(source)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	gj, err := geojson.NewGeoJson(f)
	if err != nil {
		return nil, err
	}
	geoms, err := gj.Geoms()
	if err != nil {
		return nil, err
	}

	g := geos.NewGeos()
	defer g.Finish()
{'fraction_non_alphanumeric': 0.14715090795241076, 'fraction_numerical': 0.026800250469630558, 'mean_word_length': 3.686619718309859, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7107062', 'n_tokens_mistral': 3379, 'n_tokens_neox': 3051, 'n_words': 748}