<commit_before>package main

import (
	"encoding/binary"
	"encoding/json"
	"flag"
	"io"
	"log"
	"os"
	"strings"

	"github.com/thomersch/grandine/converter/fileformat"

	"github.com/golang/protobuf/proto"
	geom "github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/geojson"
	"github.com/twpayne/go-geom/encoding/wkb"
)

const (
	cookie  = "ABCD"
	version = 0
)

type GeoJSON struct {
	Type     string
	Features []geoJSONFeature
}

type geoJSONFeature struct {
	Type     string
	Props    map[string]interface{} `json:"properties"`
	Geometry struct {
		Type        string
		Coordinates json.RawMessage
	}
}

func (g *geoJSONFeature) WKB() []byte {
	var (
		err       error
		elem      geom.T
		unmarshal = func(c interface{}) {
			err = json.Unmarshal(g.Geometry.Coordinates, c)
		}
	)

	switch g.Geometry.Type {
	case "Point":
		var coords geom.Coord
		unmarshal(&coords)
		elem = geom.NewPoint(geom.XY).MustSetCoords(coords)
	case "LineString":
		var coords []geom.Coord
		unmarshal(&coords)
		elem = geom.NewLineString(geom.XY).MustSetCoords(coords)
	case "Polygon":
		var coords [][]geom.Coord
		unmarshal(&coords)
		elem = geom.NewPolygon(geom.XY).MustSetCoords(coords)
	case "MultiPolygon":
		var coords [][][]geom.Coord
		unmarshal(&coords)
		elem = geom.NewMultiPolygon(geom.XY).MustSetCoords(coords)
	default:
		log.Fatalf("type %v not yet implemented", g.Geometry.Type)
	}
	if err != nil {
		log.Fatal(err)
	}
	buf, err := wkb.Marshal(elem, binary.LittleEndian)
	if err != nil {
		log.Fatal(err)
	}
	return buf
}

func (g *geoJSONFeature) Properties() map[string]interface{} {
	return g.Props
}

type WKBable interface {
	WKB() []byte
	Properties() map[string]interface{}
}

type GeoJSONable interface {
	Geom() geom.T
	Properties() map[string]interface{}
}

func main() {
	source := flag.String("src", "geo.geojson", "")
	dest := flag.String("dest", "geo.unnamed", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	out, err := os.Create(*dest)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	if strings.HasSuffix(*source, ".geojson") {
		log.Println("Decoding geojson, encoding .unnamed")
		writeFileHeader(out)

		var gj GeoJSON
		if err := json.NewDecoder(f).Decode(&gj); err != nil {
			log.Fatal(err)
		}

		// for now write a block for every feature
		for _, feat := range gj.Features {
			writeBlock(out, []WKBable{&feat})
		}
	} else {
		log.Println("Decoding .unnamed, writing geojson")
		readFileHeader(f)
		var gjfc geojson.FeatureCollection
		for _, ft := range readBlocks(f) {
			gjfc.Features = append(gjfc.Features, &geojson.Feature{
				Geometry:   ft.Geom(),
				Properties: ft.Properties(),
			})
		}
		buf, err := gjfc.MarshalJSON()
		if err != nil {
			log.Fatal(err)
		}
		out.Write(buf)
	}
}

func writeFileHeader(w io.Writer) {
	// Cookie
	w.Write([]byte(cookie))

	// Version
	binary.Write(w, binary.LittleEndian, uint32(version))
}

func writeBlock(w io.Writer, fs []WKBable) {
	blockBody := &fileformat.Body{}
	for _, f := range fs {
		var (
			tags []*fileformat.Tag
		)
		for k, v := range f.Properties() {<commit_msg>blocks with 100 elements each<commit_after>package main

import (
	"encoding/binary"
	"encoding/json"
	"flag"
	"io"
	"log"
	"os"
	"strings"

	"github.com/thomersch/grandine/converter/fileformat"

	"github.com/golang/protobuf/proto"
	geom "github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/geojson"
	"github.com/twpayne/go-geom/encoding/wkb"
)

const (
	cookie  = "ABCD"
	version = 0
)

type GeoJSON struct {
	Type     string
	Features []geoJSONFeature
}

type geoJSONFeature struct {
	Type     string
	Props    map[string]interface{} `json:"properties"`
	Geometry struct {
		Type        string
		Coordinates json.RawMessage
	}
}

func (g *geoJSONFeature) WKB() []byte {
	var (
		err       error
		elem      geom.T
		unmarshal = func(c interface{}) {
			err = json.Unmarshal(g.Geometry.Coordinates, c)
		}
	)

	switch g.Geometry.Type {
	case "Point":
		var coords geom.Coord
		unmarshal(&coords)
		elem = geom.NewPoint(geom.XY).MustSetCoords(coords)
	case "LineString":
		var coords []geom.Coord
		unmarshal(&coords)
		elem = geom.NewLineString(geom.XY).MustSetCoords(coords)
	case "Polygon":
		var coords [][]geom.Coord
		unmarshal(&coords)
		elem = geom.NewPolygon(geom.XY).MustSetCoords(coords)
	case "MultiPolygon":
		var coords [][][]geom.Coord
		unmarshal(&coords)
		elem = geom.NewMultiPolygon(geom.XY).MustSetCoords(coords)
	default:
		log.Fatalf("type %v not yet implemented", g.Geometry.Type)
	}
	if err != nil {
		log.Fatal(err)
	}
	buf, err := wkb.Marshal(elem, binary.LittleEndian)
	if err != nil {
		log.Fatal(err)
	}
	return buf
}

func (g *geoJSONFeature) Properties() map[string]interface{} {
	return g.Props
}

type WKBable interface {
	WKB() []byte
	Properties() map[string]interface{}
}

type GeoJSONable interface {
	Geom() geom.T
	Properties() map[string]interface{}
}

func main() {
	source := flag.String("src", "geo.geojson", "")
	dest := flag.String("dest", "geo.unnamed", "")
	flag.Parse()

	f, err := os.Open(*source)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	out, err := os.Create(*dest)
	if err != nil {
		log.Fatal(err)
	}
	defer out.Close()

	if strings.HasSuffix(*source, ".geojson") {
		log.Println("Decoding geojson, encoding .unnamed")
		writeFileHeader(out)

		var gj GeoJSON
		if err := json.NewDecoder(f).Decode(&gj); err != nil {
			log.Fatal(err)
		}

		// for now write a block for every feature
		for _, featBlock := range geomBlocks(100, gj.Features) {
			// TODO: oh god, why.
			var fs []WKBable
			for _, feat := range featBlock {
				ft := feat
				fs = append(fs, &ft)
			}
			writeBlock(out, fs)
		}
	} else {
		log.Println("Decoding .unnamed, writing geojson")
		readFileHeader(f)
		var gjfc geojson.FeatureCollection
		for _, ft := range readBlocks(f) {
			gjfc.Features = append(gjfc.Features, &geojson.Feature{
				Geometry:   ft.Geom(),
				Properties: ft.Properties(),
			})
		}
		buf, err := gjfc.MarshalJSON()
		if err != nil {
			log.Fatal(err)
		}
		out.Write(buf)
	}
}

func writeFileHeader(w io.Writer) {
	// Cookie
	w.Write([]byte(cookie))

	// Version
	binary.Write(w, binary.LittleEndian, uint32(version))
}

// geomBlocks slices a slice of geometries into slices with a max size
func geomBlocks(size int, src []geoJSONFeature) [][]geoJSONFeature {
	if len(src) <= size {
		return [][]geoJSONFeature{src}
	}

	var (
		i   int
		res [][]geoJSONFeature
		end int
	)
	for end < len(src) {
		end = (i + 1) * size
		if end > len(src) {
			end = len(src)
		}
		res = append(res, src[i*size:end])
		i++
	}
	return res
}

func writeBlock(w io.Writer, fs []WKBable) {
	blockBody := &fileformat.Body{}
	for _, f := range fs {
		var (
			tags []*fileformat.Tag
		)
		for k, v := range f.Properties() {
{'fraction_non_alphanumeric': 0.15176867752825482, 'fraction_numerical': 0.0019081168354616175, 'mean_word_length': 3.5885521885521885, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4402474', 'n_tokens_mistral': 2831, 'n_tokens_neox': 2565, 'n_words': 565}