<commit_before>package catTracks

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"sync"
	"time"

	"compress/gzip"
	"log"
	"os"
	"path/filepath"

	"github.com/boltdb/bolt"
	"github.com/davecgh/go-spew/spew"
	"github.com/kpawlik/geojson"
	"github.com/rotblauer/tileTester2/note"
	"github.com/rotblauer/trackpoints/trackPoint"
	gm "googlemaps.github.io/maps"
)

var punktlichTileDBPathRelHome = filepath.Join("punktlich.rotblauer.com", "tester.db")

type LastKnown map[string]trackPoint.TrackPoint
type Metadata struct {
	KeyN               int
	KeyNUpdated        time.Time
	LastUpdatedAt      time.Time
	LastUpdatedBy      string
	LastUpdatedPointsN int
	TileDBLastUpdated  time.Time
}

type QueryFilterPlaces struct {
	Uuids []string `schema:"uuids"`
	Names []string `schema:"names"`
	// PushTokens []string `schema:"pushTokens"` // TODO

	// start,end x arrive,depart,report
	StartArrivalT   time.Time `schema:"startArrivalT"`
	EndArrivalT     time.Time `schema:"endArrivalT"`
	StartDepartureT time.Time `schema:"startDepartureT"`
	EndDepartureT   time.Time `schema:"endDepartureT"`
	StartReportedT  time.Time `schema:"startReportedT"`
	EndReportedT    time.Time `schema:"endReportedT"`

	ReverseChrono bool `schema:"rc"` // when true, oldest first; default to newest first

	// paginatables
	StartIndex int `schema:"startI"` // 0 indexed;
	EndIndex   int `schema:"endI"`   // diff end-start = per/pagination lim

	// for geo rect bounding, maybe
	LatMin *float64 `schema:"latmin"`
	LatMax *float64 `schema:"latmax"`
	LngMin *float64 `schema:"lngmin"`
	LngMax *float64 `schema:"lngmax"`

	IncludeStats bool `schema:"stats"`

	GoogleNearby       bool `schema:"googleNearby"`
	GoogleNearbyPhotos bool `schema:"googleNearbyPhotos"`
}

// var DefaultQFP = QueryFilterPlaces{
// 	Names:    []string{},
// 	EndIndex: 30, // given zero values, reverse and StartIndex=0, this returns 30 most recent places
// 	LatMin:   math.MaxFloat64,
// 	LatMax:   math.MaxFloat64,
// 	LngMin:   math.MaxFloat64,
// 	LatMax:   math.MaxFloat64,
// }

// // ByTime implements Sort interface for NoteVisit
// type ByTime []note.NoteVisit

// func (bt ByTime) Len() int {
// 	return len(bt)
// }

// func (bt ByTime) Swap(i, j int) {
// 	bt[i], bt[j] = bt[j], bt[i]
// }

// // Less compares ARRIVALTIME. This might need to be expanded or differentiated.
// func (bt ByTime) Less(i, j int) bool {
// 	return bt[i].ArrivalTime.Before(bt[j].ArrivalTime)
// }

type VisitsResponse struct {
	Visits    []*note.NoteVisit `json:"visits"`
	Stats     bolt.BucketStats  `json"bucketStats"`
	StatsTook time.Duration     `json:"statsTook"` // how long took to get bucket stats (for 10mm++ points, long time)
	Scanned   int               `json:"scanned"`   // num visits checked before mtaching filters
	Matches   int               `json:"matches"`   // num visits matching before paging/index filters
}

// btw places are actually visits. fucked that one up.
func getPlaces(qf QueryFilterPlaces) (out []byte, err error) {
	// TODO
	// - wire to router with query params
	// - filter during key iter
	// - sortable interface places
	// - places to json, new type in Note

	log.Println("handling visits q:", spew.Sdump(qf))

	var res = VisitsResponse{}
	var visits = []*note.NoteVisit{}
	var scannedN, matchingN int // nice convenience returnables for query stats, eg. matched 4/14 visits, querier can know this

	// FIXME: only Update r b/c case we want to go grab the google nearby for a visit. not necessary otherwise
	err = GetDB("master").Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(placesKey))

		if qf.IncludeStats {
			t1 := time.Now()
			res.Stats = b.Stats()
			res.StatsTook = time.Since(t1)
		}

		if b == nil {
			return fmt.Errorf("no places bolt bucket exists")
		}<commit_msg>omit empty json fields<commit_after>package catTracks

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"sync"
	"time"

	"compress/gzip"
	"log"
	"os"
	"path/filepath"

	"github.com/boltdb/bolt"
	"github.com/davecgh/go-spew/spew"
	"github.com/kpawlik/geojson"
	"github.com/rotblauer/tileTester2/note"
	"github.com/rotblauer/trackpoints/trackPoint"
	gm "googlemaps.github.io/maps"
)

var punktlichTileDBPathRelHome = filepath.Join("punktlich.rotblauer.com", "tester.db")

type LastKnown map[string]trackPoint.TrackPoint
type Metadata struct {
	KeyN               int
	KeyNUpdated        time.Time
	LastUpdatedAt      time.Time
	LastUpdatedBy      string
	LastUpdatedPointsN int
	TileDBLastUpdated  time.Time
}

type QueryFilterPlaces struct {
	Uuids []string `schema:"uuids"`
	Names []string `schema:"names"`
	// PushTokens []string `schema:"pushTokens"` // TODO

	// start,end x arrive,depart,report
	StartArrivalT   time.Time `schema:"startArrivalT"`
	EndArrivalT     time.Time `schema:"endArrivalT"`
	StartDepartureT time.Time `schema:"startDepartureT"`
	EndDepartureT   time.Time `schema:"endDepartureT"`
	StartReportedT  time.Time `schema:"startReportedT"`
	EndReportedT    time.Time `schema:"endReportedT"`

	ReverseChrono bool `schema:"rc"` // when true, oldest first; default to newest first

	// paginatables
	StartIndex int `schema:"startI"` // 0 indexed;
	EndIndex   int `schema:"endI"`   // diff end-start = per/pagination lim

	// for geo rect bounding, maybe
	LatMin *float64 `schema:"latmin"`
	LatMax *float64 `schema:"latmax"`
	LngMin *float64 `schema:"lngmin"`
	LngMax *float64 `schema:"lngmax"`

	IncludeStats bool `schema:"stats,omitempty"`

	GoogleNearby       bool `schema:"googleNearby"`
	GoogleNearbyPhotos bool `schema:"googleNearbyPhotos"`
}

// var DefaultQFP = QueryFilterPlaces{
// 	Names:    []string{},
// 	EndIndex: 30, // given zero values, reverse and StartIndex=0, this returns 30 most recent places
// 	LatMin:   math.MaxFloat64,
// 	LatMax:   math.MaxFloat64,
// 	LngMin:   math.MaxFloat64,
// 	LatMax:   math.MaxFloat64,
// }

// // ByTime implements Sort interface for NoteVisit
// type ByTime []note.NoteVisit

// func (bt ByTime) Len() int {
// 	return len(bt)
// }

// func (bt ByTime) Swap(i, j int) {
// 	bt[i], bt[j] = bt[j], bt[i]
// }

// // Less compares ARRIVALTIME. This might need to be expanded or differentiated.
// func (bt ByTime) Less(i, j int) bool {
// 	return bt[i].ArrivalTime.Before(bt[j].ArrivalTime)
// }

type VisitsResponse struct {
	Visits    []*note.NoteVisit `json:"visits"`
	Stats     bolt.BucketStats  `json"bucketStats,omitempty"`
	StatsTook time.Duration     `json:"statsTook,omitempty"` // how long took to get bucket stats (for 10mm++ points, long time)
	Scanned   int               `json:"scanned"`             // num visits checked before mtaching filters
	Matches   int               `json:"matches"`             // num visits matching before paging/index filters
}

// btw places are actually visits. fucked that one up.
func getPlaces(qf QueryFilterPlaces) (out []byte, err error) {
	// TODO
	// - wire to router with query params
	// - filter during key iter
	// - sortable interface places
	// - places to json, new type in Note

	log.Println("handling visits q:", spew.Sdump(qf))

	var res = VisitsResponse{}
	var visits = []*note.NoteVisit{}
	var scannedN, matchingN int // nice convenience returnables for query stats, eg. matched 4/14 visits, querier can know this

	// FIXME: only Update r b/c case we want to go grab the google nearby for a visit. not necessary otherwise
	err = GetDB("master").Update(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(placesKey))

		if qf.IncludeStats {
			t1 := time.Now()
			res.Stats = b.Stats()
			res.StatsTook = time.Since(t1)
		}

		if b == nil {
			return fmt.Errorf("no places bolt bucket exists")
		}
{'fraction_non_alphanumeric': 0.1300823637076742, 'fraction_numerical': 0.007844162635638646, 'mean_word_length': 3.5863309352517985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1026251', 'n_tokens_mistral': 2759, 'n_tokens_neox': 2618, 'n_words': 705}