<commit_before>package main

import (
    "flag"
    "os"
    "bufio"
    "fmt"
    "path/filepath"
    "strings"
    "strconv"
    "GoMySQL"
    "runtime"
    "math"
    "sort"
    "log"
    "xiwi"
    "github.com/ungerik/go-cairo"
    "time"
)

var GRAPH_PADDING = 100 // what to pad graph by on each side
var TILE_PIXEL_LEN = 256

var flagDB         = flag.String("db", "", "MySQL database to connect to")
var flagGrayScale  = flag.Bool("gs", false, "Make grayscale tiles")
var flagHeatMap    = flag.Bool("hm", false, "Make heatmap tiles")
var flagDoSingle   = flag.Bool("single-tile", false, "Only generate a large single tile, no labels or world index information")

var flagSkipTiles  = flag.Bool("skip-tiles", false, "Do not generate tiles (still generates index information)")
var flagSkipLabels = flag.Bool("skip-labels", false, "Do not generate labels (still generates index information)")

var flagMaxCores = flag.Int("cores",-1,"Max number of system cores to use, default is all of them")

func main() {
    // parse command line options
    flag.Parse()

    if flag.NArg() != 1 {
        log.Fatal("need to specify output prefix")
    }
    outPrefix := flag.Arg(0)

    // connect to the db
    db := xiwi.ConnectToDB(*flagDB)
    if db == nil {
        return
    }
    defer db.Close()

    // read in the graph
    graph := ReadGraph(db)

    // build the quad tree
    graph.BuildQuadTree()
    
    runtime.GC()

    if *flagDoSingle {
        DrawTile(graph, graph.BoundsX, graph.BoundsY, 1, 1, 18000, 18000, outPrefix)
    } else {
        // Create index file
        indexFile := outPrefix + "/world_index.json"

        if err := os.MkdirAll(filepath.Dir(indexFile),0755); err != nil {
            fmt.Println(err)
            return
        }
        fo, _ := os.Create(indexFile)
        defer fo.Close()
        w := bufio.NewWriter(fo)

        num_papers := len(graph.papers)

        fmt.Fprintf(w,"world_index({\"latestid\":%d,\"numpapers\":%d,\"xmin\":%d,\"ymin\":%d,\"xmax\":%d,\"ymax\":%d,\"pixelw\":%d,\"pixelh\":%d",graph.LatestId,num_papers,graph.MinX,graph.MinY,graph.MaxX,graph.MaxY,TILE_PIXEL_LEN,TILE_PIXEL_LEN)

        graph.QueryNewPapersId(db)
        if graph.NewPapersId != 0 {
            fmt.Fprintf(w,",\"newid\":%d",graph.NewPapersId)
        }

        graph.QueryLastMetaDownload(db)
        if graph.LastMetaDownload != "" {
            fmt.Fprintf(w,",\"lastdl\":\"%s\"",graph.LastMetaDownload)
        }

        GenerateAllTiles(graph, w, outPrefix)
        runtime.GC()

        GenerateAllLabelZones(graph, w, outPrefix)

        fmt.Fprintf(w,"})")
        w.Flush()
    }
}

type CairoColor struct {
    r, g, b float32
}

type CategoryLabel struct {
    x,y,radius int
    label string
}

type Paper struct {
    id          uint
    x           int
    y           int
    radius      int
    age         float32
    heat        float32
    col         CairoColor
    maincat     string
    label       string
}

type PaperSortId []*Paper
func (p PaperSortId) Len() int           { return len(p) }
func (p PaperSortId) Less(i, j int) bool { return p[i].id < p[j].id }
func (p PaperSortId) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }


type Graph struct {
    papers  []*Paper
    qt      *QuadTree
    catLabels []*CategoryLabel
    MinX, MinY, MaxX, MaxY int
    BoundsX, BoundsY int
    LatestId, NewPapersId uint
    LastMetaDownload string
}

func (graph *Graph) GetPaperById(id uint) *Paper {
    lo := 0
    hi := len(graph.papers) - 1
    for lo <= hi {
        mid := (lo + hi) / 2
        if id == graph.papers[mid].id {
            return graph.papers[mid]
        } else if id < graph.papers[mid].id {
            hi = mid - 1
        } else {
            lo = mid + 1
        }
    }
    return nil
}

func cleanJsonString(input string) string {
    // TODO work out exactly which chars are causing
    // parsing error and blacklist or escape them
    // inplace of this whitelist
    validChars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 -/.,<>"

    output := make([]rune, 0)

    for _, r := range input {
        if strings.ContainsRune(validChars, r) {
            output = append(output, r)
        }
    }

    return string(output)
}

func idToDaysAgo(id uint) uint {
    tId := time.Date(((int(id) / 10000000) + 1800),time.Month((((int(id) % 10000000) / 625000) + 1)),(((int(id) % 625000) / 15625) + 1),0,0,0,0,time.UTC)
    days := uint(time.Now().Sub(tId).Hours()/24)
    return days
}

func (graph *Graph) CalculateCategoryLabels() {
    categories := []struct{
        maincat, label string
    }{
        {"hep-th","high energy theory,(hep-th),,"},
        {"hep-ph","high energy phenomenology,(hep-ph),,"},
        {"hep-ex","high energy experiment,(hep-ex),,"},
        {"gr-qc","general relativity/quantum cosmology,(gr-gc),,"},
        //{"astro-ph.GA","astronomy,astro-ph.GA,,"},
        {"hep-lat","high energy lattice,(hep-lat),,"},
        //{"astro-ph.CO","cosmology,astro-ph.CO,,"},
        {"astro-ph","astrophysics,(astro-ph),,"},
        {"cond-mat","condensed matter,(cond-mat),,"},
        {"math","math,(math),,"},
        {"cs","computer science,(cs),,"},
        {"nucl-ex","nuclear experiment,(nucl-ex),,"},
        {"nucl-th","nuclear theory,(nucl-th),,"},
        {"quant-ph","quantum physics,(quant-ph),,"},
        {"physics","general physics,(physics),,"},
    }

    for _, category := range(categories) {

        var sumMass, sumMassX, sumMassY int64
        var minX,minY,maxX,maxY int
        for _, paper := range(graph.papers) {
            if paper.maincat == category.maincat {
                mass := int64(paper.radius*paper.radius)
                sumMass += mass
                sumMassX += mass*int64(paper.x)
                sumMassY += mass*int64(paper.y)
                if paper.x < minX { minX = paper.x }
                if paper.x > maxX { maxX = paper.x }
                if paper.y < minY { minY = paper.y }
                if paper.y > maxY { maxY = paper.y }
            }<commit_msg>search pan/zoom working<commit_after>package main

import (
    "flag"
    "os"
    "bufio"
    "fmt"
    "path/filepath"
    "strings"
    "strconv"
    "GoMySQL"
    "runtime"
    "math"
    "sort"
    "log"
    "xiwi"
    "github.com/ungerik/go-cairo"
    "time"
)

var GRAPH_PADDING = 100 // what to pad graph by on each side
var TILE_PIXEL_LEN = 256

var flagDB         = flag.String("db", "", "MySQL database to connect to")
var flagGrayScale  = flag.Bool("gs", false, "Make grayscale tiles")
var flagHeatMap    = flag.Bool("hm", false, "Make heatmap tiles")
var flagDoSingle   = flag.Bool("single-tile", false, "Only generate a large single tile, no labels or world index information")

var flagSkipTiles  = flag.Bool("skip-tiles", false, "Do not generate tiles (still generates index information)")
var flagSkipLabels = flag.Bool("skip-labels", false, "Do not generate labels (still generates index information)")

var flagMaxCores = flag.Int("cores",-1,"Max number of system cores to use, default is all of them")

func main() {
    // parse command line options
    flag.Parse()

    if flag.NArg() != 1 {
        log.Fatal("need to specify output prefix")
    }
    outPrefix := flag.Arg(0)

    // connect to the db
    db := xiwi.ConnectToDB(*flagDB)
    if db == nil {
        return
    }
    defer db.Close()

    // read in the graph
    graph := ReadGraph(db)

    // build the quad tree
    graph.BuildQuadTree()
    
    runtime.GC()

    if *flagDoSingle {
        DrawTile(graph, graph.BoundsX, graph.BoundsY, 1, 1, 18000, 18000, outPrefix)
    } else {
        // Create index file
        indexFile := outPrefix + "/world_index.json"

        if err := os.MkdirAll(filepath.Dir(indexFile),0755); err != nil {
            fmt.Println(err)
            return
        }
        fo, _ := os.Create(indexFile)
        defer fo.Close()
        w := bufio.NewWriter(fo)

        num_papers := len(graph.papers)

        fmt.Fprintf(w,"world_index({\"latestid\":%d,\"numpapers\":%d,\"xmin\":%d,\"ymin\":%d,\"xmax\":%d,\"ymax\":%d,\"pixelw\":%d,\"pixelh\":%d",graph.LatestId,num_papers,graph.MinX,graph.MinY,graph.MaxX,graph.MaxY,TILE_PIXEL_LEN,TILE_PIXEL_LEN)

        graph.QueryNewPapersId(db)
        if graph.NewPapersId != 0 {
            fmt.Fprintf(w,",\"newid\":%d",graph.NewPapersId)
        }

        graph.QueryLastMetaDownload(db)
        if graph.LastMetaDownload != "" {
            fmt.Fprintf(w,",\"lastdl\":\"%s\"",graph.LastMetaDownload)
        }

        GenerateAllTiles(graph, w, outPrefix)
        runtime.GC()

        GenerateAllLabelZones(graph, w, outPrefix)

        fmt.Fprintf(w,"})")
        w.Flush()
    }
}

type CairoColor struct {
    r, g, b float32
}

type CategoryLabel struct {
    x,y,radius int
    label string
}

type Paper struct {
    id          uint
    x           int
    y           int
    radius      int
    age         float32
    heat        float32
    col         CairoColor
    maincat     string
    label       string
}

type PaperSortId []*Paper
func (p PaperSortId) Len() int           { return len(p) }
func (p PaperSortId) Less(i, j int) bool { return p[i].id < p[j].id }
func (p PaperSortId) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }


type Graph struct {
    papers  []*Paper
    qt      *QuadTree
    catLabels []*CategoryLabel
    MinX, MinY, MaxX, MaxY int
    BoundsX, BoundsY int
    LatestId, NewPapersId uint
    LastMetaDownload string
}

func (graph *Graph) GetPaperById(id uint) *Paper {
    lo := 0
    hi := len(graph.papers) - 1
    for lo <= hi {
        mid := (lo + hi) / 2
        if id == graph.papers[mid].id {
            return graph.papers[mid]
        } else if id < graph.papers[mid].id {
            hi = mid - 1
        } else {
            lo = mid + 1
        }
    }
    return nil
}

func cleanJsonString(input string) string {
    // TODO work out exactly which chars are causing
    // parsing error and blacklist or escape them
    // inplace of this whitelist
    validChars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 -/.,<>"

    output := make([]rune, 0)

    for _, r := range input {
        if strings.ContainsRune(validChars, r) {
            output = append(output, r)
        }
    }

    return string(output)
}

func idToDaysAgo(id uint) uint {
    tId := time.Date(((int(id) / 10000000) + 1800),time.Month((((int(id) % 10000000) / 625000) + 1)),(((int(id) % 625000) / 15625) + 1),0,0,0,0,time.UTC)
    days := uint(time.Now().Sub(tId).Hours()/24)
    return days
}

func (graph *Graph) CalculateCategoryLabels() {
    categories := []struct{
        maincat, label string
    }{
        {"hep-th","high energy theory,(hep-th),,"},
        {"hep-ph","high energy phenomenology,(hep-ph),,"},
        {"hep-ex","high energy experiment,(hep-ex),,"},
        {"gr-qc","general relativity/quantum cosmology,(gr-gc),,"},
        //{"astro-ph.GA","astronomy,astro-ph.GA,,"},
        {"hep-lat","high energy lattice,(hep-lat),,"},
        //{"astro-ph.CO","cosmology,astro-ph.CO,,"},
        {"astro-ph","astrophysics,(astro-ph),,"},
        {"cond-mat","condensed matter,(cond-mat),,"},
        {"math","math,(math),,"},
        {"cs","computer science,(cs),,"},
        {"nucl-ex","nuclear experiment,(nucl-ex),,"},
        //{"nucl-th","nuclear theory,(nucl-th),,"},
        {"quant-ph","quantum physics,(quant-ph),,"},
        {"physics","general physics,(physics),,"},
    }

    for _, category := range(categories) {

        var sumMass, sumMassX, sumMassY int64
        var minX,minY,maxX,maxY int
        for _, paper := range(graph.papers) {
            if paper.maincat == category.maincat {
                mass := int64(paper.radius*paper.radius)
                sumMass += mass
                sumMassX += mass*int64(paper.x)
                sumMassY += mass*int64(paper.y)
                if paper.x < minX { minX = paper.x }
                if paper.x > maxX { maxX = paper.x }
                if paper.y < minY { minY = paper.y }
                if paper.y > maxY { maxY = paper.y }
            }
{'fraction_non_alphanumeric': 0.1529382634532849, 'fraction_numerical': 0.016672169032684054, 'mean_word_length': 2.271328293736501, 'pattern_counts': {'":': 20, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5850729', 'n_tokens_mistral': 4330, 'n_tokens_neox': 4014, 'n_words': 934}