<commit_before>package main

// read in all files from netdb dir into a slice of routerinfos

// for every unique requesting IP
// look up that IP in the db
// - if it exists, check the creation time
// - if the creation time is within the threshold, serve up the routerinfos
// - if the creation time is outside the threshold, or if it does not exist generate a new slice of routerinfos from the current master set

// at some regular interval, update the master slice with fresh netdb routerinfos

// can serve up html/ul of routerinfos
// can serve up su3 signed file
// https://geti2p.net/en/docs/spec/updates

import (
	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"
)

func proxiedHandler(h http.Handler) http.Handler {
	return remoteAddrFixup{h}
}

type remoteAddrFixup struct {
	h http.Handler
}

func (h remoteAddrFixup) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if prior, ok := r.Header["X-Forwarded-For"]; ok {
		r.RemoteAddr = prior[0]
	}
	h.h.ServeHTTP(w, r)
}

type Reseeder struct {
	NetDBDir        string
	nextMap         chan []string
	RefreshInterval time.Duration
	Proxy           bool
	Verbose         bool
	RateLimit       int

	listTemplate *template.Template
}

func (rs *Reseeder) Start(addr, port, cert, key string) {
	var err error

	go rs.runMap()
	go rs.refresher()

	// parse the template for routerInfo lists
	rs.listTemplate, err = template.New("routerinfos").Parse(`<html><head><title>NetDB</title></head><body><ul>{{ range . }}<li><a href="{{ . }}">{{ . }}</a></li>{{ end }}</ul></body></html>`)
	if err != nil {
		log.Fatalln("error parsing routerInfo list template", err)
		return
	}

	r := mux.NewRouter()
	s := r.PathPrefix("/netdb").Subrouter()

	s.HandleFunc("/", rs.listHandler)
	s.HandleFunc("/i2pseeds.su3", rs.su3Handler)
	s.HandleFunc(`/routerInfo-{hash:[A-Za-z0-9+/\-=~]+}.dat`, rs.routerInfoHandler)

	// timeout
	muxWithMiddlewares := http.TimeoutHandler(r, time.Second*5, "Timeout!")

	if rs.Proxy {
		muxWithMiddlewares = proxiedHandler(muxWithMiddlewares)
	}

	th := throttled.RateLimit(throttled.PerMin(rs.RateLimit), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(0))
	muxWithMiddlewares = th.Throttle(muxWithMiddlewares)

	if rs.Verbose {<commit_msg>fix order of rate limiter middleware<commit_after>package main

// read in all files from netdb dir into a slice of routerinfos

// for every unique requesting IP
// look up that IP in the db
// - if it exists, check the creation time
// - if the creation time is within the threshold, serve up the routerinfos
// - if the creation time is outside the threshold, or if it does not exist generate a new slice of routerinfos from the current master set

// at some regular interval, update the master slice with fresh netdb routerinfos

// can serve up html/ul of routerinfos
// can serve up su3 signed file
// https://geti2p.net/en/docs/spec/updates

import (
	"github.com/PuerkitoBio/throttled"
	"github.com/PuerkitoBio/throttled/store"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"time"
)

func proxiedHandler(h http.Handler) http.Handler {
	return remoteAddrFixup{h}
}

type remoteAddrFixup struct {
	h http.Handler
}

func (h remoteAddrFixup) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if prior, ok := r.Header["X-Forwarded-For"]; ok {
		r.RemoteAddr = prior[0]
	}
	h.h.ServeHTTP(w, r)
}

type Reseeder struct {
	NetDBDir        string
	nextMap         chan []string
	RefreshInterval time.Duration
	Proxy           bool
	Verbose         bool
	RateLimit       int

	listTemplate *template.Template
}

func (rs *Reseeder) Start(addr, port, cert, key string) {
	var err error

	go rs.runMap()
	go rs.refresher()

	// parse the template for routerInfo lists
	rs.listTemplate, err = template.New("routerinfos").Parse(`<html><head><title>NetDB</title></head><body><ul>{{ range . }}<li><a href="{{ . }}">{{ . }}</a></li>{{ end }}</ul></body></html>`)
	if err != nil {
		log.Fatalln("error parsing routerInfo list template", err)
		return
	}

	r := mux.NewRouter()
	s := r.PathPrefix("/netdb").Subrouter()

	s.HandleFunc("/", rs.listHandler)
	s.HandleFunc("/i2pseeds.su3", rs.su3Handler)
	s.HandleFunc(`/routerInfo-{hash:[A-Za-z0-9+/\-=~]+}.dat`, rs.routerInfoHandler)

	// timeout
	muxWithMiddlewares := http.TimeoutHandler(r, time.Second*5, "Timeout!")

	th := throttled.RateLimit(throttled.PerMin(rs.RateLimit), &throttled.VaryBy{RemoteAddr: true}, store.NewMemStore(1000))
	muxWithMiddlewares = th.Throttle(muxWithMiddlewares)

	if rs.Proxy {
		muxWithMiddlewares = proxiedHandler(muxWithMiddlewares)
	}

	if rs.Verbose {
{'fraction_non_alphanumeric': 0.13804573804573805, 'fraction_numerical': 0.004781704781704782, 'mean_word_length': 3.8892276422764227, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5057810', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1683, 'n_words': 447}