<commit_before>// Copyright (c) 2014 <NAME>
// Licensed under the MIT license

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/etix/stoppableListener"
	"github.com/garyburd/redigo/redis"
	"html/template"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

type HTTP struct {
	geoip      *GeoIP
	redis      *redisobj
	template   *template.Template
	SListener  *stoppableListener.StoppableListener
	stats      *Stats
	cache      *Cache
	Restarting bool
}

type Mirror struct {
	ID             string   `redis:"ID" yaml:"-"`
	HttpURL        string   `redis:"http" yaml:"HttpURL"`
	RsyncURL       string   `redis:"rsync" yaml:"RsyncURL"`
	FtpURL         string   `redis:"ftp" yaml:"FtpURL"`
	SponsorName    string   `redis:"sponsorName" yaml:"SponsorName"`
	SponsorURL     string   `redis:"sponsorURL" yaml:"SponsorURL"`
	SponsorLogoURL string   `redis:"sponsorLogo" yaml:"SponsorLogoURL"`
	AdminName      string   `redis:"adminName" yaml:"AdminName"`
	AdminEmail     string   `redis:"adminEmail" yaml:"AdminEmail"`
	CustomData     string   `redis:"customData" yaml:"CustomData"`
	ContinentOnly  bool     `redis:"continentOnly" yaml:"ContinentOnly"`
	CountryOnly    bool     `redis:"countryOnly" yaml:"CountryOnly"`
	ASOnly         bool     `redis:"asOnly" yaml:"ASOnly"`
	Score          int      `redis:"score" yaml:"Score"`
	Latitude       float32  `redis:"latitude" yaml:"Latitude"`
	Longitude      float32  `redis:"longitude" yaml:"Longitude"`
	ContinentCode  string   `redis:"continentCode" yaml:"ContinentCode"`
	CountryCodes   string   `redis:"countryCodes" yaml:"CountryCodes"`
	Asnum          int      `redis:"asnum" yaml:"ASNum"`
	Enabled        bool     `redis:"enabled" yaml:"Enabled"`
	Up             bool     `redis:"up" json:"-" yaml:"-"`
	ExcludeReason  string   `redis:"excludeReason" json:",omitempty" yaml:"-"`
	StateSince     int64    `redis:"stateSince" json:",omitempty" yaml:"-"`
	Distance       float32  `redis:"-" yaml:"-"`
	CountryFields  []string `redis:"-" json:"-" yaml:"-"`
	Weight         int      `redis:"-" json:"-" yaml:"-"`
	LastSync       int64    `redis:"lastSync" yaml:"-"`

	FileInfo *FileInfo `redis:"-" json:"-" yaml:"-"` // Details of the requested file on this specific mirror
}

type Mirrors []Mirror

func (s Mirrors) Len() int      { return len(s) }
func (s Mirrors) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type FileInfo struct {
	Path    string    `redis:"-"`
	Size    int64     `redis:"size" json:",omitempty"`
	ModTime time.Time `redis:"modTime" json:",omitempty"`
	Sha1    string    `redis:"sha1" json:",omitempty"`
}

type MirrorlistPage struct {
	FileInfo     FileInfo
	MapURL       string `json:"-"`
	IP           string
	ClientInfo   GeoIPRec
	MirrorList   Mirrors
	ExcludedList Mirrors `json:",omitempty"`
	Fallback     bool    `json:",omitempty"`
}

func HTTPServer(redis *redisobj, cache *Cache) *HTTP {
	h := new(HTTP)
	h.redis = redis
	h.geoip = NewGeoIP()
	h.template = template.Must(h.LoadTemplates())
	h.cache = cache
	h.stats = NewStats(redis)
	http.Handle("/", NewGzipHandler(h.requestDispatcher))

	if err := h.geoip.LoadGeoIP(); err != nil {
		log.Critical("Can't load the GeoIP databases: %v", err)
		if len(GetConfig().Fallbacks) > 0 {
			log.Warning("All requests will be served by the backup mirrors!")
		} else {
			log.Error("Please configure fallback mirrors!")
		}
	}
	rand.Seed(time.Now().UnixNano())
	return h
}

func (h *HTTP) SetListener(l net.Listener) {
	/* Make the listener stoppable to be able to shutdown the server gracefully */
	h.SListener = stoppableListener.Handle(l)
}

func (h *HTTP) Terminate() {
	/* Close the listener, killing all active connections */
	h.SListener.Close()<commit_msg>Few more lines of comments<commit_after>// Copyright (c) 2014 <NAME>
// Licensed under the MIT license

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/etix/stoppableListener"
	"github.com/garyburd/redigo/redis"
	"html/template"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

type HTTP struct {
	geoip      *GeoIP
	redis      *redisobj
	template   *template.Template
	SListener  *stoppableListener.StoppableListener
	stats      *Stats
	cache      *Cache
	Restarting bool
}

type Mirror struct {
	ID             string   `redis:"ID" yaml:"-"`
	HttpURL        string   `redis:"http" yaml:"HttpURL"`
	RsyncURL       string   `redis:"rsync" yaml:"RsyncURL"`
	FtpURL         string   `redis:"ftp" yaml:"FtpURL"`
	SponsorName    string   `redis:"sponsorName" yaml:"SponsorName"`
	SponsorURL     string   `redis:"sponsorURL" yaml:"SponsorURL"`
	SponsorLogoURL string   `redis:"sponsorLogo" yaml:"SponsorLogoURL"`
	AdminName      string   `redis:"adminName" yaml:"AdminName"`
	AdminEmail     string   `redis:"adminEmail" yaml:"AdminEmail"`
	CustomData     string   `redis:"customData" yaml:"CustomData"`
	ContinentOnly  bool     `redis:"continentOnly" yaml:"ContinentOnly"`
	CountryOnly    bool     `redis:"countryOnly" yaml:"CountryOnly"`
	ASOnly         bool     `redis:"asOnly" yaml:"ASOnly"`
	Score          int      `redis:"score" yaml:"Score"`
	Latitude       float32  `redis:"latitude" yaml:"Latitude"`
	Longitude      float32  `redis:"longitude" yaml:"Longitude"`
	ContinentCode  string   `redis:"continentCode" yaml:"ContinentCode"`
	CountryCodes   string   `redis:"countryCodes" yaml:"CountryCodes"`
	Asnum          int      `redis:"asnum" yaml:"ASNum"`
	Enabled        bool     `redis:"enabled" yaml:"Enabled"`
	Up             bool     `redis:"up" json:"-" yaml:"-"`
	ExcludeReason  string   `redis:"excludeReason" json:",omitempty" yaml:"-"`
	StateSince     int64    `redis:"stateSince" json:",omitempty" yaml:"-"`
	Distance       float32  `redis:"-" yaml:"-"`
	CountryFields  []string `redis:"-" json:"-" yaml:"-"`
	Weight         int      `redis:"-" json:"-" yaml:"-"`
	LastSync       int64    `redis:"lastSync" yaml:"-"`

	FileInfo *FileInfo `redis:"-" json:"-" yaml:"-"` // Details of the requested file on this specific mirror
}

type Mirrors []Mirror

func (s Mirrors) Len() int      { return len(s) }
func (s Mirrors) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type FileInfo struct {
	Path    string    `redis:"-"`
	Size    int64     `redis:"size" json:",omitempty"`
	ModTime time.Time `redis:"modTime" json:",omitempty"`
	Sha1    string    `redis:"sha1" json:",omitempty"`
}

type MirrorlistPage struct {
	FileInfo     FileInfo
	MapURL       string `json:"-"`
	IP           string
	ClientInfo   GeoIPRec
	MirrorList   Mirrors
	ExcludedList Mirrors `json:",omitempty"`
	Fallback     bool    `json:",omitempty"`
}

func HTTPServer(redis *redisobj, cache *Cache) *HTTP {
	h := new(HTTP)
	h.redis = redis
	h.geoip = NewGeoIP()
	h.template = template.Must(h.LoadTemplates())
	h.cache = cache
	h.stats = NewStats(redis)
	http.Handle("/", NewGzipHandler(h.requestDispatcher))

	// Load the GeoIP database
	if err := h.geoip.LoadGeoIP(); err != nil {
		log.Critical("Can't load the GeoIP databases: %v", err)
		if len(GetConfig().Fallbacks) > 0 {
			log.Warning("All requests will be served by the backup mirrors!")
		} else {
			log.Error("Please configure fallback mirrors!")
		}
	}

	// Initialize the random number generator
	rand.Seed(time.Now().UnixNano())
	return h
}

func (h *HTTP) SetListener(l net.Listener) {
	/* Make the listener stoppable to be able to shutdown the server gracefully */
	h.SListener = stoppableListener.Handle(l)
}

func (h *HTTP) Terminate() {
	/* Close the listener, killing all active connections */
	h.SListener.Close()
{'fraction_non_alphanumeric': 0.14374586366644607, 'fraction_numerical': 0.005029781601588352, 'mean_word_length': 3.56004828002414, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1898363', 'n_tokens_mistral': 2760, 'n_tokens_neox': 2612, 'n_words': 644}