<commit_before>// Application that serves up the contents of /tmp/glog via HTTP, giving access
// to logs w/o needing to SSH into the server.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/golang/glog"
	metrics "github.com/rcrowley/go-metrics"
	"skia.googlesource.com/buildbot.git/go/common"
	"skia.googlesource.com/buildbot.git/go/util"
)

var (
	port           = flag.String("port", ":10115", "HTTP service address (e.g., ':10115')")
	dir            = flag.String("dir", "/tmp/glog", "Directory to serve log files from.")
	graphiteServer = flag.String("graphite_server", "skiamonitor.com:2003", "Where is Graphite metrics ingestion server running.")
	stateFile      = flag.String("state_file", "/tmp/logserver.state", "File where logserver stores all encountered log files. This ensures that metrics are not duplicated for already processed log files.")

	appLogThreshold = flag.Int64(
		"app_log_threshold", 1<<30,
		"If any app's logs for a log level use up more than app_log_threshold value then the files with the oldest modified time are deleted till size is less than app_log_threshold - app_log_threshold_buffer.")
	appLogThresholdBuffer = flag.Int64(
		"app_log_threshold_buffer", 10*1<<20,
		"If any app's logs for a log level use up more than app_log_threshold then the files with the oldest modified time are deleted till size is less than app_log_threshold - app_log_threshold_buffer.")
	dirWatchDuration = flag.Duration("dir_watch_duration", 10*time.Second, "How long dir watcher sleeps for before checking the dir.")
)

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
//
// To use the operating system's file system implementation,
// use http.Dir:
//
//     http.Handle("/", FileServer(http.Dir("/tmp")))
//
// Differs from net/http FileServer by making directory listings better.
func FileServer(root http.FileSystem) http.Handler {
	return &fileHandler{root}
}

type fileHandler struct {
	root http.FileSystem
}

func (f *fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	upath := r.URL.Path
	if !strings.HasPrefix(upath, "/") {
		upath = "/" + upath
		r.URL.Path = upath
	}
	serveFile(w, r, f.root, path.Clean(upath))
}

// FileInfoNameSlice is for sorting files by their names.
type FileInfoNameSlice []os.FileInfo

func (p FileInfoNameSlice) Len() int           { return len(p) }
func (p FileInfoNameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
func (p FileInfoNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// FileInfoModifiedSlice is for sorting files by their modified time.
type FileInfoModifiedSlice []os.FileInfo<commit_msg>Fix logserver handling of .INFO files<commit_after>// Application that serves up the contents of /tmp/glog via HTTP, giving access
// to logs w/o needing to SSH into the server.
package main

import (
	"bufio"
	"encoding/gob"
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/golang/glog"
	metrics "github.com/rcrowley/go-metrics"
	"skia.googlesource.com/buildbot.git/go/common"
	"skia.googlesource.com/buildbot.git/go/util"
)

var (
	port           = flag.String("port", ":10115", "HTTP service address (e.g., ':10115')")
	dir            = flag.String("dir", "/tmp/glog", "Directory to serve log files from.")
	graphiteServer = flag.String("graphite_server", "skiamonitor.com:2003", "Where is Graphite metrics ingestion server running.")
	stateFile      = flag.String("state_file", "/tmp/logserver.state", "File where logserver stores all encountered log files. This ensures that metrics are not duplicated for already processed log files.")

	appLogThreshold = flag.Int64(
		"app_log_threshold", 1<<30,
		"If any app's logs for a log level use up more than app_log_threshold value then the files with the oldest modified time are deleted till size is less than app_log_threshold - app_log_threshold_buffer.")
	appLogThresholdBuffer = flag.Int64(
		"app_log_threshold_buffer", 10*1<<20,
		"If any app's logs for a log level use up more than app_log_threshold then the files with the oldest modified time are deleted till size is less than app_log_threshold - app_log_threshold_buffer.")
	dirWatchDuration = flag.Duration("dir_watch_duration", 10*time.Second, "How long dir watcher sleeps for before checking the dir.")
)

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
//
// To use the operating system's file system implementation,
// use http.Dir:
//
//     http.Handle("/", FileServer(http.Dir("/tmp")))
//
// Differs from net/http FileServer by making directory listings better.
func FileServer(root http.FileSystem) http.Handler {
	return &fileHandler{root}
}

type fileHandler struct {
	root http.FileSystem
}

func (f *fileHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	upath := r.URL.Path
	if !strings.HasPrefix(upath, "/") {
		upath = "/" + upath
		r.URL.Path = upath
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	serveFile(w, r, f.root, path.Clean(upath))
}

// FileInfoNameSlice is for sorting files by their names.
type FileInfoNameSlice []os.FileInfo

func (p FileInfoNameSlice) Len() int           { return len(p) }
func (p FileInfoNameSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
func (p FileInfoNameSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

// FileInfoModifiedSlice is for sorting files by their modified time.
type FileInfoModifiedSlice []os.FileInfo
{'fraction_non_alphanumeric': 0.12145249427110875, 'fraction_numerical': 0.010047593865679535, 'mean_word_length': 4.130198915009042, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6621314', 'n_tokens_mistral': 1904, 'n_tokens_neox': 1814, 'n_words': 597}