<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ContextLogic/eventmaster/eventmaster"
	log "github.com/Sirupsen/logrus"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	metrics "github.com/rcrowley/go-metrics"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
)

type dbConfig struct {
	CassandraAddr  []string `json:"cassandra_addr"`
	Keyspace       string   `json:"cassandra_keyspace"`
	Consistency    string   `json:"consistency"`
	ESAddr         []string `json:"es_addr"`
	ESUsername     string   `json:"es_username"`
	ESPassword     string   `json:"es_password"`
	FlushInterval  int      `json:"flush_interval"`
	UpdateInterval int      `json:"update_interval"`
	CertFile       string   `json:"cert_file"`
	KeyFile        string   `json:"key_file"`
	CAFile         string   `json:"ca_file"`
}

func getConfig() dbConfig {
	dbConf := dbConfig{}
	confFile, err := ioutil.ReadFile("db_config.json")
	if err != nil {
		fmt.Println("No db_config file specified")
	} else {
		err = json.Unmarshal(confFile, &dbConf)
		if err != nil {
			fmt.Println("Error parsing db_config.json, using defaults:", err)
		}
	}
	if dbConf.Keyspace == "" {
		dbConf.Keyspace = "event_master"
	}
	if dbConf.Consistency == "" {
		dbConf.Consistency = "local_quorum"
	}
	if dbConf.FlushInterval == 0 {
		dbConf.FlushInterval = 5
	}
	if dbConf.UpdateInterval == 0 {
		dbConf.UpdateInterval = 5
	}
	if dbConf.CassandraAddr == nil || len(dbConf.CassandraAddr) == 0 {
		dbConf.CassandraAddr = append(dbConf.CassandraAddr, "127.0.0.1:9042")
	}
	if dbConf.ESAddr == nil || len(dbConf.ESAddr) == 0 {
		dbConf.ESAddr = append(dbConf.ESAddr, "http://127.0.0.1:9200")
	}
	return dbConf
}

func getHTTPServer(store *EventStore, registry metrics.Registry) *http.Server {
	r := httprouter.New()
	h := httpHandler{
		store: store,
	}

	// API endpoints
	r.POST("/v1/event", wrapHandler(h.handleAddEvent, registry))
	r.GET("/v1/event", wrapHandler(h.handleGetEvent, registry))
	r.POST("/v1/topic", wrapHandler(h.handleAddTopic, registry))
	r.PUT("/v1/topic/:name", wrapHandler(h.handleUpdateTopic, registry))
	r.GET("/v1/topic", wrapHandler(h.handleGetTopic, registry))
	r.DELETE("/v1/topic/:name", wrapHandler(h.handleDeleteTopic, registry))
	r.POST("/v1/dc", wrapHandler(h.handleAddDc, registry))
	r.PUT("/v1/dc/:name", wrapHandler(h.handleUpdateDc, registry))
	r.GET("/v1/dc/", wrapHandler(h.handleGetDc, registry))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", wrapHandler(h.handleGitHubEvent, registry))

	// UI endpoints
	r.GET("/", HandleMainPage)
	r.GET("/add_event", HandleCreatePage)
	r.GET("/topic", HandleTopicPage)
	r.GET("/dc", HandleDcPage)

	// JS file endpoints
	r.ServeFiles("/js/*filepath", http.Dir("ui/js"))
	r.ServeFiles("/bootstrap/*filepath", http.Dir("ui/bootstrap"))
	r.ServeFiles("/css/*filepath", http.Dir("ui/css"))

	return &http.Server{
		Handler: r,
	}
}

func main() {
	var config Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		log.Fatalf("Error parsing flags: %v", err)
	}

	r := metrics.NewRegistry()

	// Set up event store
	dbConf := getConfig()
	store, err := NewEventStore(dbConf, config, r)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	httpS := getHTTPServer(store, r)

	// Create the EventMaster server
	grpcServer, err := NewGRPCServer(&config, store, r)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}
<commit_msg>Expose metrics on metrics client using expvar<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ContextLogic/eventmaster/eventmaster"
	log "github.com/Sirupsen/logrus"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	metrics "github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/exp"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
)

type dbConfig struct {
	CassandraAddr  []string `json:"cassandra_addr"`
	Keyspace       string   `json:"cassandra_keyspace"`
	Consistency    string   `json:"consistency"`
	ESAddr         []string `json:"es_addr"`
	ESUsername     string   `json:"es_username"`
	ESPassword     string   `json:"es_password"`
	FlushInterval  int      `json:"flush_interval"`
	UpdateInterval int      `json:"update_interval"`
	CertFile       string   `json:"cert_file"`
	KeyFile        string   `json:"key_file"`
	CAFile         string   `json:"ca_file"`
}

func getConfig() dbConfig {
	dbConf := dbConfig{}
	confFile, err := ioutil.ReadFile("db_config.json")
	if err != nil {
		fmt.Println("No db_config file specified")
	} else {
		err = json.Unmarshal(confFile, &dbConf)
		if err != nil {
			fmt.Println("Error parsing db_config.json, using defaults:", err)
		}
	}
	if dbConf.Keyspace == "" {
		dbConf.Keyspace = "event_master"
	}
	if dbConf.Consistency == "" {
		dbConf.Consistency = "local_quorum"
	}
	if dbConf.FlushInterval == 0 {
		dbConf.FlushInterval = 5
	}
	if dbConf.UpdateInterval == 0 {
		dbConf.UpdateInterval = 5
	}
	if dbConf.CassandraAddr == nil || len(dbConf.CassandraAddr) == 0 {
		dbConf.CassandraAddr = append(dbConf.CassandraAddr, "127.0.0.1:9042")
	}
	if dbConf.ESAddr == nil || len(dbConf.ESAddr) == 0 {
		dbConf.ESAddr = append(dbConf.ESAddr, "http://127.0.0.1:9200")
	}
	return dbConf
}

func getHTTPServer(store *EventStore, registry metrics.Registry) *http.Server {
	r := httprouter.New()
	h := httpHandler{
		store: store,
	}

	// API endpoints
	r.POST("/v1/event", wrapHandler(h.handleAddEvent, registry))
	r.GET("/v1/event", wrapHandler(h.handleGetEvent, registry))
	r.POST("/v1/topic", wrapHandler(h.handleAddTopic, registry))
	r.PUT("/v1/topic/:name", wrapHandler(h.handleUpdateTopic, registry))
	r.GET("/v1/topic", wrapHandler(h.handleGetTopic, registry))
	r.DELETE("/v1/topic/:name", wrapHandler(h.handleDeleteTopic, registry))
	r.POST("/v1/dc", wrapHandler(h.handleAddDc, registry))
	r.PUT("/v1/dc/:name", wrapHandler(h.handleUpdateDc, registry))
	r.GET("/v1/dc/", wrapHandler(h.handleGetDc, registry))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", wrapHandler(h.handleGitHubEvent, registry))

	// UI endpoints
	r.GET("/", HandleMainPage)
	r.GET("/add_event", HandleCreatePage)
	r.GET("/topic", HandleTopicPage)
	r.GET("/dc", HandleDcPage)

	// JS file endpoints
	r.ServeFiles("/js/*filepath", http.Dir("ui/js"))
	r.ServeFiles("/bootstrap/*filepath", http.Dir("ui/bootstrap"))
	r.ServeFiles("/css/*filepath", http.Dir("ui/css"))

	return &http.Server{
		Handler: r,
	}
}

func main() {
	var config Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		log.Fatalf("Error parsing flags: %v", err)
	}

	r := metrics.NewRegistry()

	// Set up event store
	dbConf := getConfig()
	store, err := NewEventStore(dbConf, config, r)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	exp.Exp(metrics.DefaultRegistry)
	sock, err := net.Listen("tcp", "0.0.0.0:12345")
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	go func() {
		fmt.Println("go-metrics server listening at port 12345")
		http.Serve(sock, nil)
	}()

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	httpS := getHTTPServer(store, r)

	// Create the EventMaster server
	grpcServer, err := NewGRPCServer(&config, store, r)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

{'fraction_non_alphanumeric': 0.16519743992271466, 'fraction_numerical': 0.010868252626494384, 'mean_word_length': 4.18922305764411, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 5, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '793708', 'n_tokens_mistral': 3175, 'n_tokens_neox': 3065, 'n_words': 614}