<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	_ "net/http/pprof"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/cloud"
	"google.golang.org/cloud/logging"

	"github.com/jmhodges/howsmyssl/gzip"
	"github.com/jmhodges/howsmyssl/tls"
)

const (
	resp500Format = `HTTP/1.%d 500 Internal Server Error
Content-Length: 26
Connection: close
Content-Type: text/plain; charset="utf-8"
Strict-Transport-Security: max-age=631138519; includeSubdomains; preload
Date: %s

500 Internal Server Error
`
	hstsHeaderValue = "max-age=631138519; includeSubdomains; preload"
	xForwardedProto = "X-Forwarded-Proto"
)

var (
	httpsAddr    = flag.String("httpsAddr", "localhost:10443", "address to boot the HTTPS server on")
	httpAddr     = flag.String("httpAddr", "localhost:10080", "address to boot the HTTP server on")
	rawVHost     = flag.String("vhost", "localhost:10443", "public domain to use in redirects and templates")
	certPath     = flag.String("cert", "./config/development.crt", "file path to the TLS certificate to serve with")
	keyPath      = flag.String("key", "./config/development.key", "file path to the TLS key to serve with")
	acmeURL      = flag.String("acmeRedirect", "/s/", "URL to join with .well-known/acme paths and redirect to")
	originsFile  = flag.String("originsConf", "", "file path to find the allowed origins configuration")
	googAcctConf = flag.String("googAcctConf", "", "file path to a Google service account JSON configuration")
	allowLogName = flag.String("allowLogName", "test_howsmyssl_allowance_checks", "the name to Google Cloud Logging log to send API allowance check data to")
	staticDir    = flag.String("staticDir", "./static", "file path to the directory of static files to serve")
	tmplDir      = flag.String("templateDir", "./templates", "file path to the directory of templates")
	adminAddr    = flag.String("adminAddr", "localhost:4567", "address to boot the admin server on")

	apiVars         = expvar.NewMap("api")
	staticVars      = expvar.NewMap("static")
	webVars         = expvar.NewMap("web")
	apiRequests     = new(expvar.Int)
	staticRequests  = new(expvar.Int)
	webRequests     = new(expvar.Int)
	apiStatuses     = NewStatusStats(apiVars)
	staticStatuses  = NewStatusStats(staticVars)
	webStatuses     = NewStatusStats(webVars)
	commonRedirects = expvar.NewInt("common_redirects")

	nonAlphaNumeric = regexp.MustCompile("[^[:alnum:]]")

	index *template.Template
)

func main() {
	flag.Parse()
	t := time.Now()
	expvar.NewInt("start_time_epoch_secs").Set(t.Unix())
	expvar.NewString("start_time_timestamp").Set(t.Format(time.RFC3339))
	expvar.Publish("uptime_secs", expvar.Func(func() interface{} {
		return int64(time.Now().Sub(t) / time.Second)
	}))
	expvar.Publish("uptime_dur", expvar.Func(func() interface{} {
		return time.Now().Sub(t).String()
	}))

	routeHost, redirectHost := calculateDomains(*rawVHost, *httpsAddr)

	apiVars.Set("requests", apiRequests)
	staticVars.Set("requests", staticRequests)
	webVars.Set("requests", webRequests)

	index = loadIndex()
	tlsConf := makeTLSConfig(*certPath, *keyPath)

	tlsListener, err := tls.Listen("tcp", *httpsAddr, tlsConf)
	if err != nil {
		log.Fatalf("unable to listen for the HTTPS server on %s: %s", *httpsAddr, err)
	}
	plaintextListener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("unable to listen for the HTTP server on %s: %s", *httpAddr, err)
	}
	l := &listener{tlsListener}

	if *acmeURL != "" {
		if !strings.HasPrefix(*acmeURL, "/") &&
			!strings.HasPrefix(*acmeURL, "https://") &&
			!strings.HasPrefix(*acmeURL, "http://") {
			fmt.Fprintf(os.Stderr, "acmeRedirect must start with 'http://', 'https://', or '/' but does not: %#v\n", *acmeURL)
			os.Exit(1)
		}
	}

	blockedOrigins := []string{}
	if *originsFile != "" {
		jc := loadOriginsConfig(*originsFile)
		blockedOrigins = jc.BlockedOrigins
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("unable to get hostname of local machine: %s", err)
	}

	var gclog logClient
	if *googAcctConf != "" {
		googConf := loadGoogleServiceAccount(*googAcctConf)
		tokSrc := googConf.conf.TokenSource(context.Background())
		gclog, err = logging.NewClient(context.Background(), googConf.ProjectID, *allowLogName, cloud.WithTokenSource(tokSrc))
		if err != nil {
			log.Fatalf("unable to make Google Cloud Logging client: %s", err)
		}
	} else {
		gclog = nullLogClient{}
	}
	oa := newOriginAllower(blockedOrigins, hostname, gclog, expvar.NewMap("origins"))

	m := tlsMux(
		routeHost,
		redirectHost,
		*acmeURL,
		makeStaticHandler(*staticDir, staticVars),
		oa,
	)

	go func() {
		err := http.ListenAndServe(*adminAddr, nil)
		if err != nil {
			log.Fatalf("unable to open admin server: %s", err)
		}
	}()

	log.Printf("Booting HTTPS on %s and HTTP on %s", *httpsAddr, *httpAddr)
	go func() {
		err := http.Serve(l, m)
		if err != nil {
			log.Fatalf("https server error: %s", err)
		}
	}()
	err = http.Serve(plaintextListener, plaintextMux(redirectHost))
	if err != nil {
		log.Fatalf("http server error: %s", err)
	}
}

// Returns routeHost, redirectHost
func calculateDomains(vhost, httpsAddr string) (string, string) {
	var routeHost, redirectHost string
	// Use cases to support:
	//   * Redirect to non-standard HTTPS port (that is, not 443) that is the same as the port we're booting the HTTPS server on.
	//   * Redirect to non-standard HTTPS port (not 443) that is not the same as the one the HTTPS server is booted on. (We are behind a proxy, or using a linux container, etc.)
	//   * Redirect to a host on the standard HTTPS port, 443, without including the port as it might mix up certain clients, or, at least, look uncool.
	//   * Do all of the above knowing that the host we are booting on with httpsAddr might not be the one we want to use in redirects and templates.
	if strings.Contains(vhost, ":") {
		var err error
		vport := ""
		// We can drop port in routeHost here because http.ServeMux
		// doesn't currently know how to match against ports (see
		// https://golang.org/issue/10463) and we strip ports inside
		// protoHandler to accommodate that fact. If ServeMux learns
		// how to handle ports, we can choose to use *rawVHost for it
		// then.
		routeHost, vport, err = net.SplitHostPort(vhost)
		if err != nil {
			log.Fatalf("unable to parse httpsAddr: %s", err)<commit_msg>set timeouts on incoming connections (#109)<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"expvar"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	_ "net/http/pprof"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/cloud"
	"google.golang.org/cloud/logging"

	"github.com/jmhodges/howsmyssl/gzip"
	"github.com/jmhodges/howsmyssl/tls"
)

const (
	resp500Format = `HTTP/1.%d 500 Internal Server Error
Content-Length: 26
Connection: close
Content-Type: text/plain; charset="utf-8"
Strict-Transport-Security: max-age=631138519; includeSubdomains; preload
Date: %s

500 Internal Server Error
`
	hstsHeaderValue = "max-age=631138519; includeSubdomains; preload"
	xForwardedProto = "X-Forwarded-Proto"
)

var (
	httpsAddr    = flag.String("httpsAddr", "localhost:10443", "address to boot the HTTPS server on")
	httpAddr     = flag.String("httpAddr", "localhost:10080", "address to boot the HTTP server on")
	rawVHost     = flag.String("vhost", "localhost:10443", "public domain to use in redirects and templates")
	certPath     = flag.String("cert", "./config/development.crt", "file path to the TLS certificate to serve with")
	keyPath      = flag.String("key", "./config/development.key", "file path to the TLS key to serve with")
	acmeURL      = flag.String("acmeRedirect", "/s/", "URL to join with .well-known/acme paths and redirect to")
	originsFile  = flag.String("originsConf", "", "file path to find the allowed origins configuration")
	googAcctConf = flag.String("googAcctConf", "", "file path to a Google service account JSON configuration")
	allowLogName = flag.String("allowLogName", "test_howsmyssl_allowance_checks", "the name to Google Cloud Logging log to send API allowance check data to")
	staticDir    = flag.String("staticDir", "./static", "file path to the directory of static files to serve")
	tmplDir      = flag.String("templateDir", "./templates", "file path to the directory of templates")
	adminAddr    = flag.String("adminAddr", "localhost:4567", "address to boot the admin server on")

	apiVars         = expvar.NewMap("api")
	staticVars      = expvar.NewMap("static")
	webVars         = expvar.NewMap("web")
	apiRequests     = new(expvar.Int)
	staticRequests  = new(expvar.Int)
	webRequests     = new(expvar.Int)
	apiStatuses     = NewStatusStats(apiVars)
	staticStatuses  = NewStatusStats(staticVars)
	webStatuses     = NewStatusStats(webVars)
	commonRedirects = expvar.NewInt("common_redirects")

	nonAlphaNumeric = regexp.MustCompile("[^[:alnum:]]")

	index *template.Template
)

func main() {
	flag.Parse()
	t := time.Now()
	expvar.NewInt("start_time_epoch_secs").Set(t.Unix())
	expvar.NewString("start_time_timestamp").Set(t.Format(time.RFC3339))
	expvar.Publish("uptime_secs", expvar.Func(func() interface{} {
		return int64(time.Now().Sub(t) / time.Second)
	}))
	expvar.Publish("uptime_dur", expvar.Func(func() interface{} {
		return time.Now().Sub(t).String()
	}))

	routeHost, redirectHost := calculateDomains(*rawVHost, *httpsAddr)

	apiVars.Set("requests", apiRequests)
	staticVars.Set("requests", staticRequests)
	webVars.Set("requests", webRequests)

	index = loadIndex()
	tlsConf := makeTLSConfig(*certPath, *keyPath)

	tlsListener, err := tls.Listen("tcp", *httpsAddr, tlsConf)
	if err != nil {
		log.Fatalf("unable to listen for the HTTPS server on %s: %s", *httpsAddr, err)
	}
	plaintextListener, err := net.Listen("tcp", *httpAddr)
	if err != nil {
		log.Fatalf("unable to listen for the HTTP server on %s: %s", *httpAddr, err)
	}
	l := &listener{tlsListener}

	if *acmeURL != "" {
		if !strings.HasPrefix(*acmeURL, "/") &&
			!strings.HasPrefix(*acmeURL, "https://") &&
			!strings.HasPrefix(*acmeURL, "http://") {
			fmt.Fprintf(os.Stderr, "acmeRedirect must start with 'http://', 'https://', or '/' but does not: %#v\n", *acmeURL)
			os.Exit(1)
		}
	}

	blockedOrigins := []string{}
	if *originsFile != "" {
		jc := loadOriginsConfig(*originsFile)
		blockedOrigins = jc.BlockedOrigins
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatalf("unable to get hostname of local machine: %s", err)
	}

	var gclog logClient
	if *googAcctConf != "" {
		googConf := loadGoogleServiceAccount(*googAcctConf)
		tokSrc := googConf.conf.TokenSource(context.Background())
		gclog, err = logging.NewClient(context.Background(), googConf.ProjectID, *allowLogName, cloud.WithTokenSource(tokSrc))
		if err != nil {
			log.Fatalf("unable to make Google Cloud Logging client: %s", err)
		}
	} else {
		gclog = nullLogClient{}
	}
	oa := newOriginAllower(blockedOrigins, hostname, gclog, expvar.NewMap("origins"))

	m := tlsMux(
		routeHost,
		redirectHost,
		*acmeURL,
		makeStaticHandler(*staticDir, staticVars),
		oa,
	)

	go func() {
		err := http.ListenAndServe(*adminAddr, nil)
		if err != nil {
			log.Fatalf("unable to open admin server: %s", err)
		}
	}()

	httpsSrv := &http.Server{
		Handler:      m,
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
	}

	httpSrv := &http.Server{
		Handler:      plaintextMux(redirectHost),
		ReadTimeout:  5 * time.Second,
		WriteTimeout: 10 * time.Second,
	}

	log.Printf("Booting HTTPS on %s and HTTP on %s", *httpsAddr, *httpAddr)
	go func() {
		err := httpsSrv.Serve(l)
		if err != nil {
			log.Fatalf("https server error: %s", err)
		}
	}()
	err = httpSrv.Serve(plaintextListener)
	if err != nil {
		log.Fatalf("http server error: %s", err)
	}
}

// Returns routeHost, redirectHost
func calculateDomains(vhost, httpsAddr string) (string, string) {
	var routeHost, redirectHost string
	// Use cases to support:
	//   * Redirect to non-standard HTTPS port (that is, not 443) that is the same as the port we're booting the HTTPS server on.
	//   * Redirect to non-standard HTTPS port (not 443) that is not the same as the one the HTTPS server is booted on. (We are behind a proxy, or using a linux container, etc.)
	//   * Redirect to a host on the standard HTTPS port, 443, without including the port as it might mix up certain clients, or, at least, look uncool.
	//   * Do all of the above knowing that the host we are booting on with httpsAddr might not be the one we want to use in redirects and templates.
	if strings.Contains(vhost, ":") {
		var err error
		vport := ""
		// We can drop port in routeHost here because http.ServeMux
		// doesn't currently know how to match against ports (see
		// https://golang.org/issue/10463) and we strip ports inside
		// protoHandler to accommodate that fact. If ServeMux learns
		// how to handle ports, we can choose to use *rawVHost for it
		// then.
		routeHost, vport, err = net.SplitHostPort(vhost)
		if err != nil {
			log.Fatalf("unable to parse httpsAddr: %s", err)
{'fraction_non_alphanumeric': 0.13819475995759503, 'fraction_numerical': 0.011434196577313342, 'mean_word_length': 4.187352710133543, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4348464', 'n_tokens_mistral': 4812, 'n_tokens_neox': 4554, 'n_words': 1260}