<commit_before>package main

import (
	"fmt"
	"github.com/golang/glog"
	"net/http"
	"strings"
)

type domainResolver interface {
	resolve(domain string) (http.Handler, error)
	init()
}

type proxy struct {
	config         *Config
	domainResolver domainResolver
}

func NewProxy(c *Config, resolver domainResolver) *proxy {
	return &proxy{c, resolver}
}

type proxyHandler func(http.ResponseWriter, *http.Request) (*Config, error)

func (ph proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	defer func() {
		if r := recover(); r != nil {
			http.Error(w, "An error occured serving request", 500)
			glog.Errorf("Recovered from error : %s", r)
		}
	}()

	if c, err := ph(w, r); err != nil {
		ph.OnError(w, r, err, c)
	}
}

func (ph proxyHandler) OnError(w http.ResponseWriter, r *http.Request, error error, c *Config) {
	if stError, ok := error.(StatusError); ok {
		sp := &StatusPage{c, stError}
		// Check if status is passivated -> setting expected state = started
		if sp.error.computedStatus == PASSIVATED_STATUS {
			reactivate(sp, c)
		}
		sp.serve(w, r)
	} else {
		sp := &StatusPage{c, StatusError{"notfound", nil}}
		sp.serve(w, r)
	}
}

func (p *proxy) start() {
	glog.Infof("Listening on port %d", p.config.port)
	http.Handle("/__static__/", http.FileServer(http.Dir(p.config.templateDir)))
	http.Handle("/", proxyHandler(p.proxy))
	glog.Fatalf("%s", http.ListenAndServe(fmt.Sprintf(":%d", p.config.port), nil))

}

func (p *proxy) proxy(w http.ResponseWriter, r *http.Request) (*Config, error) {

	if p.config.forceFwSsl && "https" != r.Header.Get("x-forwarded-proto") {

		http.Redirect(w, r, fmt.Sprintf("https://%s%s", hostnameOf(r.Host), r.URL.String()), http.StatusMovedPermanently)
		return p.config, nil
	}

	host := hostnameOf(r.Host)
	if server, err := p.domainResolver.resolve(host); err != nil {
		return p.config, err
	} else {
		server.ServeHTTP(w, r)
		return p.config, nil
	}
}

func hostnameOf(host string) string {
	hostname := strings.Split(host, ":")[0]

	if len(hostname) > 4 && hostname[0:4] == "www." {
		hostname = hostname[4:]
	}

	return hostname
}

func reactivate(sp *StatusPage, c *Config) {
	client, _ := c.getEtcdClient()
	_, error := client.Set(c.servicePrefix+"/"+sp.error.status.service.name+"/"+sp.error.status.service.index+"/status/expected", STARTED_STATUS, 0)
	if error != nil {
		glog.Errorf("Fail: setting expected state = 'started' for instance %s. Error:%s", sp.error.status.service.name, error)
	}
	glog.Infof("Instance %s is ready for re-activation", sp.error.status.service.name)
}
<commit_msg>NXIO-423: robots.txt block index bots<commit_after>package main

import (
	"fmt"
	"github.com/golang/glog"
	"net/http"
	"strings"
)

type domainResolver interface {
	resolve(domain string) (http.Handler, error)
	init()
}

type proxy struct {
	config         *Config
	domainResolver domainResolver
}

func NewProxy(c *Config, resolver domainResolver) *proxy {
	return &proxy{c, resolver}
}

type proxyHandler func(http.ResponseWriter, *http.Request) (*Config, error)

func (ph proxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	defer func() {
		if r := recover(); r != nil {
			http.Error(w, "An error occured serving request", 500)
			glog.Errorf("Recovered from error : %s", r)
		}
	}()

	if c, err := ph(w, r); err != nil {
		ph.OnError(w, r, err, c)
	}
}

func (ph proxyHandler) OnError(w http.ResponseWriter, r *http.Request, error error, c *Config) {
	if stError, ok := error.(StatusError); ok {
		sp := &StatusPage{c, stError}
		// Check if status is passivated -> setting expected state = started
		if sp.error.computedStatus == PASSIVATED_STATUS {
			reactivate(sp, c)
		}
		sp.serve(w, r)
	} else {
		sp := &StatusPage{c, StatusError{"notfound", nil}}
		sp.serve(w, r)
	}
}

func (p *proxy) start() {
	glog.Infof("Listening on port %d", p.config.port)
	http.Handle("/__static__/", http.FileServer(http.Dir(p.config.templateDir)))
	http.HandleFunc("/robots.txt", robots)
	http.Handle("/", proxyHandler(p.proxy))
	glog.Fatalf("%s", http.ListenAndServe(fmt.Sprintf(":%d", p.config.port), nil))

}

func robots(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "User-agent: *\nDisallow: /")
}

func (p *proxy) proxy(w http.ResponseWriter, r *http.Request) (*Config, error) {

	if p.config.forceFwSsl && "https" != r.Header.Get("x-forwarded-proto") {

		http.Redirect(w, r, fmt.Sprintf("https://%s%s", hostnameOf(r.Host), r.URL.String()), http.StatusMovedPermanently)
		return p.config, nil
	}

	host := hostnameOf(r.Host)
	if server, err := p.domainResolver.resolve(host); err != nil {
		return p.config, err
	} else {
		server.ServeHTTP(w, r)
		return p.config, nil
	}
}

func hostnameOf(host string) string {
	hostname := strings.Split(host, ":")[0]

	if len(hostname) > 4 && hostname[0:4] == "www." {
		hostname = hostname[4:]
	}

	return hostname
}

func reactivate(sp *StatusPage, c *Config) {
	client, _ := c.getEtcdClient()
	_, error := client.Set(c.servicePrefix+"/"+sp.error.status.service.name+"/"+sp.error.status.service.index+"/status/expected", STARTED_STATUS, 0)
	if error != nil {
		glog.Errorf("Fail: setting expected state = 'started' for instance %s. Error:%s", sp.error.status.service.name, error)
	}
	glog.Infof("Instance %s is ready for re-activation", sp.error.status.service.name)
}

{'fraction_non_alphanumeric': 0.17024886877828055, 'fraction_numerical': 0.003959276018099547, 'mean_word_length': 4.273359840954274, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '578805', 'n_tokens_mistral': 2032, 'n_tokens_neox': 1946, 'n_words': 454}