<commit_before>package proxy

import (
	log "github.com/Sirupsen/logrus"
	"github.com/Sirupsen/logrus/formatters/logstash"
	"net/http"
	"os"
	"regexp"
)

type ProxyConfig struct {
	Master      string
	Slaves      []string
	AwsConfig   *AWSConfig
	LogLocation string
}

func init() {
	log.SetFormatter(&logstash.LogstashFormatter{Type: "solr-proxy"})
}

type Proxy struct {
	updater *Updater
	reader  *Reader
	config  *ProxyConfig
}

func NewProxy(proxyConfig *ProxyConfig) *Proxy {
	updater := NewUpdater(proxyConfig.Master)
	reader := NewReader(proxyConfig.Slaves)

	if proxyConfig.LogLocation == "stdout" {
		log.SetOutput(os.Stdout)
	} else {
		f, err := os.OpenFile(proxyConfig.LogLocation, os.O_WRONLY|os.O_CREATE, 0755)
		if err != nil {
			panic(err)
		}
		log.SetOutput(f)
	}

	return &Proxy{
		updater: updater,
		reader:  reader,
		config:  proxyConfig,
	}
}

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	re := regexp.MustCompile("\\/solr\\/(.*)\\/update$")
	matches := re.FindStringSubmatch(req.URL.Path)

	req.Close = true

	if len(matches) > 0 {
		p.updater.ServeHTTP(w, req, p.config.AwsConfig, matches[1])
	} else {
		p.reader.ServeHTTP(w, req)
	}
}

func writeLog(message string, params ...string) {
	log.Printf(message, params)
}
<commit_msg>fix build<commit_after>package proxy

import (
	log "github.com/Sirupsen/logrus"
	"net/http"
	"os"
	"regexp"
)

type ProxyConfig struct {
	Master      string
	Slaves      []string
	AwsConfig   *AWSConfig
	LogLocation string
}

func init() {
	log.SetFormatter(&log.JSONFormatter{})
}

type Proxy struct {
	updater *Updater
	reader  *Reader
	config  *ProxyConfig
}

func NewProxy(proxyConfig *ProxyConfig) *Proxy {
	updater := NewUpdater(proxyConfig.Master)
	reader := NewReader(proxyConfig.Slaves)

	if proxyConfig.LogLocation == "stdout" {
		log.SetOutput(os.Stdout)
	} else {
		f, err := os.OpenFile(proxyConfig.LogLocation, os.O_WRONLY|os.O_CREATE, 0755)
		if err != nil {
			panic(err)
		}
		log.SetOutput(f)
	}

	return &Proxy{
		updater: updater,
		reader:  reader,
		config:  proxyConfig,
	}
}

func (p *Proxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	re := regexp.MustCompile("\\/solr\\/(.*)\\/update$")
	matches := re.FindStringSubmatch(req.URL.Path)

	req.Close = true

	if len(matches) > 0 {
		p.updater.ServeHTTP(w, req, p.config.AwsConfig, matches[1])
	} else {
		p.reader.ServeHTTP(w, req)
	}
}

func writeLog(message string, params ...string) {
	log.Printf(message, params)
}

{'fraction_non_alphanumeric': 0.14343109682603455, 'fraction_numerical': 0.004821213338690237, 'mean_word_length': 3.970059880239521, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3556532', 'n_tokens_mistral': 1002, 'n_tokens_neox': 968, 'n_words': 187}