<commit_before>package main

import (
	"net/http"
	"strings"
	"time"

	"github.com/segmentio/conf"
	"github.com/segmentio/events"
	_ "github.com/segmentio/events/ecslogs"
	"github.com/segmentio/events/httpevents"
	_ "github.com/segmentio/events/text"
	"github.com/segmentio/nsq-go/nsqlookup"
)

func main() {
	config := struct {
		Bind    string `conf:"bind"    help:"The network address to listen for incoming connections on."`
		Verbose bool   `conf:"verbose" help:"Turn on verbose mode."`
		Debug   bool   `conf:"debug"   help:"Turn on debug mode."`
	}{
		Bind: ":4181",
	}

	args := conf.Load(&config)
	events.DefaultLogger.EnableDebug = config.Debug
	events.DefaultLogger.EnableSource = config.Debug

	var resolvers []nsqlookup.Resolver
	for _, addr := range args {
		switch {
		case strings.HasPrefix(addr, "consul://"):
			address, service := splitConsulAddressService(addr)
			resolvers = append(resolvers, &nsqlookup.ConsulResolver{
				Address: address,
				Service: service,
			})
		default:
			resolvers = append(resolvers, nsqlookup.Servers{addr})
		}
	}

	var proxy = &nsqlookup.ProxyEngine{
		Transport: http.DefaultTransport,
		Resolver: &nsqlookup.CachedResolver{
			Resolver: nsqlookup.MultiResolver(resolvers...),
			Timeout:  10 * time.Second,
		},
	}

	var handler http.Handler = nsqlookup.HTTPHandler{
		Engine: proxy,
	}

	if config.Verbose {
		proxy.Transport = httpevents.NewTransport(nil, proxy.Transport)
		handler = httpevents.NewHandler(nil, handler)
	}

	events.Log("starting nsqlookup-proxy listening on %{address}s", config.Bind)
	http.ListenAndServe(config.Bind, handler)
}

func splitConsulAddressService(addr string) (address string, service string) {
	addr = addr[9:] // strip leading "consul://"

	if off := strings.IndexByte(addr, '/'); off >= 0 {
		address, service = addr[:off], addr[off+1:]
	} else {
		address, service = addr, "nsqlookupd"
	}

	address = "http://" + address
	return
}<commit_msg>log calls to consul<commit_after>package main

import (
	"net/http"
	"strings"
	"time"

	"github.com/segmentio/conf"
	"github.com/segmentio/events"
	_ "github.com/segmentio/events/ecslogs"
	"github.com/segmentio/events/httpevents"
	_ "github.com/segmentio/events/text"
	"github.com/segmentio/nsq-go/nsqlookup"
)

func main() {
	config := struct {
		Bind    string `conf:"bind"    help:"The network address to listen for incoming connections on."`
		Verbose bool   `conf:"verbose" help:"Turn on verbose mode."`
		Debug   bool   `conf:"debug"   help:"Turn on debug mode."`
	}{
		Bind: ":4181",
	}

	args := conf.Load(&config)
	events.DefaultLogger.EnableDebug = config.Debug
	events.DefaultLogger.EnableSource = config.Debug

	var transport http.RoundTripper = http.DefaultTransport
	var resolvers []nsqlookup.Resolver

	if config.Verbose {
		transport = httpevents.NewTransport(nil, transport)
	}

	for _, addr := range args {
		switch {
		case strings.HasPrefix(addr, "consul://"):
			address, service := splitConsulAddressService(addr)
			resolvers = append(resolvers, &nsqlookup.ConsulResolver{
				Address:   address,
				Service:   service,
				Transport: transport,
			})
		default:
			resolvers = append(resolvers, nsqlookup.Servers{addr})
		}
	}

	var proxy = &nsqlookup.ProxyEngine{
		Transport: transport,
		Resolver: &nsqlookup.CachedResolver{
			Resolver: nsqlookup.MultiResolver(resolvers...),
			Timeout:  10 * time.Second,
		},
	}

	var handler http.Handler = nsqlookup.HTTPHandler{
		Engine: proxy,
	}

	if config.Verbose {
		handler = httpevents.NewHandler(nil, handler)
	}

	events.Log("starting nsqlookup-proxy listening on %{address}s", config.Bind)
	http.ListenAndServe(config.Bind, handler)
}

func splitConsulAddressService(addr string) (address string, service string) {
	addr = addr[9:] // strip leading "consul://"

	if off := strings.IndexByte(addr, '/'); off >= 0 {
		address, service = addr[:off], addr[off+1:]
	} else {
		address, service = addr, "nsqlookupd"
	}

	address = "http://" + address
	return
}
{'fraction_non_alphanumeric': 0.13623772349534122, 'fraction_numerical': 0.004532863258625032, 'mean_word_length': 4.158441558441559, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2822056', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1362, 'n_words': 289}