<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"

	"github.com/codahale/hdrhistogram"
)

type MeasuredResponse struct {
	sz      uint64
	code    int
	latency int64
	timeout bool
	err     error
}

func newClient(
	compress bool,
	https bool,
	reuse bool,
) *http.Client {
	tr := http.Transport{
		DisableCompression: !compress,
		DisableKeepAlives:  !reuse,
	}
	if https {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	return &http.Client{Transport: &tr}
}

func sendRequest(
	client *http.Client,
	url *url.URL,
	host *string,
	received chan *MeasuredResponse,
) {
	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		fmt.Fprintln(os.Stderr, "\n")
	}
	req.Host = *host

	// FIX: find a way to measure latency with the http client.
	start := time.Now()
	response, err := client.Do(req)

	elapsed := time.Since(start)
	if err != nil {
		received <- &MeasuredResponse{0, 0, 0, false, err}
	} else {
		sz, _ := io.Copy(ioutil.Discard, response.Body)
		response.Body.Close()
		received <- &MeasuredResponse{uint64(sz), response.StatusCode, elapsed.Nanoseconds(), false, nil}
	}
}

func exUsage(msg string) {
	fmt.Fprintf(os.Stderr, "%s\n", msg)
	fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", path.Base(os.Args[0]))
	flag.PrintDefaults()
	os.Exit(64)
}

func main() {
	qps := flag.Int("qps", 1, "QPS to send to backends per request thread")
	concurrency := flag.Uint("concurrency", 1, "Number of request threads")
	host := flag.String("host", "web", "value of Host header to set")
	urldest := flag.String("url", "http://localhost:4140/", "Destination url")
	interval := flag.Duration("interval", 10*time.Second, "reporting interval")
	reuse := flag.Bool("reuse", false, "reuse connections")
	compress := flag.Bool("compress", false, "use compression")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
	}

	flag.Parse()

	if *qps < 1 {
		exUsage("qps must be at least 1")
	}

	if *concurrency < 1 {
		exUsage("concurrency must be at least 1")
	}

	dstURL, err := url.Parse(*urldest)
	if err != nil {
		exUsage(fmt.Sprintf("invalid URL: '%s': %s\n", urldest, err.Error()))
	}

	// Repsonse tracking metadata.
	count := uint64(0)
	size := uint64(0)
	good := uint64(0)
	bad := uint64(0)
	failed := uint64(0)
	// from 0 to 1 minute in nanoseconds
	// FIX: verify that these buckets work correctly for our use case.
	hist := hdrhistogram.New(0, 60000000000, 5)
	received := make(chan *MeasuredResponse)
	timeout := time.After(*interval)

	timeToWait := time.Millisecond * time.Duration(1000 / *qps)

	doTLS := dstURL.Scheme == "https"
	client := newClient(*compress, doTLS, *reuse)

	for i := uint(0); i < *concurrency; i++ {
		ticker := time.NewTicker(timeToWait)<commit_msg>limit connections to concurrency<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"

	"github.com/codahale/hdrhistogram"
)

type MeasuredResponse struct {
	sz      uint64
	code    int
	latency int64
	timeout bool
	err     error
}

func newClient(
	compress bool,
	https bool,
	reuse bool,
	maxConn uint,
) *http.Client {
	tr := http.Transport{
		DisableCompression:  !compress,
		DisableKeepAlives:   !reuse,
		MaxIdleConnsPerHost: int(maxConn),
	}
	if https {
		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	return &http.Client{Transport: &tr}
}

func sendRequest(
	client *http.Client,
	url *url.URL,
	host *string,
	received chan *MeasuredResponse,
) {
	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		fmt.Fprintln(os.Stderr, "\n")
	}
	req.Host = *host

	// FIX: find a way to measure latency with the http client.
	start := time.Now()
	response, err := client.Do(req)

	elapsed := time.Since(start)
	if err != nil {
		received <- &MeasuredResponse{0, 0, 0, false, err}
	} else {
		sz, _ := io.Copy(ioutil.Discard, response.Body)
		response.Body.Close()
		received <- &MeasuredResponse{uint64(sz), response.StatusCode, elapsed.Nanoseconds(), false, nil}
	}
}

func exUsage(msg string) {
	fmt.Fprintf(os.Stderr, "%s\n", msg)
	fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", path.Base(os.Args[0]))
	flag.PrintDefaults()
	os.Exit(64)
}

func main() {
	qps := flag.Int("qps", 1, "QPS to send to backends per request thread")
	concurrency := flag.Uint("concurrency", 1, "Number of request threads")
	host := flag.String("host", "web", "value of Host header to set")
	urldest := flag.String("url", "http://localhost:4140/", "Destination url")
	interval := flag.Duration("interval", 10*time.Second, "reporting interval")
	reuse := flag.Bool("reuse", false, "reuse connections")
	compress := flag.Bool("compress", false, "use compression")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [flags]\n", path.Base(os.Args[0]))
		flag.PrintDefaults()
	}

	flag.Parse()

	if *qps < 1 {
		exUsage("qps must be at least 1")
	}

	if *concurrency < 1 {
		exUsage("concurrency must be at least 1")
	}

	dstURL, err := url.Parse(*urldest)
	if err != nil {
		exUsage(fmt.Sprintf("invalid URL: '%s': %s\n", urldest, err.Error()))
	}

	// Repsonse tracking metadata.
	count := uint64(0)
	size := uint64(0)
	good := uint64(0)
	bad := uint64(0)
	failed := uint64(0)
	// from 0 to 1 minute in nanoseconds
	// FIX: verify that these buckets work correctly for our use case.
	hist := hdrhistogram.New(0, 60000000000, 5)
	received := make(chan *MeasuredResponse)
	timeout := time.After(*interval)

	timeToWait := time.Millisecond * time.Duration(1000 / *qps)

	doTLS := dstURL.Scheme == "https"
	client := newClient(*compress, doTLS, *reuse, *concurrency)

	for i := uint(0); i < *concurrency; i++ {
		ticker := time.NewTicker(timeToWait)
{'fraction_non_alphanumeric': 0.16391489361702127, 'fraction_numerical': 0.020425531914893616, 'mean_word_length': 4.083044982698962, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1573291', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2187, 'n_words': 553}