<commit_before>package main

import (
	"flag"
	"fmt"
	_ "github.com/cgilmour/maxopen"
	"github.com/cgilmour/uuid"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"runtime"
	"sync"
	"time"
)

var (
	requests       = flag.Uint("requests", 1, "Number of requests to send before completion.")
	connectTimeout = flag.Uint("connect-timeout", 1000, "Number of milliseconds to permit connection attempts before timing out.")
	spinupPeriod   = flag.Uint("spinup-period", 500, "Duration to slowly spin up connections (avoiding initial spike).")
	numWorkers     = flag.Uint("workers", uint(runtime.NumCPU()), "Number of worker goroutines for client connections.")
)

const (
	headerRequestID = "X-Request-ID"
)

func usage() {
	_, cmd := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage: %s [-requests=n] [-connect-timeout=n] url [url...]\n", cmd)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Fprintf(os.Stderr, "No URLs provided\n")
		return
	}

	urls := make([]string, 0, len(flag.Args()))

	for _, param := range flag.Args() {
		if param == "" {
			fmt.Fprintf(os.Stderr, "Empty parameter provided, expected URL\n")
			flag.Usage()
			return
		}
		// Check that we can build requests using the provided URL
		_, err := http.NewRequest("GET", param, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot use parameter '%s' as URL: %s\n", param, err)
			return
		}
		urls = append(urls, param)
	}

	st := &sessionTransport{m: map[string]*session{}}
	uuidList := []string{}

	bc := &byteCounter{}
	client := &http.Client{Transport: st}

	wg := &sync.WaitGroup{}
	ch := make(chan *session, int(*numWorkers))

	spinupInterval := time.Duration(*spinupPeriod) * time.Millisecond / time.Duration(*numWorkers)
	for i := 0; i < int(*numWorkers); i++ {
		wg.Add(1)
		go func(factor int) {
			defer wg.Done()
			delay := time.Duration(factor) * spinupInterval
			for s := range ch {
				if delay != 0 {
					time.Sleep(delay)
					delay = 0
				}
				s.initiated = time.Now()
				s.connections = make([]*timedConnection, 0, len(urls))
				for _, url := range urls {
					req, err := http.NewRequest("GET", url, nil)
					if err != nil {
						fmt.Fprintf(os.Stderr, "Error creating request to '%s': %s\n", url, err)
						continue
					}
					req.Close = true
					req.Header.Set(headerRequestID, s.uuid)
					resp, err := client.Do(req)
					if err != nil {
						fmt.Fprintf(os.Stderr, "Error execuing request to '%s': %s\n", url, err)
						continue
					}
					io.Copy(ioutil.Discard, resp.Body)
					resp.Body.Close()
				}
				s.completed = time.Now()
			}
		}(i)
	}

	ticker := time.NewTicker(1 * time.Second) // 100 * time.Millisecond)
	go func() {
		for range ticker.C {
			rp, tp, rb, tb := bc.Sample()
			fmt.Fprintln(os.Stderr, "Packets sent:", tp, "received", rp, "Bytes sent:", tb, "bytes received:", rb)
		}
	}()

	for i := uint(0); i < *requests; i++ {
		u, err := uuid.New4()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating UUID: %s\n", err)
			continue
		}
		s := newSession(u, bc)
		st.mu.Lock()
		st.m[u] = s
		st.mu.Unlock()
		uuidList = append(uuidList, u)
		ch <- s
	}
	close(ch)
	ticker.Stop()
	wg.Wait()

	// Summary headers
	fmt.Printf("Session\tStartTime")
	for i := range urls {
		fmt.Printf("\tC%[1]dConnSetup\tC%[1]dReqSent\tC%[1]dRespStarted\tC%[1]dRespCompleted", i+1)
	}
	fmt.Printf("\tDuration\n")<commit_msg>Add startup delay option. Fix usage message.<commit_after>package main

import (
	"flag"
	"fmt"
	_ "github.com/cgilmour/maxopen"
	"github.com/cgilmour/uuid"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path"
	"runtime"
	"sync"
	"time"
)

var (
	requests       = flag.Uint("requests", 1, "Number of requests to send before completion.")
	connectTimeout = flag.Uint("connect-timeout", 1000, "Number of milliseconds to permit connection attempts before timing out.")
	spinupPeriod   = flag.Uint("spinup-period", 500, "Duration to slowly spin up connections (avoiding initial spike).")
	numWorkers     = flag.Uint("workers", uint(runtime.NumCPU()), "Number of worker goroutines for client connections.")
	startupDelay   = flag.Uint("startup-delay", uint(0), "Number of milliseconds to delay before starting the requests.")
)

const (
	headerRequestID = "X-Request-ID"
)

func usage() {
	_, cmd := path.Split(os.Args[0])
	fmt.Fprintf(os.Stderr, "Usage: %s [-requests=n] [-connect-timeout=n] [-spinup-period=n] [-workers=n] [-startup-delay=n] url [url...]\n", cmd)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	if len(flag.Args()) == 0 {
		fmt.Fprintf(os.Stderr, "No URLs provided\n")
		return
	}

	urls := make([]string, 0, len(flag.Args()))

	for _, param := range flag.Args() {
		if param == "" {
			fmt.Fprintf(os.Stderr, "Empty parameter provided, expected URL\n")
			flag.Usage()
			return
		}
		// Check that we can build requests using the provided URL
		_, err := http.NewRequest("GET", param, nil)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot use parameter '%s' as URL: %s\n", param, err)
			return
		}
		urls = append(urls, param)
	}

	st := &sessionTransport{m: map[string]*session{}}
	uuidList := []string{}

	bc := &byteCounter{}
	client := &http.Client{Transport: st}

	wg := &sync.WaitGroup{}
	ch := make(chan *session, int(*numWorkers))

	spinupInterval := time.Duration(*spinupPeriod) * time.Millisecond / time.Duration(*numWorkers)
	for i := 0; i < int(*numWorkers); i++ {
		wg.Add(1)
		go func(factor int) {
			defer wg.Done()
			delay := time.Duration(factor) * spinupInterval
			for s := range ch {
				if delay != 0 {
					time.Sleep(delay)
					delay = 0
				}
				s.initiated = time.Now()
				s.connections = make([]*timedConnection, 0, len(urls))
				for _, url := range urls {
					req, err := http.NewRequest("GET", url, nil)
					if err != nil {
						fmt.Fprintf(os.Stderr, "Error creating request to '%s': %s\n", url, err)
						continue
					}
					req.Close = true
					req.Header.Set(headerRequestID, s.uuid)
					resp, err := client.Do(req)
					if err != nil {
						fmt.Fprintf(os.Stderr, "Error execuing request to '%s': %s\n", url, err)
						continue
					}
					io.Copy(ioutil.Discard, resp.Body)
					resp.Body.Close()
				}
				s.completed = time.Now()
			}
		}(i)
	}

	if *startupDelay > 0 {
		fmt.Fprintf(os.Stderr, "Delaying start for %d milliseconds", *startupDelay)
		time.Sleep(time.Duration(*startupDelay) * time.Millisecond)
	}

	ticker := time.NewTicker(1 * time.Second) // 100 * time.Millisecond)
	go func() {
		for range ticker.C {
			rp, tp, rb, tb := bc.Sample()
			fmt.Fprintln(os.Stderr, "Packets sent:", tp, "received", rp, "Bytes sent:", tb, "bytes received:", rb)
		}
	}()

	for i := uint(0); i < *requests; i++ {
		u, err := uuid.New4()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error creating UUID: %s\n", err)
			continue
		}
		s := newSession(u, bc)
		st.mu.Lock()
		st.m[u] = s
		st.mu.Unlock()
		uuidList = append(uuidList, u)
		ch <- s
	}
	close(ch)
	ticker.Stop()
	wg.Wait()

	// Summary headers
	fmt.Printf("Session\tStartTime")
	for i := range urls {
		fmt.Printf("\tC%[1]dConnSetup\tC%[1]dReqSent\tC%[1]dRespStarted\tC%[1]dRespCompleted", i+1)
	}
	fmt.Printf("\tDuration\n")
{'fraction_non_alphanumeric': 0.1733594515181195, 'fraction_numerical': 0.007835455435847209, 'mean_word_length': 3.3008423586040916, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '811213', 'n_tokens_mistral': 3010, 'n_tokens_neox': 2660, 'n_words': 652}