<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request, must be < than poll-interval")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL         string
	Description string
	// More fields to come
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	URL        string
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to DB '%s' at %s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request, 10)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, len(list.urls), respCh)

	// The actual fetching of URLs
	for {
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup request object
			req := request{
				URL:  u.URL,
				done: make(chan struct{}),
			}
			// Send it in it's own goroutine
			go send(req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

func send(req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine to be able to abort the request if reached timeout
	go func() {
		resp, err := http.Get(req.URL)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)

	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: "ns",
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.URL}
			fields := map[string]interface{}{
				"value": req.Duration.Nanoseconds(),<commit_msg>Pass preferred precision via flag<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request, must be < than poll-interval")
	precision             = flag.String("precision", "ns", "Precision, valid values are ns and ms")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL         string
	Description string
	// More fields to come
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	URL        string
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}
	if *precision != "ns" && *precision != "ms" {
		fmt.Println("precision must be ms or ns")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to DB '%s' at %s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request, 10)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, len(list.urls), respCh)

	// The actual fetching of URLs
	for {
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup request object
			req := request{
				URL:  u.URL,
				done: make(chan struct{}),
			}
			// Send it in it's own goroutine
			go send(req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

func send(req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine to be able to abort the request if reached timeout
	go func() {
		resp, err := http.Get(req.URL)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)

	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: *precision,
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.URL}
			dur := req.Duration.Nanoseconds()
			if *precision == "ms" {
				dur = dur / 1000
			}
			fields := map[string]interface{}{
				"value": dur,
{'fraction_non_alphanumeric': 0.12864996945632254, 'fraction_numerical': 0.004276114844227245, 'mean_word_length': 3.2392542723977216, 'pattern_counts': {'":': 4, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1097103', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2713, 'n_words': 833}