<commit_before>package main

import (
	"bufio"
	"bytes"
	"log"
	"net/http"
	"strings"
)

func loop() {
	client := &http.Client{}
	fqdn := "localhost"

	resp, err := client.Post("http://localhost:1234/poll", "", strings.NewReader(fqdn))
	if err != nil {
		log.Fatal(err)
		return
	}
	defer resp.Body.Close()
	request, _ := http.ReadRequest(bufio.NewReader(resp.Body))
	id := request.Header.Get("id") // Needed so they can be linked.
	log.Printf("Got request for %s", request.URL.String())
	request.RequestURI = ""

	scrapeResp, err := client.Do(request)
	if err != nil {
		log.Fatal(err)
		return
	}
	log.Printf("Scraped %s", request.URL.String())
	scrapeResp.Header.Set("id", id)
	buf := &bytes.Buffer{}
	scrapeResp.Write(buf)
	log.Println(buf.Len())

	_, err = client.Post("http://localhost:1234/push", "", buf)
	if err != nil {
		log.Fatal(err)
		return
	}
	log.Printf("Pushed scrape result for %s", request.URL.String())
}

func main() {
	for {
		loop()
	}<commit_msg>Make client more resilient and performant<commit_after>package main

import (
	"bufio"
	"bytes"
	"log"
	"net/http"
	"strings"
	"time"
)

func doScrape(request *http.Request) {
	client := &http.Client{}
	id := request.Header.Get("id") // Needed so they can be linked.

	scrapeResp, err := client.Do(request)
	if err != nil {
		log.Printf("Failed to scrape %s: %s", request.URL.String(), err)
		return
	}
	log.Printf("Scraped %s", request.URL.String())
	scrapeResp.Header.Set("id", id)
	buf := &bytes.Buffer{}
	scrapeResp.Write(buf)
	log.Println(buf.Len())

	_, err = client.Post("http://localhost:1234/push", "", buf)
	if err != nil {
		log.Printf("Failed to push scrape result for %s: %s", request.URL.String(), err)
		return
	}
	log.Printf("Pushed scrape result for %s", request.URL.String())
}

func loop() {
	client := &http.Client{}
	fqdn := "localhost"

	resp, err := client.Post("http://localhost:1234/poll", "", strings.NewReader(fqdn))
	if err != nil {
		log.Printf("Error polling: %s", err)
		time.Sleep(time.Second) // Don't pound the server.
		return
	}
	defer resp.Body.Close()
	request, _ := http.ReadRequest(bufio.NewReader(resp.Body))
	log.Printf("Got request for %s", request.URL.String())
	request.RequestURI = ""

	go doScrape(request)
}

func main() {
	for {
		loop()
	}
{'fraction_non_alphanumeric': 0.18539823008849557, 'fraction_numerical': 0.007079646017699115, 'mean_word_length': 4.013303769401331, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2693334', 'n_tokens_mistral': 906, 'n_tokens_neox': 851, 'n_words': 188}