<commit_before>// Package httpcontrol allows for HTTP transport level control around
// timeouts and retries.
package httpcontrol

import (
	"container/heap"
	"crypto/tls"
	"flag"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/daaku/go.pqueue"
)

// Provides the ability to collect stats for interesting events.
type Stats interface {
	// This is called when a request is retried with the original request, the
	// failed response (if any), try count and error.
	Retry(req *http.Request, res *http.Response, try uint, err error)
}

// Look at http.Transport for the meaning of most of the fields here.
type Transport struct {
	Proxy                 func(*http.Request) (*url.URL, error)
	TLSClientConfig       *tls.Config
	DisableKeepAlives     bool
	DisableCompression    bool
	MaxIdleConnsPerHost   int
	DialTimeout           time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout        time.Duration
	MaxTries              uint // Max retries for known safe failures.
	Stats                 Stats
	Debug                 bool // Verbose logging of request & response
	transport             *http.Transport
	closeMonitor          chan bool
	pqMutex               sync.Mutex
	pq                    pqueue.PriorityQueue
}

var knownFailureSuffixes = []string{
	"connection refused",
	"connection reset by peer.",
	"connection timed out.",
	"no such host.",
	"remote error: handshake failure",
	"unexpected EOF.",
}

func shouldRetryError(err error) bool {
	s := err.Error()<commit_msg>also retry on Temporary net errors<commit_after>// Package httpcontrol allows for HTTP transport level control around
// timeouts and retries.
package httpcontrol

import (
	"container/heap"
	"crypto/tls"
	"flag"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/daaku/go.pqueue"
)

// Provides the ability to collect stats for interesting events.
type Stats interface {
	// This is called when a request is retried with the original request, the
	// failed response (if any), try count and error.
	Retry(req *http.Request, res *http.Response, try uint, err error)
}

// Look at http.Transport for the meaning of most of the fields here.
type Transport struct {
	Proxy                 func(*http.Request) (*url.URL, error)
	TLSClientConfig       *tls.Config
	DisableKeepAlives     bool
	DisableCompression    bool
	MaxIdleConnsPerHost   int
	DialTimeout           time.Duration
	ResponseHeaderTimeout time.Duration
	RequestTimeout        time.Duration
	MaxTries              uint // Max retries for known safe failures.
	Stats                 Stats
	Debug                 bool // Verbose logging of request & response
	transport             *http.Transport
	closeMonitor          chan bool
	pqMutex               sync.Mutex
	pq                    pqueue.PriorityQueue
}

var knownFailureSuffixes = []string{
	"connection refused",
	"connection reset by peer.",
	"connection timed out.",
	"no such host.",
	"remote error: handshake failure",
	"unexpected EOF.",
}

func shouldRetryError(err error) bool {
	if neterr, ok := err.(net.Error); ok {
		if neterr.Temporary() {
			return true
		}
	}

	s := err.Error()
{'fraction_non_alphanumeric': 0.08716173694147263, 'fraction_numerical': 0.0, 'mean_word_length': 2.7444051825677267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2828062', 'n_tokens_mistral': 944, 'n_tokens_neox': 930, 'n_words': 323}