<commit_before>// Package httputil offers functions to read and download files via HTTP.
package httputil

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"
)

var (
	// DefaultTransport specifies the http.RoundTripper that is used for any network traffic, and may be replaced with a dummy implementation for unit testing.
	DefaultTransport = http.DefaultTransport
	UserAgent = "Bazelisk"
	linkPattern = regexp.MustCompile(`<(.*?)>; rel="(\w+)"`)

	RetryClock = Clock(&realClock{})
	MaxRetries = 4
	// MaxRequestDuration defines the maximum amount of time that a request and its retries may take in total
	MaxRequestDuration = time.Second * 10
	retryHeaders = []string{"Retry-After", "X-RateLimit-Reset", "Rate-Limit-Reset"}
)

type Clock interface {
	Sleep(time.Duration)
	Now() time.Time
}

type realClock struct {}

func (*realClock) Sleep(d time.Duration) {
	time.Sleep(d)
}<commit_msg>Increase httputil.MaxRequestDuration to 30s (#257)<commit_after>// Package httputil offers functions to read and download files via HTTP.
package httputil

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"
)

var (
	// DefaultTransport specifies the http.RoundTripper that is used for any network traffic, and may be replaced with a dummy implementation for unit testing.
	DefaultTransport = http.DefaultTransport
	UserAgent = "Bazelisk"
	linkPattern = regexp.MustCompile(`<(.*?)>; rel="(\w+)"`)

	RetryClock = Clock(&realClock{})
	MaxRetries = 4
	// MaxRequestDuration defines the maximum amount of time that a request and its retries may take in total
	MaxRequestDuration = time.Second * 30
	retryHeaders = []string{"Retry-After", "X-RateLimit-Reset", "Rate-Limit-Reset"}
)

type Clock interface {
	Sleep(time.Duration)
	Now() time.Time
}

type realClock struct {}

func (*realClock) Sleep(d time.Duration) {
	time.Sleep(d)
}
{'fraction_non_alphanumeric': 0.12276214833759591, 'fraction_numerical': 0.005626598465473146, 'mean_word_length': 4.588571428571429, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3352575', 'n_tokens_mistral': 677, 'n_tokens_neox': 650, 'n_words': 200}