<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

// WebClient is a client that can retrieve a URL body and POST to a given URL.
type WebClient interface {
	GetURL(url string, maxLength int64) (r io.Reader, statusCode int, err error)
	PostURL(url string, body io.Reader) (statusCode int, err error)
	SetTimeout(d time.Duration)
}

// NewWebClient returns a new WebClient.
func NewWebClient() WebClient {
	return &webClientDefault{client: &http.Client{Timeout: 30 * time.Second}}
}

type webClientDefault struct {
	client *http.Client
}

func (w *webClientDefault) SetTimeout(d time.Duration) {
	w.client.Timeout = d
}

func (w *webClientDefault) GetURL(url string, maxLength int64) (r io.Reader, statusCode int, err error) {
	httpResp, err := w.client.Get(url)
	if err != nil {
		return nil, -1, err
	}

	// We read the entire body into a buffer now to avoid potential problems with timeouts later
	body, err := readBody(httpResp, maxLength)<commit_msg>try to create new web client to fight "re-use of closed conn" issue<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

// WebClient is a client that can retrieve a URL body and POST to a given URL.
type WebClient interface {
	GetURL(url string, maxLength int64) (r io.Reader, statusCode int, err error)
	PostURL(url string, body io.Reader) (statusCode int, err error)
	SetTimeout(d time.Duration)
}

// NewWebClient returns a new WebClient.
func NewWebClient() WebClient {
	return &webClientDefault{client: &http.Client{Timeout: 30 * time.Second}}
}

type webClientDefault struct {
	client *http.Client
}

func (w *webClientDefault) SetTimeout(d time.Duration) {
	w.client.Timeout = d
}

func (w *webClientDefault) GetURL(url string, maxLength int64) (r io.Reader, statusCode int, err error) {
	client := http.Client{Timeout: 30 * time.Second}
	httpResp, err := client.Get(url)
	if err != nil {
		return nil, -1, err
	}

	// We read the entire body into a buffer now to avoid potential problems with timeouts later
	body, err := readBody(httpResp, maxLength)
{'fraction_non_alphanumeric': 0.10857142857142857, 'fraction_numerical': 0.007619047619047619, 'mean_word_length': 4.35969387755102, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4592284', 'n_tokens_mistral': 721, 'n_tokens_neox': 699, 'n_words': 247}