<commit_before>package util

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"time"
)

type HTTPClientIface interface {
	Get(*http.Request) ([]byte, int, error)
	Post(*http.Request) ([]byte, int, error)
	Do(*http.Request) ([]byte, int, error)
}

type HTTPClient struct {
	Timeout int
	client  *http.Client
}

func NewHTTPClient(timeout int, client *http.Client) *HTTPClient {
	var c = http.DefaultClient
	if client != nil {
		c = client
	}
	hc := &HTTPClient{
		Timeout: timeout,
		client:  c,
	}
	hc.client.Timeout = time.Duration(timeout) * time.Second

	return hc
}

// NewHTTPReqeust makes a http request
func NewHTTPReqeust(method, url string, queries, headers map[string]string, body []byte) (*http.Request, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}

	if queries != nil {
		query := req.URL.Query()
		for key, val := range queries {
			query.Set(key, val)
		}
		req.URL.RawQuery = query.Encode()
	}

	if headers != nil {
		for key, val := range headers {
			req.Header.Set(key, val)
		}
	}

	if body != nil {
		req.Body = ioutil.NopCloser(bytes.NewReader(body))
	}

	return req, nil
}

func (hc *HTTPClient) Do(req *http.Request) ([]byte, int, error) {
	res, err := hc.client.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if res.StatusCode != http.StatusOK {
		return nil, res.StatusCode, fmt.Errorf("resp code is %d", res.StatusCode)
	}

	b, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()

	if err != nil {
		return nil, res.StatusCode, err
	}

	return b, res.StatusCode, nil
}

// Get returns http response body in []byte, timeout in second
func (hc *HTTPClient) Get(req *http.Request) ([]byte, int, error) {
	req.Method = "GET"
	return hc.Do(req)
}

// Post do http post
func (hc *HTTPClient) Post(req *http.Request) ([]byte, int, error) {
	req.Method = "POST"
	return hc.Do(req)
}

// GetReadCloser for downloading file
func (hc *HTTPClient) GetReadCloser(req *http.Request) (io.ReadCloser, string, int, error) {
	var err error
	var res *http.Response

	res, err = hc.client.Do(req)
	if err != nil {<commit_msg>do http parallel<commit_after>package util

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"sync"
	"time"
)

type HTTPClientIface interface {
	Get(*http.Request) ([]byte, int, error)
	Post(*http.Request) ([]byte, int, error)
	Do(*http.Request) ([]byte, int, error)
}

type Response struct {
	Response *http.Response
	Err      error
}

type HTTPClient struct {
	Timeout int
	client  *http.Client
}

func NewHTTPClient(timeout int, client *http.Client) *HTTPClient {
	var c = http.DefaultClient
	if client != nil {
		c = client
	}
	hc := &HTTPClient{
		Timeout: timeout,
		client:  c,
	}
	hc.client.Timeout = time.Duration(timeout) * time.Second

	return hc
}

// NewHTTPReqeust makes a http request
func NewHTTPReqeust(method, url string, queries, headers map[string]string, body []byte) (*http.Request, error) {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}

	if queries != nil {
		query := req.URL.Query()
		for key, val := range queries {
			query.Set(key, val)
		}
		req.URL.RawQuery = query.Encode()
	}

	if headers != nil {
		for key, val := range headers {
			req.Header.Set(key, val)
		}
	}

	if body != nil {
		req.Body = ioutil.NopCloser(bytes.NewReader(body))
	}

	return req, nil
}

func (hc *HTTPClient) Do(req *http.Request) ([]byte, int, error) {
	res, err := hc.client.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if res.StatusCode != http.StatusOK {
		return nil, res.StatusCode, fmt.Errorf("resp code is %d", res.StatusCode)
	}

	b, err := ioutil.ReadAll(res.Body)
	defer res.Body.Close()

	if err != nil {
		return nil, res.StatusCode, err
	}

	return b, res.StatusCode, nil
}

// DoParallel http requests, and get response list. Response could be nil
func (hc *HTTPClient) DoParallel(reqs ...*http.Request) []*Response {
	var reqNum = len(reqs)
	var resps = make([]*Response, reqNum)
	var wg sync.WaitGroup
	for i := 0; i < reqNum; i++ {
		wg.Add(1)
		defer wg.Done()
		var resp Response
		resp.Response, resp.Err = hc.client.Do(reqs[i])
		resps[i] = &resp
	}
	wg.Wait()
	return resps
}

// Get returns http response body in []byte, timeout in second
func (hc *HTTPClient) Get(req *http.Request) ([]byte, int, error) {
	req.Method = "GET"
	return hc.Do(req)
}

// Post do http post
func (hc *HTTPClient) Post(req *http.Request) ([]byte, int, error) {
	req.Method = "POST"
	return hc.Do(req)
}

// GetReadCloser for downloading file
func (hc *HTTPClient) GetReadCloser(req *http.Request) (io.ReadCloser, string, int, error) {
	var err error
	var res *http.Response

	res, err = hc.client.Do(req)
	if err != nil {
{'fraction_non_alphanumeric': 0.14996849401386264, 'fraction_numerical': 0.002100399075824407, 'mean_word_length': 3.632295719844358, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '708211', 'n_tokens_mistral': 1872, 'n_tokens_neox': 1803, 'n_words': 509}