<commit_before>package chyle

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

// ErrCantReadHTTPResponse is triggered when reading http
// response body failed
type ErrCantReadHTTPResponse struct {
	URL *url.URL
}

// Error output error as string
func (e ErrCantReadHTTPResponse) Error() string {
	return fmt.Sprintf("can't read http response from %s", e.URL)
}

// ErrWrongHTTPStatusCode is triggered when status code is greater or equal to
// 400
type ErrWrongHTTPStatusCode struct {
	request *http.Request
	body    []byte
}

// Error output error as string
func (e ErrWrongHTTPStatusCode) Error() string {
	return fmt.Sprintf("an error occured when contacting remote api through %s, status code %d, body %s", e.request.URL, e.request.Response.StatusCode, e.body)
}

// setHeaders setup headers on request from a map header key -> header value
func setHeaders(request *http.Request, headers map[string]string) {
	for k, v := range headers {
		request.Header.Set(k, v)
	}
}

// sendRequest picks a request and send it with given client and handle all error
// boilerplate and return status code and body as byte slice
func sendRequest(client *http.Client, request *http.Request) (int, []byte, error) {
	rep, err := client.Do(request)

	if err != nil {
		return 0, nil, err
	}

	defer func() {
		err = rep.Body.Close()

		if err != nil {
			log.Fatal(err)
		}
	}()

	b, err := ioutil.ReadAll(rep.Body)

	if err != nil {
		return 0, nil, ErrCantReadHTTPResponse{request.URL}
	}

	if rep.StatusCode >= 400 {
		return rep.StatusCode, b, ErrWrongHTTPStatusCode{request, b}
	}

	return rep.StatusCode, b, nil
}
<commit_msg>ref(chyle/http) : remove useless error add generic http error<commit_after>package chyle

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

// ErrResponse is triggered when status code is greater or equal to 400
type ErrResponse struct {
	request  *http.Request
	response *http.Response
	body     []byte
}

// Error output error as string
func (e ErrResponse) Error() string {
	return fmt.Sprintf("an error occured when contacting remote api through %s, status code %d, body %s", e.request.URL, e.response.StatusCode, e.body)
}

// setHeaders setup headers on request from a map header key -> header value
func setHeaders(request *http.Request, headers map[string]string) {
	for k, v := range headers {
		request.Header.Set(k, v)
	}
}

// sendRequest picks a request and send it with given client
func sendRequest(client *http.Client, request *http.Request) (int, []byte, error) {
	response, err := client.Do(request)

	if err != nil {
		return 0, []byte{}, err
	}

	defer func() {
		err = response.Body.Close()

		if err != nil {
			log.Fatal(err)
		}
	}()

	b, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return response.StatusCode, b, ErrResponse{request, response, b}
	}

	if response.StatusCode >= 400 {
		return response.StatusCode, b, ErrResponse{request, response, b}
	}

	return response.StatusCode, b, nil
}

{'fraction_non_alphanumeric': 0.11605937921727395, 'fraction_numerical': 0.005060728744939271, 'mean_word_length': 3.884678747940692, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2428693', 'n_tokens_mistral': 1007, 'n_tokens_neox': 965, 'n_words': 331}