<commit_before>package espsdk

import (
	"io/ioutil"
	"net/http"
	"time"

	log "github.com/Sirupsen/logrus"
)

// A Result contains information relative to a completed request, including
// the time elapsed to fulfill the request and any errors.
type result struct {
	Response *response     `json:"response"`
	Payload  []byte        `json:"-"`
	Duration time.Duration `json:"response_ms"`
	Err      error         `json:"-"`
}

func getResult(c *http.Client, req *http.Request) *result {
	httpCommand := req.Method + " " + string(req.URL.Path)
	start := start(httpCommand)
	resp, err := c.Do(req)
	duration := elapsed(httpCommand, start) / time.Millisecond
	if err != nil {
		log.Fatal(err)
		return buildResult(resp, nil, duration)
	}
	defer resp.Body.Close()

	payload, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
		return buildResult(resp, payload, duration)
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		log.Warnf("HTTP %s", resp.Status)
	}
	return buildResult(resp, payload, duration)
}

func buildResult(resp *http.Response, payload []byte, duration time.Duration) *result {
	return &result{
		&response{
			resp.StatusCode,
			resp.Status,
		},
		payload, duration, nil}
}
<commit_msg>isolate fields for easier parsing<commit_after>package espsdk

import (
	"io/ioutil"
	"net/http"
	"time"

	log "github.com/Sirupsen/logrus"
)

// A Result contains information relative to a completed request, including
// the time elapsed to fulfill the request and any errors.
type result struct {
	Response *response     `json:"response"`
	Payload  []byte        `json:"-"`
	Duration time.Duration `json:"response_ms"`
	Err      error         `json:"-"`
}

func getResult(c *http.Client, req *http.Request) *result {
	httpCommand := req.Method + " " + string(req.URL.Path)
	start := start(httpCommand)
	resp, err := c.Do(req)
	duration := elapsed(httpCommand, start) / time.Millisecond
	if err != nil {
		log.Fatal(err)
		return buildResult(resp, nil, duration)
	}
	defer resp.Body.Close()

	payload, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
		return buildResult(resp, payload, duration)
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		log.WithFields(log.Fields{
			"object":      "response",
			"status_code": resp.StatusCode,
			"status":      resp.Status,
		}).Warn()
	}
	return buildResult(resp, payload, duration)
}

func buildResult(resp *http.Response, payload []byte, duration time.Duration) *result {
	return &result{
		&response{
			resp.StatusCode,
			resp.Status,
		},
		payload, duration, nil}
}

{'fraction_non_alphanumeric': 0.1371939370384765, 'fraction_numerical': 0.004663816556548776, 'mean_word_length': 3.7843866171003717, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1611971', 'n_tokens_mistral': 907, 'n_tokens_neox': 859, 'n_words': 229}