<commit_before>package writers

import (
	"bytes"
	"errors"
	"net/http"
	"strings"
)

// NewHttp is Http constructor.
func NewHttp() *Http {
	h := &Http{}
	return h
}

// Http is a writer that simply serialize all readers data to Http.
type Http struct {
	Base
	Url      string
	Method   string
	Headers  string
	Username string
	Password string
}

// headersAsMap parses the headers data as string and returns them as map.
func (h *Http) headersAsMap() map[string]string {
	if h.Headers == "" {
		return nil
	}

	headersInMap := make(map[string]string)

	pairs := strings.Split(h.Headers, ",")

	for _, pairInString := range pairs {
		pair := strings.Split(pairInString, "=")
		if len(pair) >= 2 {
			headersInMap[strings.TrimSpace(pair[0])] = strings.TrimSpace(pair[1])
		}
	}

	return headersInMap
}

// NewHttpRequest builds and returns http.Request struct.
func (h *Http) NewHttpRequest(dataJson []byte) (*http.Request, error) {
	var err error

	if h.Url == "" {
		return nil, errors.New("Url is undefined.")
	}

	if h.Method == "" {
		return nil, errors.New("Method is undefined.")
	}

	req, err := http.NewRequest(h.Method, h.Url, bytes.NewBuffer(dataJson))
	if err != nil {
		return nil, err
	}

	for key, value := range h.headersAsMap() {
		req.Header.Set(key, value)
	}

	if h.Username != "" {
		req.SetBasicAuth(h.Username, h.Password)
	}

	return req, err
}

// Run executes the writer.
func (h *Http) Run() error {
	if h.Data == nil {
		return errors.New("Data field is nil.")
	}

	dataJson, err := h.ToJson()
	if err != nil {
		return err
	}

	req, err := h.NewHttpRequest(dataJson)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return nil
}<commit_msg>Logs http writer output.<commit_after>package writers

import (
	"bytes"
	"errors"
	"github.com/Sirupsen/logrus"
	"io/ioutil"
	"net/http"
	"strings"
)

// NewHttp is Http constructor.
func NewHttp() *Http {
	h := &Http{}
	return h
}

// Http is a writer that simply serialize all readers data to Http.
type Http struct {
	Base
	Url      string
	Method   string
	Headers  string
	Username string
	Password string
}

// headersAsMap parses the headers data as string and returns them as map.
func (h *Http) headersAsMap() map[string]string {
	if h.Headers == "" {
		return nil
	}

	headersInMap := make(map[string]string)

	pairs := strings.Split(h.Headers, ",")

	for _, pairInString := range pairs {
		pair := strings.Split(pairInString, "=")
		if len(pair) >= 2 {
			headersInMap[strings.TrimSpace(pair[0])] = strings.TrimSpace(pair[1])
		}
	}

	return headersInMap
}

// NewHttpRequest builds and returns http.Request struct.
func (h *Http) NewHttpRequest(dataJson []byte) (*http.Request, error) {
	var err error

	if h.Url == "" {
		return nil, errors.New("Url is undefined.")
	}

	if h.Method == "" {
		return nil, errors.New("Method is undefined.")
	}

	req, err := http.NewRequest(h.Method, h.Url, bytes.NewBuffer(dataJson))
	if err != nil {
		return nil, err
	}

	for key, value := range h.headersAsMap() {
		req.Header.Set(key, value)
	}

	if h.Username != "" {
		req.SetBasicAuth(h.Username, h.Password)
	}

	return req, err
}

// Run executes the writer.
func (h *Http) Run() error {
	if h.Data == nil {
		return errors.New("Data field is nil.")
	}

	dataJson, err := h.ToJson()
	if err != nil {
		return err
	}

	req, err := h.NewHttpRequest(dataJson)
	if err != nil {
		return err
	}

	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":      err.Error(),
			"Function":   "func (h *Http) Run() error",
			"req.URL":    req.URL.String(),
			"req.Method": req.Method,
		}).Error("Failed to send HTTP request")

		return err
	}

	if resp.Body != nil {
		respBody, err := ioutil.ReadAll(resp.Body)
		if err == nil {
			logrus.WithFields(logrus.Fields{
				"Function":   "func (h *Http) Run() error",
				"req.URL":    req.URL.String(),
				"req.Method": req.Method,
				"resp.Body":  string(respBody),
			}).Info("Logging resp.Body")
		}

		resp.Body.Close()
	}

	return nil
}
{'fraction_non_alphanumeric': 0.14992721979621543, 'fraction_numerical': 0.001455604075691412, 'mean_word_length': 3.285862785862786, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '715104', 'n_tokens_mistral': 1572, 'n_tokens_neox': 1470, 'n_words': 414}