<commit_before>package runner

import (
	"bytes"
	"flag"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	maxPendingReports = 64
	numPublishRetries = 8
	backoffTimeMs     = 1000
)

/*
return_code
status
*/

type ReportPayload struct {
	path     string
	data     map[string]string
	filename string
}

type Reporter struct {
	publishUri      string
	publishChannel  chan ReportPayload
	shutdownChannel chan bool
}

func httpPost(uri string, params map[string]string, file string) (resp *http.Response, err error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			log.Printf("[reporter] Couldn't write field %s", key)
			return nil, err
		}
	}

	if len(file) > 0 {
		handle, err := os.Open(file)
		if err != nil {
			return nil, err
		}

        err = writer.WriteField("name", file)
        if err != nil {
            return nil, err
        }

		fileField, err := writer.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(fileField, handle)
		if err != nil {
			return nil, err
		}
	}

	_ = writer.Close()
	return http.Post(uri, writer.FormDataContentType(), body)
}

func transportSend(r *Reporter) {
	for req := range r.publishChannel {
		path := r.publishUri + req.path
        if req.data == nil {
            req.data = make(map[string]string)
        }
        req.data["date"] = time.Now().UTC().Format(time.RFC3339)
		for tryCnt := 1; tryCnt <= numPublishRetries; tryCnt++ {
			log.Printf("[reporter] POST %s try: %d", path, tryCnt)
			resp, err := httpPost(path, req.data, req.filename)
			if resp != nil && resp.StatusCode / 100 == 2 {
				break
			}

			if resp != nil && resp.StatusCode == http.StatusGone {
				panic("Unknown error occurred with publish endpoint")
			}

			log.Printf("[reporter] POST %s failed, try: %d, resp: %s, err: %s",
				path, tryCnt, resp.Status, err)
			/* We are unable to publish to the endpoint.
			 * Fail fast and let the above layers handle the outage */
			if tryCnt == numPublishRetries {
				panic("Couldn't to connect to publish endpoint")
			}
			log.Printf("[reporter] Sleep for %d ms", backoffTimeMs)<commit_msg>Fix time format<commit_after>package runner

import (
	"bytes"
	"flag"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	maxPendingReports = 64
	numPublishRetries = 8
	backoffTimeMs     = 1000
)

type ReportPayload struct {
	path     string
	data     map[string]string
	filename string
}

type Reporter struct {
	publishUri      string
	publishChannel  chan ReportPayload
	shutdownChannel chan bool
}

func httpPost(uri string, params map[string]string, file string) (resp *http.Response, err error) {
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	for key, val := range params {
		err = writer.WriteField(key, val)
		if err != nil {
			log.Printf("[reporter] Couldn't write field %s", key)
			return nil, err
		}
	}

	if len(file) > 0 {
		handle, err := os.Open(file)
		if err != nil {
			return nil, err
		}

        err = writer.WriteField("name", file)
        if err != nil {
            return nil, err
        }

		fileField, err := writer.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return nil, err
		}

		_, err = io.Copy(fileField, handle)
		if err != nil {
			return nil, err
		}
	}

	_ = writer.Close()
	return http.Post(uri, writer.FormDataContentType(), body)
}

func transportSend(r *Reporter) {
	for req := range r.publishChannel {
		path := r.publishUri + req.path
        if req.data == nil {
            req.data = make(map[string]string)
        }

        req.data["date"] = time.Now().UTC().Format("2006-01-02T15:04:05.0Z")
		for tryCnt := 1; tryCnt <= numPublishRetries; tryCnt++ {
			log.Printf("[reporter] POST %s try: %d", path, tryCnt)
			resp, err := httpPost(path, req.data, req.filename)
			if resp != nil && resp.StatusCode / 100 == 2 {
				break
			}

			if resp != nil && resp.StatusCode == http.StatusGone {
				panic("Unknown error occurred with publish endpoint")
			}

			log.Printf("[reporter] POST %s failed, try: %d, resp: %s, err: %s",
				path, tryCnt, resp.Status, err)
			/* We are unable to publish to the endpoint.
			 * Fail fast and let the above layers handle the outage */
			if tryCnt == numPublishRetries {
				panic("Couldn't to connect to publish endpoint")
			}
			log.Printf("[reporter] Sleep for %d ms", backoffTimeMs)
{'fraction_non_alphanumeric': 0.13297989838745305, 'fraction_numerical': 0.009940357852882704, 'mean_word_length': 2.9894273127753306, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1395948', 'n_tokens_mistral': 1737, 'n_tokens_neox': 1568, 'n_words': 443}