<commit_before>// Package netutil implements http request/response, networking, and JSON-related utility functions
package netutil

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
)

const (
	// Constants that may appear in HTTP headers:

	// ContentType is the content type.
	ContentType = "Content-Type"

	// ContentLength is the content length.
	ContentLength = "Content-Length"

	// HTTP Methods:

	// Get is the GET method.
	Get = "GET"

	// Post is the POST method.
	Post = "POST"

	// Patch is the PATCH method.
	Patch = "PATCH"

	// Put is the PUT method.
	Put = "PUT"

	// Delete is the DELETE method.
	Delete = "DELETE"
)

var (
	// ErrTooLong is returned when a BufferResponse would not fit in the buffer budget.
	ErrTooLong = errors.New("response body too long")
)

// TODO(edpin): Rename this to get rid of 'Interface'.

// HTTPClientInterface is a minimal HTTP client interface. An instance of
// http.Client implements this interface.
type HTTPClientInterface interface {
	Do(req *http.Request) (resp *http.Response, err error)
}

// SendJSONError sends an error in a JSON struct with an error message
// composed of a prefix and the actual error message.
func SendJSONError(resp http.ResponseWriter, prefix string, error error) {
	SendJSONErrorString(resp, fmt.Sprintf("%s%v", prefix, error.Error()))
}

// SendJSONErrorString sends a free-form error string in a JSON struct.
func SendJSONErrorString(resp http.ResponseWriter, error string) {
	resp.Header().Set(ContentType, "application/json")
	resp.Write([]byte(fmt.Sprintf(`{"error":%q}`, error)))
}

// SendJSONReply encodes a reply and sends it out on w as a JSON
// object. Make sure the reply type, if it's a struct (which it most
// likely will be) has *public* fields or nothing will be sent (just
// '{}').
func SendJSONReply(resp http.ResponseWriter, reply interface{}) {
	js, err := json.Marshal(reply)
	if err != nil {
		http.Error(resp, err.Error(), http.StatusInternalServerError)
		return
	}
	resp.Header().Set(ContentType, "application/json")
	resp.Write(js)
}

// BufferRequest reads the body of the request 'req' into a buffer of
// size up to maxBufLen bytes. If a buffer cannot be allocated to fit
// the request, nil is returned and an error is sent back to the
// client via 'resp'. The request body is always closed after reading,
// even in case of an error.
func BufferRequest(resp http.ResponseWriter, req *http.Request, maxBufLen int64) []byte {
	var buf []byte
	defer req.Body.Close()
	if req.ContentLength > 0 {
		if req.ContentLength <= maxBufLen {
			buf = make([]byte, req.ContentLength)
		} else {
			// Return an error
			SendJSONErrorString(resp, "invalid request")
			return nil
		}
	} else {
		buf = make([]byte, maxBufLen)
	}
	n, err := req.Body.Read(buf)
	if err != nil && err != io.EOF {
		SendJSONError(resp, "read:", err)
		return nil
	}
	return buf[:n]<commit_msg>netutil: add HTTP headers to disable caching across the board.<commit_after>// Package netutil implements http request/response, networking, and JSON-related utility functions
package netutil

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
)

const (
	// Constants that may appear in HTTP headers:

	// ContentType is the content type.
	ContentType = "Content-Type"

	// ContentLength is the content length.
	ContentLength = "Content-Length"

	// HTTP Methods:

	// Get is the GET method.
	Get = "GET"

	// Post is the POST method.
	Post = "POST"

	// Patch is the PATCH method.
	Patch = "PATCH"

	// Put is the PUT method.
	Put = "PUT"

	// Delete is the DELETE method.
	Delete = "DELETE"
)

var (
	// ErrTooLong is returned when a BufferResponse would not fit in the buffer budget.
	ErrTooLong = errors.New("response body too long")
)

// TODO(edpin): Rename this to get rid of 'Interface'.

// HTTPClientInterface is a minimal HTTP client interface. An instance of
// http.Client implements this interface.
type HTTPClientInterface interface {
	Do(req *http.Request) (resp *http.Response, err error)
}

// SendJSONError sends an error in a JSON struct with an error message
// composed of a prefix and the actual error message.
func SendJSONError(resp http.ResponseWriter, prefix string, error error) {
	SendJSONErrorString(resp, fmt.Sprintf("%s%v", prefix, error.Error()))
}

// SendJSONErrorString sends a free-form error string in a JSON struct.
func SendJSONErrorString(resp http.ResponseWriter, error string) {
	resp.Header().Set(ContentType, "application/json")
	DisableCaching(resp)
	resp.Write([]byte(fmt.Sprintf(`{"error":%q}`, error)))
}

// SendJSONReply encodes a reply and sends it out on w as a JSON
// object. Make sure the reply type, if it's a struct (which it most
// likely will be) has *public* fields or nothing will be sent (just
// '{}').
func SendJSONReply(resp http.ResponseWriter, reply interface{}) {
	js, err := json.Marshal(reply)
	if err != nil {
		http.Error(resp, err.Error(), http.StatusInternalServerError)
		return
	}
	resp.Header().Set(ContentType, "application/json")
	DisableCaching(resp)
	resp.Write(js)
}

// DisableCaching adds HTTP headers to the response that instructs all known browsers and proxies not to cache results.
func DisableCaching(resp http.ResponseWriter) {
	resp.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate")
	resp.Header().Set("Pragma", "no-cache")
	resp.Header().Set("Expires", "0")
}

// BufferRequest reads the body of the request 'req' into a buffer of
// size up to maxBufLen bytes. If a buffer cannot be allocated to fit
// the request, nil is returned and an error is sent back to the
// client via 'resp'. The request body is always closed after reading,
// even in case of an error.
func BufferRequest(resp http.ResponseWriter, req *http.Request, maxBufLen int64) []byte {
	var buf []byte
	defer req.Body.Close()
	if req.ContentLength > 0 {
		if req.ContentLength <= maxBufLen {
			buf = make([]byte, req.ContentLength)
		} else {
			// Return an error
			SendJSONErrorString(resp, "invalid request")
			return nil
		}
	} else {
		buf = make([]byte, maxBufLen)
	}
	n, err := req.Body.Read(buf)
	if err != nil && err != io.EOF {
		SendJSONError(resp, "read:", err)
		return nil
	}
	return buf[:n]
{'fraction_non_alphanumeric': 0.11665858781709484, 'fraction_numerical': 0.0011310389400549362, 'mean_word_length': 3.828393135725429, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4637833', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1881, 'n_words': 721}