<commit_before>package store

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/djherbis/buffer"
	"github.com/djherbis/nio"
	"github.com/pkg/errors"

	"github.com/oklog/prototype/pkg/fs"
	"github.com/oklog/ulid"
)

// QueryResult contains statistics about, and matching records for, a query.
type QueryResult struct {
	From     string `json:"from"`
	To       string `json:"to"`
	Q        string `json:"q"`
	Regex    bool   `json:"regex"`
	Duration string `json:"duration"`

	NodesQueried    int   `json:"nodes_queried"`
	SegmentsQueried int   `json:"segments_queried"`
	MaxDataSetSize  int64 `json:"max_data_set_size"`
	ErrorCount      int   `json:"error_count,omitempty"`

	Records io.ReadCloser // TODO(pb): audit to ensure closing is valid throughout
}

// EncodeTo encodes the QueryResult to the HTTP response writer.
// It also closes the records ReadCloser.
func (qr *QueryResult) EncodeTo(w http.ResponseWriter) {
	w.Header().Set(httpHeaderFrom, qr.From)
	w.Header().Set(httpHeaderTo, qr.To)
	w.Header().Set(httpHeaderQ, qr.Q)
	w.Header().Set(httpHeaderRegex, fmt.Sprint(qr.Regex))

	w.Header().Set(httpHeaderNodesQueried, strconv.Itoa(qr.NodesQueried))
	w.Header().Set(httpHeaderSegmentsQueried, strconv.Itoa(qr.SegmentsQueried))
	w.Header().Set(httpHeaderMaxDataSetSize, strconv.FormatInt(qr.MaxDataSetSize, 10))
	w.Header().Set(httpHeaderErrorCount, strconv.Itoa(qr.ErrorCount))
	w.Header().Set(httpHeaderDuration, qr.Duration)

	if qr.Records != nil {
		io.Copy(w, qr.Records) // TODO(pb): CopyBuffer
		qr.Records.Close()
	}
}

// DecodeFrom decodes the QueryResult from the HTTP response.<commit_msg>CopyBuffer in the API<commit_after>package store

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/djherbis/buffer"
	"github.com/djherbis/nio"
	"github.com/pkg/errors"

	"github.com/oklog/prototype/pkg/fs"
	"github.com/oklog/ulid"
)

// QueryResult contains statistics about, and matching records for, a query.
type QueryResult struct {
	From     string `json:"from"`
	To       string `json:"to"`
	Q        string `json:"q"`
	Regex    bool   `json:"regex"`
	Duration string `json:"duration"`

	NodesQueried    int   `json:"nodes_queried"`
	SegmentsQueried int   `json:"segments_queried"`
	MaxDataSetSize  int64 `json:"max_data_set_size"`
	ErrorCount      int   `json:"error_count,omitempty"`

	Records io.ReadCloser // TODO(pb): audit to ensure closing is valid throughout
}

// EncodeTo encodes the QueryResult to the HTTP response writer.
// It also closes the records ReadCloser.
func (qr *QueryResult) EncodeTo(w http.ResponseWriter) {
	w.Header().Set(httpHeaderFrom, qr.From)
	w.Header().Set(httpHeaderTo, qr.To)
	w.Header().Set(httpHeaderQ, qr.Q)
	w.Header().Set(httpHeaderRegex, fmt.Sprint(qr.Regex))

	w.Header().Set(httpHeaderNodesQueried, strconv.Itoa(qr.NodesQueried))
	w.Header().Set(httpHeaderSegmentsQueried, strconv.Itoa(qr.SegmentsQueried))
	w.Header().Set(httpHeaderMaxDataSetSize, strconv.FormatInt(qr.MaxDataSetSize, 10))
	w.Header().Set(httpHeaderErrorCount, strconv.Itoa(qr.ErrorCount))
	w.Header().Set(httpHeaderDuration, qr.Duration)

	if qr.Records != nil {
		// CopyBuffer can be useful for complex query pipelines.
		// TODO(pb): validate the 1MB buffer size with profiling
		buf := make([]byte, 1024*1024)
		io.CopyBuffer(w, qr.Records, buf)
		qr.Records.Close()
	}
}

// DecodeFrom decodes the QueryResult from the HTTP response.
{'fraction_non_alphanumeric': 0.13767281105990783, 'fraction_numerical': 0.004896313364055299, 'mean_word_length': 4.592592592592593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6299665', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1249, 'n_words': 262}