<commit_before>/*

Client for interacting with ts4 API.

*/

package ts4

import (
	"bytes"
	"crypto/sha1"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
)

// ErrBlobNotFound is returned from a get/stat request
// if the blob does not exist.
var ErrBlobNotFound = errors.New("blob not found")

var defaultServerAddr = "http://localhost:8010"

type BlobStore struct {
	ServerAddr string
	client     *http.Client
}

func New(serverAddr string) *BlobStore {
	if serverAddr == "" {
		serverAddr = defaultServerAddr
	}
	return &BlobStore{
		ServerAddr: serverAddr,
		client:     &http.Client{},
	}
}

type BlobInfo struct {
	Hash string `json:"hash"`
	Time string `json:"time"`
}

// Query send all blobs from start to end over the blobs channel
func (bs *BlobStore) Query(start, end string, blobs chan<- []byte) error {
	//TODO(tsileo)
	// In a for loop decode json from /blobs into []*BlobInfo
	// and send them over the channel, make as many requests as necessary under the hood
}

// Get fetch the given blob.
func (bs *BlobStore) Get(hash string) ([]byte, error) {
	request, err := http.NewRequest("GET", bs.ServerAddr+"/blob/"+hash, nil)
	if err != nil {
		return nil, err
	}
	resp, err := bs.client.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	switch {
	case resp.StatusCode == 200:
		return body, nil
	case resp.StatusCode == 404:
		return nil, ErrBlobNotFound
	default:
		return nil, fmt.Errorf("failed to get blob %v: %v", hash, string(body))
	}
}

// Put upload the given blob
func (bs *BlobStore) Put(blob []byte) error {
	hash := fmt.Sprintf("%x", sha1.Sum(blob))
	body := &bytes.Buffer{}<commit_msg>working query for the client<commit_after>/*

Client for interacting with ts4 API.

*/

package ts4

import (
	"bytes"
	"crypto/sha1"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"mime/multipart"
	"net/http"
)

// ErrBlobNotFound is returned from a get/stat request
// if the blob does not exist.
var ErrBlobNotFound = errors.New("blob not found")

var defaultServerAddr = "http://localhost:8010"

type BlobStore struct {
	ServerAddr string
	client     *http.Client
}

func New(serverAddr string) *BlobStore {
	if serverAddr == "" {
		serverAddr = defaultServerAddr
	}
	return &BlobStore{
		ServerAddr: serverAddr,
		client:     &http.Client{},
	}
}

type BlobInfo struct {
	Hash string `json:"hash"`
	Time string `json:"time"`
}

// Query send all blobs from start to end over the blobs channel,
// start default to 0 and end to time.Now().UTC() if left empty.
func (bs *BlobStore) Query(start, end string, blobs chan<- []byte) error {
	//TODO(tsileo)
	// In a for loop decode json from /blobs into []*BlobInfo
	// and send them over the channel, make as many requests as necessary under the hood
	for {
		res, err := bs.query(start, end)
		if err != nil {
			return err
		}
		if len(res) == 0 {
			break
		}
		for _, blobinfo := range res {
			start = blobinfo.Time
			blob, err := bs.Get(blobinfo.Hash)
			if err != nil {
				return err
			}
			blobs <- blob
		}
	}
	return nil
}

func (bs *BlobStore) query(start, end string) ([]*BlobInfo, error) {
	request, err := http.NewRequest("GET", bs.ServerAddr+"/blobs?start="+start+"&end="+end, nil)
	if err != nil {
		return nil, err
	}
	resp, err := bs.client.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	res := []*BlobInfo{}
	if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
		return nil, err
	}
	switch {
	case resp.StatusCode == 200:
		return res, nil
	case resp.StatusCode == 404:
		return nil, ErrBlobNotFound
	default:
		return nil, fmt.Errorf("failed to query blobs")
	}
}

// Get fetch the given blob.
func (bs *BlobStore) Get(hash string) ([]byte, error) {
	request, err := http.NewRequest("GET", bs.ServerAddr+"/blob/"+hash, nil)
	if err != nil {
		return nil, err
	}
	resp, err := bs.client.Do(request)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	switch {
	case resp.StatusCode == 200:
		return body, nil
	case resp.StatusCode == 404:
		return nil, ErrBlobNotFound
	default:
		return nil, fmt.Errorf("failed to get blob %v: %v", hash, string(body))
	}
}

// Put upload the given blob
func (bs *BlobStore) Put(blob []byte) error {
	hash := fmt.Sprintf("%x", sha1.Sum(blob))
	body := &bytes.Buffer{}
{'fraction_non_alphanumeric': 0.13937753721244925, 'fraction_numerical': 0.008119079837618403, 'mean_word_length': 3.539406345957011, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3574695', 'n_tokens_mistral': 1641, 'n_tokens_neox': 1558, 'n_words': 505}