<commit_before>package harvest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/pkg/errors"
)

const HARVEST_DOMAIN = "harvestapp.com"

type API struct {
	client    *http.Client
	BaseURL   string
	SubDomain string
	User      string
	Password  string
}

func NewBasicAuthAPI(subdomain, user, password string) *API {
	a := API{}
	a.client = http.DefaultClient
	a.SubDomain = subdomain
	a.User = user
	a.Password = password
	a.BaseURL = fmt.Sprintf("https://%s.%s", subdomain, HARVEST_DOMAIN)
	return &a
}

func (a *API) Get(path string, args Arguments, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	req, err := http.NewRequest("GET", urlWithParams, nil)
	req.Header.Set("Accept", "application/json")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid GET request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Errorf("HTTP request failure on %s: %s %s", url, string(body), err)
	}

	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(target)
	if err != nil {
		body, _ := ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "JSON decode failed on %s: %s", url, string(body))
	}

	return nil
}

func (a *API) Put(path string, args Arguments, postData interface{}, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	buffer := new(bytes.Buffer)
	if postData != nil {
		json.NewEncoder(buffer).Encode(postData)
	}

	req, err := http.NewRequest("PUT", urlWithParams, buffer)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	req.Header.Set("User-Agent", "github.com/adlio/harvest")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid PUT request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	// Harvest V1 API returns an empty response, with a Location header including the
	// URI of the created object (e.g. /projects/254454)
	redirectDestination := resp.Header.Get("Location")
	if redirectDestination != "" {
		return a.Get(redirectDestination, args, target)
	} else {
		return errors.Errorf("PUT to %s failed to return a Location header. This means we couldn't fetch the new state of the record.", url)
	}

	return nil
}

func (a *API) Post(path string, args Arguments, postData interface{}, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	buffer := new(bytes.Buffer)
	if postData != nil {
		json.NewEncoder(buffer).Encode(postData)
	}

	req, err := http.NewRequest("POST", urlWithParams, buffer)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	req.Header.Set("User-Agent", "github.com/adlio/harvest")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid POST request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	// Harvest V1 API returns an empty response, with a Location header including the
	// URI of the created object (e.g. /projects/254454)
	redirectDestination := resp.Header.Get("Location")
	if redirectDestination != "" {
		return a.Get(redirectDestination, args, target)
	} else {
		return errors.Errorf("POST to %s failed to return a Location header. This means we couldn't fetch the new state of the record.", url)
	}

	return nil
}

func (a *API) Delete(path string, args Arguments) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	req, err := http.NewRequest("DELETE", urlWithParams, nil)
	req.Header.Set("Accept", "application/json")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid DELETE request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	return nil
}
<commit_msg>PUT requests sometimes return data directly instead of a Location: redirect. Updates code to JSON-parse this.<commit_after>package harvest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/pkg/errors"
)

const HARVEST_DOMAIN = "harvestapp.com"

type API struct {
	client    *http.Client
	BaseURL   string
	SubDomain string
	User      string
	Password  string
}

func NewBasicAuthAPI(subdomain, user, password string) *API {
	a := API{}
	a.client = http.DefaultClient
	a.SubDomain = subdomain
	a.User = user
	a.Password = password
	a.BaseURL = fmt.Sprintf("https://%s.%s", subdomain, HARVEST_DOMAIN)
	return &a
}

func (a *API) Get(path string, args Arguments, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	req, err := http.NewRequest("GET", urlWithParams, nil)
	req.Header.Set("Accept", "application/json")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid GET request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Errorf("HTTP request failure on %s: %s %s", url, string(body), err)
	}

	decoder := json.NewDecoder(resp.Body)
	err = decoder.Decode(target)
	if err != nil {
		body, _ := ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "JSON decode failed on %s: %s", url, string(body))
	}

	return nil
}

func (a *API) Put(path string, args Arguments, postData interface{}, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	buffer := new(bytes.Buffer)
	if postData != nil {
		json.NewEncoder(buffer).Encode(postData)
	}

	req, err := http.NewRequest("PUT", urlWithParams, buffer)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	req.Header.Set("User-Agent", "github.com/adlio/harvest")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid PUT request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	// Harvest V1 API returns an empty response, with a Location header including the
	// URI of the created object (e.g. /projects/254454)
	redirectDestination := resp.Header.Get("Location")
	if redirectDestination != "" {
		return a.Get(redirectDestination, args, target)
	} else {
		decoder := json.NewDecoder(resp.Body)
		err = decoder.Decode(target)
		if err != nil {
			body, _ := ioutil.ReadAll(resp.Body)
			return errors.Wrapf(err, "JSON decode failed on %s: %s", url, string(body))
		}
	}

	return nil
}

func (a *API) Post(path string, args Arguments, postData interface{}, target interface{}) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	buffer := new(bytes.Buffer)
	if postData != nil {
		json.NewEncoder(buffer).Encode(postData)
	}

	req, err := http.NewRequest("POST", urlWithParams, buffer)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json; charset=utf-8")
	req.Header.Set("User-Agent", "github.com/adlio/harvest")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid POST request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	// Harvest V1 API returns an empty response, with a Location header including the
	// URI of the created object (e.g. /projects/254454)
	redirectDestination := resp.Header.Get("Location")
	if redirectDestination != "" {
		return a.Get(redirectDestination, args, target)
	} else {
		return errors.Errorf("POST to %s failed to return a Location header. This means we couldn't fetch the new state of the record.", url)
	}

	return nil
}

func (a *API) Delete(path string, args Arguments) error {
	url := fmt.Sprintf("%s%s", a.BaseURL, path)
	urlWithParams := fmt.Sprintf("%s?%s", url, args.ToURLValues().Encode())

	req, err := http.NewRequest("DELETE", urlWithParams, nil)
	req.Header.Set("Accept", "application/json")
	if a.User != "" && a.Password != "" {
		req.SetBasicAuth(a.User, a.Password)
	}
	if err != nil {
		return errors.Wrapf(err, "Invalid DELETE request %s", url)
	}

	resp, err := a.client.Do(req)
	if err != nil {
		return errors.Wrapf(err, "HTTP request failure on %s", url)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		var body []byte
		body, err = ioutil.ReadAll(resp.Body)
		return errors.Wrapf(err, "HTTP request failure on %s: %s %s", url, string(body), err)
	}

	return nil
}

{'fraction_non_alphanumeric': 0.16155781235442093, 'fraction_numerical': 0.006335600484487096, 'mean_word_length': 4.087203791469194, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2660663', 'n_tokens_mistral': 3929, 'n_tokens_neox': 3772, 'n_words': 1034}