<commit_before>package fetcher

/* fetches */

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/0xb10c/memo/memod/config"
	"github.com/0xb10c/memo/memod/encoder"
	"github.com/0xb10c/memo/memod/logger"
	"github.com/0xb10c/memo/memod/processor"

	"github.com/jasonlvhit/gocron"
)

// SetupMempoolFetcher sets up a periodic mempool fetch job
func SetupMempoolFetcher() {
	mempoolFetchInterval := config.GetInt("mempool.fetchInterval")
	s := gocron.NewScheduler()
	s.Every(uint64(mempoolFetchInterval)).Seconds().Do(getMempool)
	logger.Info.Println("Setup mempool fetcher. First fetch in", mempoolFetchInterval, "seconds")
	<-s.Start()
	defer s.Clear()
}

func getMempool() {

	body, err := fetchMempool()
	if err != nil {
		logger.Error.Printf("Could not fetch mempool: %v", err.Error())
	}

	mempool, err := encoder.DecodeFetchedMempoolBody(body) // decode fetched response body
	if err != nil {
		logger.Error.Printf("Failed to decode response body as JSON: %s", err.Error())
		return
	}

	processor.ProcessMempool(mempool)

}

func fetchMempool() (body []byte, err error) {
	if config.GetString("mempool.fetchInterface") == "REST" {

		body, err := fetchMempoolFromREST()
		if err != nil {
			return nil, fmt.Errorf("Could not fetch mempool from REST: %s", err.Error())
		}

		return body, nil

	} else if config.GetString("mempool.fetchInterface") == "JSON-RPC" {

		body, err := fetchMempoolFromJSONRPC()
		if err != nil {
			return nil, fmt.Errorf("Could not fetch mempool from JSON-RPC: %s", err.Error())
		}

		return body, nil

	} else {
		return nil, errors.New("Unknown interface " + config.GetString("mempool.fetchInterface"))
	}
}

// fetches the current mempool
func fetchMempoolFromREST() ([]byte, error) {
	defer logger.TrackTime(time.Now(), "fetchMempoolFromREST()")

	resp, err := getMempoolContentsREST()
	if err != nil {
		return nil, err
	}

	body, err := readResponseBody(resp)
	if err != nil {
		return nil, err
	}

	return body, nil
}

// make a HTTP GET Request to the Bitcoin Core REST API
func getMempoolContentsREST() (*http.Response, error) {
	timeout := time.Duration(config.GetInt("bitcoind.rest.responseTimeout")) * time.Second
	client := http.Client{
		Timeout: timeout,
	}

	urlPrefix := config.GetString("bitcoind.rest.protocol") +
		"://" + config.GetString("bitcoind.rest.host") +
		":" + config.GetString("bitcoind.rest.port")
	const urlSuffix = "/rest/mempool/contents.json"

	logger.Trace.Println("Fetching mempool contents from ", urlPrefix+urlSuffix)

	resp, err := client.Get(urlPrefix + urlSuffix)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func fetchMempoolFromJSONRPC() ([]byte, error) {
	defer logger.TrackTime(time.Now(), "fetchMempoolFromJSONRPC()")

	rpcURL := config.GetString("bitcoind.jsonrpc.protocol") +
		"://" + config.GetString("bitcoind.jsonrpc.username") +
		":" + config.GetString("bitcoind.jsonrpc.password") +
		"@" + config.GetString("bitcoind.jsonrpc.host") +
		":" + config.GetString("bitcoind.jsonrpc.port")

	logger.Trace.Println("Fetching mempool via JSON-RPC")

	timeout := time.Duration(config.GetInt("bitcoind.jsonrpc.responseTimeout")) * time.Second
	client := http.Client{
		Timeout: timeout,
	}

	bodyReq := strings.NewReader("{\"jsonrpc\": \"1.0\", \"id\":\"memod-via-rpc\", \"method\": \"getrawmempool\", \"params\": [true] }")
	req, err := http.NewRequest("POST", rpcURL, bodyReq)
	if err != nil {
		logger.Error.Println(err)
		return nil, err
	}
	req.Header.Set("Content-Type", "text/plain")

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("JSON-RPC Request failed with status code %d %s", resp.StatusCode, http.StatusText(resp.StatusCode))
	}

	body, err := readResponseBody(resp)
	if err != nil {
		return nil, err
	}

	// The JSON-RPC response is encapsulated in a JSON result object
	// like {"result":{...},"error":null,"id":"memod-via-rpc"}
	// The REST response isn't. We remove this here.
	body = body[10 : len(body)-36]

	return body, nil
}

// read the response body
func readResponseBody(resp *http.Response) ([]byte, error) {
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Error.Println(err.Error())
	}
	defer resp.Body.Close()

	return body, nil
}<commit_msg>Add trace time tracker to readResponseBody<commit_after>package fetcher

/* fetches */

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/0xb10c/memo/memod/config"
	"github.com/0xb10c/memo/memod/encoder"
	"github.com/0xb10c/memo/memod/logger"
	"github.com/0xb10c/memo/memod/processor"

	"github.com/jasonlvhit/gocron"
)

// SetupMempoolFetcher sets up a periodic mempool fetch job
func SetupMempoolFetcher() {
	mempoolFetchInterval := config.GetInt("mempool.fetchInterval")
	s := gocron.NewScheduler()
	s.Every(uint64(mempoolFetchInterval)).Seconds().Do(getMempool)
	logger.Info.Println("Setup mempool fetcher. First fetch in", mempoolFetchInterval, "seconds")
	<-s.Start()
	defer s.Clear()
}

func getMempool() {

	body, err := fetchMempool()
	if err != nil {
		logger.Error.Printf("Could not fetch mempool: %v", err.Error())
	}

	mempool, err := encoder.DecodeFetchedMempoolBody(body) // decode fetched response body
	if err != nil {
		logger.Error.Printf("Failed to decode response body as JSON: %s", err.Error())
		return
	}

	processor.ProcessMempool(mempool)

}

func fetchMempool() (body []byte, err error) {
	if config.GetString("mempool.fetchInterface") == "REST" {

		body, err := fetchMempoolFromREST()
		if err != nil {
			return nil, fmt.Errorf("Could not fetch mempool from REST: %s", err.Error())
		}

		return body, nil

	} else if config.GetString("mempool.fetchInterface") == "JSON-RPC" {

		body, err := fetchMempoolFromJSONRPC()
		if err != nil {
			return nil, fmt.Errorf("Could not fetch mempool from JSON-RPC: %s", err.Error())
		}

		return body, nil

	} else {
		return nil, errors.New("Unknown interface " + config.GetString("mempool.fetchInterface"))
	}
}

// fetches the current mempool
func fetchMempoolFromREST() ([]byte, error) {
	defer logger.TrackTime(time.Now(), "fetchMempoolFromREST()")

	resp, err := getMempoolContentsREST()
	if err != nil {
		return nil, err
	}

	body, err := readResponseBody(resp)
	if err != nil {
		return nil, err
	}

	return body, nil
}

// make a HTTP GET Request to the Bitcoin Core REST API
func getMempoolContentsREST() (*http.Response, error) {
	timeout := time.Duration(config.GetInt("bitcoind.rest.responseTimeout")) * time.Second
	client := http.Client{
		Timeout: timeout,
	}

	urlPrefix := config.GetString("bitcoind.rest.protocol") +
		"://" + config.GetString("bitcoind.rest.host") +
		":" + config.GetString("bitcoind.rest.port")
	const urlSuffix = "/rest/mempool/contents.json"

	logger.Trace.Println("Fetching mempool contents from ", urlPrefix+urlSuffix)

	resp, err := client.Get(urlPrefix + urlSuffix)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

func fetchMempoolFromJSONRPC() ([]byte, error) {
	defer logger.TrackTime(time.Now(), "fetchMempoolFromJSONRPC()")

	rpcURL := config.GetString("bitcoind.jsonrpc.protocol") +
		"://" + config.GetString("bitcoind.jsonrpc.username") +
		":" + config.GetString("bitcoind.jsonrpc.password") +
		"@" + config.GetString("bitcoind.jsonrpc.host") +
		":" + config.GetString("bitcoind.jsonrpc.port")

	logger.Trace.Println("Fetching mempool via JSON-RPC")

	timeout := time.Duration(config.GetInt("bitcoind.jsonrpc.responseTimeout")) * time.Second
	client := http.Client{
		Timeout: timeout,
	}

	bodyReq := strings.NewReader("{\"jsonrpc\": \"1.0\", \"id\":\"memod-via-rpc\", \"method\": \"getrawmempool\", \"params\": [true] }")
	req, err := http.NewRequest("POST", rpcURL, bodyReq)
	if err != nil {
		logger.Error.Println(err)
		return nil, err
	}
	req.Header.Set("Content-Type", "text/plain")

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("JSON-RPC Request failed with status code %d %s", resp.StatusCode, http.StatusText(resp.StatusCode))
	}

	body, err := readResponseBody(resp)
	if err != nil {
		return nil, err
	}

	// The JSON-RPC response is encapsulated in a JSON result object
	// like {"result":{...},"error":null,"id":"memod-via-rpc"}
	// The REST response isn't. We remove this here.
	body = body[10 : len(body)-36]

	return body, nil
}

// read the response body
func readResponseBody(resp *http.Response) ([]byte, error) {
	defer logger.TrackTime(time.Now(), "readResponseBody()")
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.Error.Println(err.Error())
	}
	defer resp.Body.Close()

	return body, nil
}
{'fraction_non_alphanumeric': 0.16089193825042883, 'fraction_numerical': 0.004574042309891366, 'mean_word_length': 4.2941888619854724, 'pattern_counts': {'":': 24, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6578794', 'n_tokens_mistral': 3201, 'n_tokens_neox': 3038, 'n_words': 688}