<commit_before>package publicip

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"github.com/koding/kite"
)

// The sites that PublicIP() uses to get the public IP from.
//
// Note that the site must return *only* the IP characters.
var echoSites = []string{
	// In the future, maybe koding.com/-/echoip first?
	"http://echoip.com",
	"http://api.ipify.org",
	"ifconfig.co",
}

// PublicIP returns an IP that is supposed to be Public.
func PublicIP() (net.IP, error) {
	return publicIP(echoSites[0])
}

// publicIP requests a URL and returns a netIP for the response.
func publicIP(host string) (net.IP, error) {
	resp, err := http.Get(host)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	n := net.ParseIP(string(bytes.TrimSpace(out)))
	if n == nil {
		return nil, fmt.Errorf("cannot parse ip %s", string(out))
	}

	return n, nil
}

// PublicIPRetry fetches the public IP, retrying as many times as requested.
// An *optional* logger is provided, to log retry progress.
func PublicIPRetry(maxRetries int, retryPause time.Duration, log kite.Logger) (net.IP, error) {
	return publicIPRetry(echoSites, maxRetries, retryPause, log)
}

func publicIPRetry(hosts []string, maxRetries int, retryPause time.Duration, log kite.Logger) (net.IP, error) {
	if maxRetries <= 0 {
		return nil, errors.New("PublicIPRetry: maxRetries must be larger than 0")
	}

	var (
		ip  net.IP
		err error
	)

	for i := 0; i < maxRetries; i++ {
		host := hosts[i%len(hosts)]
		ip, err = publicIP(host)

		// If there's no error, we successfully got the IP.
		if err == nil {
			return ip, nil
		}

		if log != nil {
			log.Warning(
				"Retrying fetch of PublicIP due to error. delay:%s, err:%s",
				retryPause, err,
			)
		}

		if retryPause > 0 {
			// Pause before retrying.
			time.Sleep(retryPause)
		}
	}

	return nil, err
}
<commit_msg>klient: Added http client with timeout<commit_after>package publicip

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"time"

	"github.com/koding/kite"
)

// The sites that PublicIP() uses to get the public IP from.
//
// Note that the site must return *only* the IP characters.
var echoSites = []string{
	// In the future, maybe koding.com/-/echoip first?
	"http://echoip.com",
	"http://api.ipify.org",
	"http://ifconfig.co",
}

var defaultClient = &http.Client{
	Timeout: 5 * time.Second,
}

// PublicIP returns an IP that is supposed to be Public.
func PublicIP() (net.IP, error) {
	return publicIP(echoSites[0])
}

// publicIP requests a URL and returns a netIP for the response.
func publicIP(host string) (net.IP, error) {
	resp, err := defaultClient.Get(host)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	out, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	n := net.ParseIP(string(bytes.TrimSpace(out)))
	if n == nil {
		return nil, fmt.Errorf("cannot parse ip %s", string(out))
	}

	return n, nil
}

// PublicIPRetry fetches the public IP, retrying as many times as requested.
// An *optional* logger is provided, to log retry progress.
func PublicIPRetry(maxRetries int, retryPause time.Duration, log kite.Logger) (net.IP, error) {
	return publicIPRetry(echoSites, maxRetries, retryPause, log)
}

func publicIPRetry(hosts []string, maxRetries int, retryPause time.Duration, log kite.Logger) (net.IP, error) {
	if maxRetries <= 0 {
		return nil, errors.New("PublicIPRetry: maxRetries must be larger than 0")
	}

	var (
		ip  net.IP
		err error
	)

	for i := 0; i < maxRetries; i++ {
		host := hosts[i%len(hosts)]
		ip, err = publicIP(host)

		// If there's no error, we successfully got the IP.
		if err == nil {
			return ip, nil
		}

		if log != nil {
			log.Warning(
				"Retrying fetch of PublicIP due to error. delay:%s, err:%s",
				retryPause, err,
			)
		}

		if retryPause > 0 {
			// Pause before retrying.
			time.Sleep(retryPause)
		}
	}

	return nil, err
}

{'fraction_non_alphanumeric': 0.1348229977403967, 'fraction_numerical': 0.0027617373838814963, 'mean_word_length': 3.3540983606557377, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6532202', 'n_tokens_mistral': 1548, 'n_tokens_neox': 1455, 'n_words': 459}