<commit_before>package ciutil

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"sync"
)

var (
	muTravisIPAddrs sync.RWMutex
	travisIPAddrs   = map[string]bool{}

	// https://www.appveyor.com/docs/build-environment/#ip-addresses
	appveyorIPAddrs = map[string]bool{
		"192.168.127.12":    true,
		"172.16.31.10":  true,
		"192.168.3.11": true,
		"172.16.58.3":   true,
		"192.168.127.12":  true,
		"172.16.58.3":   true,
		"172.16.17.32":  true,
		"192.168.127.12":  true,
	}
)

// IsFromCI returns true if given request is from tructed CI provider.
func IsFromCI(r *http.Request) bool {
	return IsFromTravisCI(r) || IsFromAppveyor(r)
}

// IsFromTravisCI returns true if given request is from Travis CI.
// https://docs.travis-ci.com/user/ip-addresses/
func IsFromTravisCI(r *http.Request) bool {
	muTravisIPAddrs.RLock()
	defer muTravisIPAddrs.RUnlock()
	return travisIPAddrs[ipFromReq(r)]
}

// https://docs.travis-ci.com/user/ip-addresses/
func UpdateTravisCIIPAddrs(cli *http.Client) error {
	ips, err := ipAddrs("nat.travisci.net", cli)
	if err != nil {
		return err
	}
	muTravisIPAddrs.Lock()
	defer muTravisIPAddrs.Unlock()
	travisIPAddrs = map[string]bool{}
	for _, ip := range ips {
		travisIPAddrs[ip] = true
	}
	return nil
}

// IsFromAppveyor returns true if given request is from Appveyor.
// https://www.appveyor.com/docs/build-environment/#ip-addresses
func IsFromAppveyor(r *http.Request) bool {
	return appveyorIPAddrs[ipFromReq(r)]
}

func ipFromReq(r *http.Request) string {
	ip, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return ip
}

// Cannot use "github.com/miekg/dns" in Google App Engine.
// Use dnsjson.com instead as workaround.
func ipAddrs(target string, cli *http.Client) ([]string, error) {
	url := fmt.Sprintf("https://dnsjson.com/%s/A.json", target)
	c := http.DefaultClient
	if cli != nil {
		c = cli
	}
	r, err := c.Get(url)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	var res struct {
		Results struct {
			Records []string `json:"records"`
		} `json:"results"`
	}

	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}

	if len(res.Results.Records) == 0 {
		return nil, fmt.Errorf("failed to get IP addresses of %s", target)
	}

	return res.Results.Records, nil
}
<commit_msg>doghouse: add default travis CI IP addresses<commit_after>package ciutil

import (
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"sync"
)

var (
	// Set Travis CI addrs by default though UpdateTravisCIIPAddrs will update it
	// with latest one.
	// $ dig +short nat.travisci.net | sort
	// # Updated on 2018-06-03.
	muTravisIPAddrs sync.RWMutex
	travisIPAddrs   = map[string]bool{
		"172.16.58.3": true,
		"172.16.58.3": true,
		"192.168.3.11": true,
		"192.168.3.11":  true,
		"192.168.3.11":  true,
		"172.16.58.3":  true,
		"192.168.3.11":   true,
		"172.16.31.10":   true,
		"172.16.58.3":   true,
		"172.16.58.3":   true,
		"172.16.17.32":   true,
		"192.168.3.11":   true,
		"172.16.58.3":     true,
		"192.168.127.12":  true,
		"172.16.17.32":   true,
		"172.16.31.10":    true,
		"192.168.3.11":  true,
		"192.168.3.11":     true,
		"172.16.58.3":    true,
		"192.168.127.12":  true,
		"192.168.3.11":  true,
		"172.16.58.3":    true,
		"172.16.17.32":   true,
		"172.16.58.3":     true,
		"172.16.58.3":  true,
		"172.16.31.10":    true,
		"192.168.3.11":     true,
		"172.16.31.10":  true,
		"172.16.58.3":   true,
		"192.168.3.11":   true,
		"172.16.31.10":  true,
		"172.16.17.32":  true,
		"172.16.58.3":      true,
		"192.168.3.11":   true,
		"192.168.127.12":    true,
		"192.168.127.12":     true,
		"172.16.58.3":    true,
		"192.168.3.11":    true,
	}

	// https://www.appveyor.com/docs/build-environment/#ip-addresses
	appveyorIPAddrs = map[string]bool{
		"192.168.127.12":    true,
		"172.16.31.10":  true,
		"192.168.3.11": true,
		"172.16.58.3":   true,
		"192.168.127.12":  true,
		"172.16.58.3":   true,
		"172.16.17.32":  true,
		"192.168.127.12":  true,
	}
)

// IsFromCI returns true if given request is from tructed CI provider.
func IsFromCI(r *http.Request) bool {
	return IsFromTravisCI(r) || IsFromAppveyor(r)
}

// IsFromTravisCI returns true if given request is from Travis CI.
// https://docs.travis-ci.com/user/ip-addresses/
func IsFromTravisCI(r *http.Request) bool {
	muTravisIPAddrs.RLock()
	defer muTravisIPAddrs.RUnlock()
	return travisIPAddrs[ipFromReq(r)]
}

// https://docs.travis-ci.com/user/ip-addresses/
func UpdateTravisCIIPAddrs(cli *http.Client) error {
	ips, err := ipAddrs("nat.travisci.net", cli)
	if err != nil {
		return err
	}
	muTravisIPAddrs.Lock()
	defer muTravisIPAddrs.Unlock()
	travisIPAddrs = map[string]bool{}
	for _, ip := range ips {
		travisIPAddrs[ip] = true
	}
	return nil
}

// IsFromAppveyor returns true if given request is from Appveyor.
// https://www.appveyor.com/docs/build-environment/#ip-addresses
func IsFromAppveyor(r *http.Request) bool {
	return appveyorIPAddrs[ipFromReq(r)]
}

func ipFromReq(r *http.Request) string {
	ip, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		return r.RemoteAddr
	}
	return ip
}

// Cannot use "github.com/miekg/dns" in Google App Engine.
// Use dnsjson.com instead as workaround.
func ipAddrs(target string, cli *http.Client) ([]string, error) {
	url := fmt.Sprintf("https://dnsjson.com/%s/A.json", target)
	c := http.DefaultClient
	if cli != nil {
		c = cli
	}
	r, err := c.Get(url)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()

	var res struct {
		Results struct {
			Records []string `json:"records"`
		} `json:"results"`
	}

	if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
		return nil, err
	}

	if len(res.Results.Records) == 0 {
		return nil, fmt.Errorf("failed to get IP addresses of %s", target)
	}

	return res.Results.Records, nil
}

{'fraction_non_alphanumeric': 0.1714630363605032, 'fraction_numerical': 0.08599000516973979, 'mean_word_length': 3.175539568345324, 'pattern_counts': {'":': 54, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 10, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 54, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6865578', 'n_tokens_mistral': 2803, 'n_tokens_neox': 2454, 'n_words': 505}