<commit_before>package manet

import (
	"net"

	ma "github.com/multiformats/go-multiaddr"
)

// Private4 and Private6 are well-known private networks
// These are exported to allow overriding for testing
var Private4, Private6 []*net.IPNet
var privateCIDR4 = []string{
	// localhost
	"127.0.0.0/8",
	// private networks
	"10.0.0.0/8",
	"172.16.58.3/10",
	"172.16.0.0/12",
	"192.168.0.0/16",
	// link local
	"169.254.0.0/16",
}
var privateCIDR6 = []string{
	// localhost
	"::1/128",
	// ULA reserved
	"fc00::/7",
	// link local
	"fe80::/10",
}

func init() {
	Private4 = parsePrivateCIDR(privateCIDR4)
	Private6 = parsePrivateCIDR(privateCIDR6)
}

func parsePrivateCIDR(cidrs []string) []*net.IPNet {
	ipnets := make([]*net.IPNet, len(cidrs))
	for i, cidr := range cidrs {
		_, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			panic(err)
		}
		ipnets[i] = ipnet
	}
	return ipnets
}

// IsPublicAddr retruns true if the IP part of the multiaddr is not in a private network
func IsPublicAddr(a ma.Multiaddr) bool {
	ip, err := a.ValueForProtocol(ma.P_IP4)
	if err == nil {
		return !inAddrRange(ip, Private4)
	}

	ip, err = a.ValueForProtocol(ma.P_IP6)
	if err == nil {
		return !inAddrRange(ip, Private6)
	}

	return false
}

// IsPrivateAddr returns true if the IP part of the mutiadr is in a private network
func IsPrivateAddr(a ma.Multiaddr) bool {
	ip, err := a.ValueForProtocol(ma.P_IP4)
	if err == nil {
		return inAddrRange(ip, Private4)
	}

	ip, err = a.ValueForProtocol(ma.P_IP6)
	if err == nil {
		return inAddrRange(ip, Private6)
	}

	return false
}

func inAddrRange(s string, ipnets []*net.IPNet) bool {
	ip := net.ParseIP(s)
	for _, ipnet := range ipnets {
		if ipnet.Contains(ip) {
			return true<commit_msg>test all known unroutable address ranges in IsPublicAddr<commit_after>package manet

import (
	"net"

	ma "github.com/multiformats/go-multiaddr"
)

// Private4 and Private6 are well-known private networks
var Private4, Private6 []*net.IPNet
var privateCIDR4 = []string{
	// localhost
	"127.0.0.0/8",
	// private networks
	"10.0.0.0/8",
	"172.16.58.3/10",
	"172.16.0.0/12",
	"192.168.0.0/16",
	// link local
	"169.254.0.0/16",
}
var privateCIDR6 = []string{
	// localhost
	"::1/128",
	// ULA reserved
	"fc00::/7",
	// link local
	"fe80::/10",
}

// Unroutable4 and Unroutable6 are well known unroutable address ranges
var Unroutable4, Unroutable6 []*net.IPNet
var unroutableCIDR4 = []string{
	"0.0.0.0/8",
	"192.0.0.0/26",
	"192.0.2.0/24",
	"172.16.58.3/24",
	"198.18.0.0/15",
	"198.51.100.0/24",
	"203.0.113.0/24",
	"172.16.58.3/4",
	"240.0.0.0/4",
	"255.255.255.255/32",
}
var unroutableCIDR6 = []string{
	"ff00::/8",
}

func init() {
	Private4 = parseCIDR(privateCIDR4)
	Private6 = parseCIDR(privateCIDR6)
	Unroutable4 = parseCIDR(unroutableCIDR4)
	Unroutable6 = parseCIDR(unroutableCIDR6)
}

func parseCIDR(cidrs []string) []*net.IPNet {
	ipnets := make([]*net.IPNet, len(cidrs))
	for i, cidr := range cidrs {
		_, ipnet, err := net.ParseCIDR(cidr)
		if err != nil {
			panic(err)
		}
		ipnets[i] = ipnet
	}
	return ipnets
}

// IsPublicAddr retruns true if the IP part of the multiaddr is a publically routable address
func IsPublicAddr(a ma.Multiaddr) bool {
	ip, err := a.ValueForProtocol(ma.P_IP4)
	if err == nil {
		return !inAddrRange(ip, Private4) && !inAddrRange(ip, Unroutable4)
	}

	ip, err = a.ValueForProtocol(ma.P_IP6)
	if err == nil {
		return !inAddrRange(ip, Private6) && !inAddrRange(ip, Unroutable6)
	}

	return false
}

// IsPrivateAddr returns true if the IP part of the mutiaddr is in a private network
func IsPrivateAddr(a ma.Multiaddr) bool {
	ip, err := a.ValueForProtocol(ma.P_IP4)
	if err == nil {
		return inAddrRange(ip, Private4)
	}

	ip, err = a.ValueForProtocol(ma.P_IP6)
	if err == nil {
		return inAddrRange(ip, Private6)
	}

	return false
}

func inAddrRange(s string, ipnets []*net.IPNet) bool {
	ip := net.ParseIP(s)
	for _, ipnet := range ipnets {
		if ipnet.Contains(ip) {
			return true
{'fraction_non_alphanumeric': 0.1529709228824273, 'fraction_numerical': 0.06776232616940582, 'mean_word_length': 3.6486486486486487, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 22, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7405037', 'n_tokens_mistral': 1821, 'n_tokens_neox': 1665, 'n_words': 383}