libnetwork: improve tests & fix some bugs
<!--
Please make sure you've read and understood our contributing guidelines;
https://github.com/moby/moby/blob/master/CONTRIBUTING.md

** Make sure all your commits include a signature generated with `git commit -s` **

For additional information on our contributing process, read our contributing
guide https://docs.docker.com/opensource/code/

If this is a bug fix, make sure your description includes "fixes #xxxx", or
"closes #xxxx"

Please provide the following information:
-->

**- What I did**
- Improved the isolation between libnetwork tests, so they won't interfere with—or depend on running after—other tests
- Improved and cleaned up the libnetwork tests in other ways; see individual commits
- Fixed bugs in the libnetwork implementation which were uncovered by the improved tests

**- How I did it**
- Changed all the libnetwork tests to run in separate network namespaces, with private `NetworkController` instances
- Eliminated lots of shared global state in tests
- Fixed instances of accidentally-shared state in the libnetwork implementation

**- How to verify it**
CI

**- Description for the changelog**
<!--
Write a short (one line) summary that describes the changes in this
pull request for inclusion in the changelog:
-->


**- A picture of a cute animal (not mandatory but encouraged)**


libnetwork/netutils/utils_linux.go
//go:build linux
// +build linux

// Network utility functions.

package netutils

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/docker/libnetwork/ipamutils"
	"github.com/docker/docker/libnetwork/ns"
	"github.com/docker/docker/libnetwork/resolvconf"
	"github.com/docker/docker/libnetwork/types"
	"github.com/pkg/errors"
	"github.com/vishvananda/netlink"
)

var (
	networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error)
)

// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
func CheckRouteOverlaps(toCheck *net.IPNet) error {
	if networkGetRoutesFct == nil {
		networkGetRoutesFct = ns.NlHandle().RouteList
	}
	networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, network := range networks {
		if network.Dst != nil && network.Scope == netlink.SCOPE_LINK && NetworkOverlaps(toCheck, network.Dst) {
			return ErrNetworkOverlaps
		}
	}
	return nil
}

// GenerateIfaceName returns an interface name using the passed in
// prefix and the length of random bytes. The api ensures that the
// there are is no interface which exists with that name.
func GenerateIfaceName(nlh *netlink.Handle, prefix string, len int) (string, error) {
	linkByName := netlink.LinkByName
	if nlh != nil {
		linkByName = nlh.LinkByName
	}
	for i := 0; i < 3; i++ {
		name, err := GenerateRandomName(prefix, len)
		if err != nil {
			continue
		}
		_, err = linkByName(name)
		if err != nil {
			if strings.Contains(err.Error(), "not found") {
				return name, nil
			}
			return "", err
		}
	}
	return "", types.InternalErrorf("could not generate interface name")
}

// ElectInterfaceAddresses looks for an interface on the OS with the
// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
// If the interface does not exist, it chooses from a predefined
// list the first IPv4 address which does not conflict with other
// interfaces on the system.
func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
	var v4Nets, v6Nets []*net.IPNet

	link, _ := ns.NlHandle().LinkByName(name)
	if link != nil {
		v4addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V4)
		if err != nil {
			return nil, nil, err
		}
		v6addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V6)
		if err != nil {
			return nil, nil, err
		}
		for _, nlAddr := range v4addr {
			v4Nets = append(v4Nets, nlAddr.IPNet)
		}
		for _, nlAddr := range v6addr {
			v6Nets = append(v6Nets, nlAddr.IPNet)
		}
	}

	if link == nil || len(v4Nets) == 0 {
		// Choose from predefined local scope networks
		v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "PredefinedLocalScopeDefaultNetworks List: %+v",
				ipamutils.PredefinedLocalScopeDefaultNetworks)
		}
		v4Nets = append(v4Nets, v4Net)
	}

	return v4Nets, v6Nets, nil
}

// FindAvailableNetwork returns a network from the passed list which does not
// overlap with existing interfaces in the system
func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) {
	// We don't check for an error here, because we don't really care if we
	// can't read /etc/resolv.conf. So instead we skip the append if resolvConf
	// is nil. It either doesn't exist, or we can't read it for some reason.
	var nameservers []string
	if rc, err := resolvconf.Get(); err == nil {
		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
	}
	for _, nw := range list {
		if err := CheckNameserverOverlaps(nameservers, nw); err == nil {
			if err := CheckRouteOverlaps(nw); err == nil {
				return nw, nil
			}
		}
	}
	return nil, fmt.Errorf("no available network")
}

//go:build linux
// +build linux

// Network utility functions.

package netutils

import (
	"fmt"
	"net"
	"strings"

	"github.com/docker/docker/libnetwork/ipamutils"
	"github.com/docker/docker/libnetwork/ns"
	"github.com/docker/docker/libnetwork/resolvconf"
	"github.com/docker/docker/libnetwork/types"
	"github.com/pkg/errors"
	"github.com/vishvananda/netlink"
)

var (
	networkGetRoutesFct func(netlink.Link, int) ([]netlink.Route, error)
)

// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes
func CheckRouteOverlaps(toCheck *net.IPNet) error {
	networkGetRoutesFct := networkGetRoutesFct
	if networkGetRoutesFct == nil {
		networkGetRoutesFct = ns.NlHandle().RouteList
	}
	networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4)
	if err != nil {
		return err
	}
	for _, network := range networks {
		if network.Dst != nil && network.Scope == netlink.SCOPE_LINK && NetworkOverlaps(toCheck, network.Dst) {
			return ErrNetworkOverlaps
		}
	}
	return nil
}

// GenerateIfaceName returns an interface name using the passed in
// prefix and the length of random bytes. The api ensures that the
// there are is no interface which exists with that name.
func GenerateIfaceName(nlh *netlink.Handle, prefix string, len int) (string, error) {
	linkByName := netlink.LinkByName
	if nlh != nil {
		linkByName = nlh.LinkByName
	}
	for i := 0; i < 3; i++ {
		name, err := GenerateRandomName(prefix, len)
		if err != nil {
			continue
		}
		_, err = linkByName(name)
		if err != nil {
			if strings.Contains(err.Error(), "not found") {
				return name, nil
			}
			return "", err
		}
	}
	return "", types.InternalErrorf("could not generate interface name")
}

// ElectInterfaceAddresses looks for an interface on the OS with the
// specified name and returns returns all its IPv4 and IPv6 addresses in CIDR notation.
// If a failure in retrieving the addresses or no IPv4 address is found, an error is returned.
// If the interface does not exist, it chooses from a predefined
// list the first IPv4 address which does not conflict with other
// interfaces on the system.
func ElectInterfaceAddresses(name string) ([]*net.IPNet, []*net.IPNet, error) {
	var v4Nets, v6Nets []*net.IPNet

	link, _ := ns.NlHandle().LinkByName(name)
	if link != nil {
		v4addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V4)
		if err != nil {
			return nil, nil, err
		}
		v6addr, err := ns.NlHandle().AddrList(link, netlink.FAMILY_V6)
		if err != nil {
			return nil, nil, err
		}
		for _, nlAddr := range v4addr {
			v4Nets = append(v4Nets, nlAddr.IPNet)
		}
		for _, nlAddr := range v6addr {
			v6Nets = append(v6Nets, nlAddr.IPNet)
		}
	}

	if link == nil || len(v4Nets) == 0 {
		// Choose from predefined local scope networks
		v4Net, err := FindAvailableNetwork(ipamutils.PredefinedLocalScopeDefaultNetworks)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "PredefinedLocalScopeDefaultNetworks List: %+v",
				ipamutils.PredefinedLocalScopeDefaultNetworks)
		}
		v4Nets = append(v4Nets, v4Net)
	}

	return v4Nets, v6Nets, nil
}

// FindAvailableNetwork returns a network from the passed list which does not
// overlap with existing interfaces in the system
func FindAvailableNetwork(list []*net.IPNet) (*net.IPNet, error) {
	// We don't check for an error here, because we don't really care if we
	// can't read /etc/resolv.conf. So instead we skip the append if resolvConf
	// is nil. It either doesn't exist, or we can't read it for some reason.
	var nameservers []string
	if rc, err := resolvconf.Get(); err == nil {
		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
	}
	for _, nw := range list {
		if err := CheckNameserverOverlaps(nameservers, nw); err == nil {
			if err := CheckRouteOverlaps(nw); err == nil {
				return nw, nil
			}
		}
	}
	return nil, fmt.Errorf("no available network")
}

`ns.NlHandle()` will always return the same value until the next time `libnetwork/ns.Init()` is called. Outside of tests `ns.Init()` is only called through a `sync.Once` so this only affects unit tests.