libnet/d/bridge: Don't set container's gateway when network is internal
Related to:

- CI failures seen on https://github.com/moby/moby/pull/46531

**- What I did**

So far, internal networks were only isolated from the host by iptables DROP rules. As a consequence, outbound connections from containers would timeout instead of being "rejected" through an immediate ICMP dest/port unreachable, a TCP RST or a failing `connect` syscall.

This was visible when internal containers were trying to resolve a domain that don't match any container on the same network (be it a truly "external" domain, or a container that don't exist/is dead). In that case, the embedded resolver would try to forward DNS queries for the different values of resolv.conf `search` option, making DNS resolution slow to return an error, and the slowness being exacerbated by some libc implementations.

This change makes `connect` syscall to return ENETUNREACH, and thus solves the broader issue of failing fast when external connections are attempted.

**- How to verify it**

CI is green.

**- Description for the changelog**

* Make `connect` syscall fail fast when a container is only attached to an internal network.

**- A picture of a cute animal (not mandatory but encouraged)**

![](https://cdn.naturettl.com/wp-content/uploads/2014/10/08201805/john-speirs_i-guess-summers-over-800x504.jpg)
libnetwork/drivers/bridge/setup_ipv4_linux.go
package bridge

import (
	"context"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/log"
	"github.com/docker/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

func selectIPv4Address(addresses []netlink.Addr, selector *net.IPNet) (netlink.Addr, error) {
	if len(addresses) == 0 {
		return netlink.Addr{}, errors.New("unable to select an address as the address pool is empty")
	}
	if selector != nil {
		for _, addr := range addresses {
			if selector.Contains(addr.IP) {
				return addr, nil
			}
		}
	}
	return addresses[0], nil
}

func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
	if !config.InhibitIPv4 {
		addrv4List, _, err := i.addresses()
		if err != nil {
			return fmt.Errorf("failed to retrieve bridge interface addresses: %v", err)
		}

		addrv4, _ := selectIPv4Address(addrv4List, config.AddressIPv4)

		if !types.CompareIPNet(addrv4.IPNet, config.AddressIPv4) {
			if addrv4.IPNet != nil {
				if err := i.nlh.AddrDel(i.Link, &addrv4); err != nil {
					return fmt.Errorf("failed to remove current ip address from bridge: %v", err)
				}
			}
			log.G(context.TODO()).Debugf("Assigning address to bridge interface %s: %s", config.BridgeName, config.AddressIPv4)
			if err := i.nlh.AddrAdd(i.Link, &netlink.Addr{IPNet: config.AddressIPv4}); err != nil {
				return &IPv4AddrAddError{IP: config.AddressIPv4, Err: err}
			}
		}
	}

	// Store bridge network and default gateway
	i.bridgeIPv4 = config.AddressIPv4
	i.gatewayIPv4 = config.AddressIPv4.IP

	return nil
}

func setupGatewayIPv4(config *networkConfiguration, i *bridgeInterface) error {
	if !i.bridgeIPv4.Contains(config.DefaultGatewayIPv4) {
		return &ErrInvalidGateway{}
	}

	// Store requested default gateway
	i.gatewayIPv4 = config.DefaultGatewayIPv4

	return nil
}

func setupLoopbackAddressesRouting(config *networkConfiguration, i *bridgeInterface) error {
	sysPath := filepath.Join("/proc/sys/net/ipv4/conf", config.BridgeName, "route_localnet")
	ipv4LoRoutingData, err := os.ReadFile(sysPath)
	if err != nil {
		return fmt.Errorf("Cannot read IPv4 local routing setup: %v", err)
	}
	// Enable loopback addresses routing only if it isn't already enabled
	if ipv4LoRoutingData[0] != '1' {
		if err := os.WriteFile(sysPath, []byte{'1', '\n'}, 0o644); err != nil {
			return fmt.Errorf("Unable to enable local routing for hairpin mode: %v", err)
		}
	}
	return nil
}

package bridge

import (
	"context"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"

	"github.com/containerd/containerd/log"
	"github.com/docker/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

func selectIPv4Address(addresses []netlink.Addr, selector *net.IPNet) (netlink.Addr, error) {
	if len(addresses) == 0 {
		return netlink.Addr{}, errors.New("unable to select an address as the address pool is empty")
	}
	if selector != nil {
		for _, addr := range addresses {
			if selector.Contains(addr.IP) {
				return addr, nil
			}
		}
	}
	return addresses[0], nil
}

func setupBridgeIPv4(config *networkConfiguration, i *bridgeInterface) error {
	// TODO(aker): the bridge driver panics if its bridgeIPv4 field isn't set. Once bridge subnet and bridge IP address
	//             are decoupled, we should assign it only when it's really needed.
	i.bridgeIPv4 = config.AddressIPv4

	if config.Internal {
		return nil
	}

	if !config.InhibitIPv4 {
		addrv4List, _, err := i.addresses()
		if err != nil {
			return fmt.Errorf("failed to retrieve bridge interface addresses: %v", err)
		}

		addrv4, _ := selectIPv4Address(addrv4List, config.AddressIPv4)

		if !types.CompareIPNet(addrv4.IPNet, config.AddressIPv4) {
			if addrv4.IPNet != nil {
				if err := i.nlh.AddrDel(i.Link, &addrv4); err != nil {
					return fmt.Errorf("failed to remove current ip address from bridge: %v", err)
				}
			}
			log.G(context.TODO()).Debugf("Assigning address to bridge interface %s: %s", config.BridgeName, config.AddressIPv4)
			if err := i.nlh.AddrAdd(i.Link, &netlink.Addr{IPNet: config.AddressIPv4}); err != nil {
				return &IPv4AddrAddError{IP: config.AddressIPv4, Err: err}
			}
		}
	}

	// Store the default gateway
	i.gatewayIPv4 = config.AddressIPv4.IP

	return nil
}

func setupGatewayIPv4(config *networkConfiguration, i *bridgeInterface) error {
	if !i.bridgeIPv4.Contains(config.DefaultGatewayIPv4) {
		return &ErrInvalidGateway{}
	}
	if config.Internal {
		return types.InvalidParameterErrorf("no gateway can be set on an internal bridge network")
	}

	// Store requested default gateway
	i.gatewayIPv4 = config.DefaultGatewayIPv4

	return nil
}

func setupLoopbackAddressesRouting(config *networkConfiguration, i *bridgeInterface) error {
	sysPath := filepath.Join("/proc/sys/net/ipv4/conf", config.BridgeName, "route_localnet")
	ipv4LoRoutingData, err := os.ReadFile(sysPath)
	if err != nil {
		return fmt.Errorf("Cannot read IPv4 local routing setup: %v", err)
	}
	// Enable loopback addresses routing only if it isn't already enabled
	if ipv4LoRoutingData[0] != '1' {
		if err := os.WriteFile(sysPath, []byte{'1', '\n'}, 0o644); err != nil {
			return fmt.Errorf("Unable to enable local routing for hairpin mode: %v", err)
		}
	}
	return nil
}

`config.AddressIPv4` and `i.bridgeIPv4` are of type `*net.IPNet`. That is the data type for an IP subnet. When not in canonical form (i.e. the host-part bits are not all zero) it can also pull double duty to also encode a particular address within the subnet. (The same is also true for `netip.Prefix`.) Since the bridge's external-facing IP address has to be within the bridge subnet anyway, I don't see the need to encode just the subnet in a separate field. It would just be `bridgeIPv4` in canonical form.