libnetwork/osl: Namespace: assorted cleanups 
### libnetwork/osl: implement Namespace.RemoveInterface

Interface.Remove() was directly accessing Namespace "internals", such
as locking/unlocking. Move the code from Interface.Remove() into the
Namespace instead.

### libnetwork/osl: Namespace: make mutex private

Make the mutex internal to the Namespace; locking/unlocking should not
be done externally, and this makes it easier to see where it's used.

### libnetwork/osl: Namespace: programRoute, removeRoute rm path arg

Remove the argument, because it was not used.

### libnetwork/osl: Namespace: make error-handling more idiomatic

Check for non-nil errors (and return early) instead of the reverse.

### libnetwork/osl: Namespace: inline setGateway and setGatewayIPv6

They were not consistently used, and the locations where they were
used were already "setters", so we may as well inline the code.

Also updating Namespace.Restore to keep the lock slightly longer,
instead of locking/unlocking for each property individually, although
we should consider to keep the long for the duration of the whole
function to make it more atomic.


### libnetwork/osl: Namespace.Restore: rename vars for readability

### libnetwork/osl: Namespace.Restore: flatten nested conditions

Flatten some nested "if"-statements, and improve error.

Errors returned by this function are not handled, and only logged, so
make them more informative if debugging is needed.

### libnetwork/osl: Namespace.Restore: conditionally fetch IPs

We're only using the results if the interface doesn't have an address
yet, so skip this step if we don't use it.

### libnetwork/osl: make constructing Interfaces more atomic

It's still not "great", but implement a `newInterface()` constructor
to create a new Interface instance, instead of creating a partial
instance and applying "options" after the fact.




**- A picture of a cute animal (not mandatory but encouraged)**


libnetwork/osl/route_linux.go
package osl

import (
	"fmt"
	"net"

	"github.com/docker/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

// Gateway returns the IPv4 gateway for the sandbox.
func (n *Namespace) Gateway() net.IP {
	n.Lock()
	defer n.Unlock()

	return n.gw
}

// GatewayIPv6 returns the IPv6 gateway for the sandbox.
func (n *Namespace) GatewayIPv6() net.IP {
	n.Lock()
	defer n.Unlock()

	return n.gwv6
}

// StaticRoutes returns additional static routes for the sandbox. Note that
// directly connected routes are stored on the particular interface they
// refer to.
func (n *Namespace) StaticRoutes() []*types.StaticRoute {
	n.Lock()
	defer n.Unlock()

	routes := make([]*types.StaticRoute, len(n.staticRoutes))
	for i, route := range n.staticRoutes {
		r := route.GetCopy()
		routes[i] = r
	}

	return routes
}

func (n *Namespace) setGateway(gw net.IP) {
	n.Lock()
	n.gw = gw
	n.Unlock()
}

func (n *Namespace) setGatewayIPv6(gwv6 net.IP) {
	n.Lock()
	n.gwv6 = gwv6
	n.Unlock()
}

// SetGateway sets the default IPv4 gateway for the sandbox.
func (n *Namespace) SetGateway(gw net.IP) error {
	// Silently return if the gateway is empty
	if len(gw) == 0 {
		return nil
	}

	err := n.programGateway(gw, true)
	if err == nil {
		n.setGateway(gw)
	}

	return err
}

// UnsetGateway the previously set default IPv4 gateway in the sandbox.
func (n *Namespace) UnsetGateway() error {
	gw := n.Gateway()

	// Silently return if the gateway is empty
	if len(gw) == 0 {
		return nil
	}

	err := n.programGateway(gw, false)
	if err == nil {
		n.setGateway(net.IP{})
	}

	return err
}

func (n *Namespace) programGateway(gw net.IP, isAdd bool) error {
	gwRoutes, err := n.nlHandle.RouteGet(gw)
	if err != nil {
		return fmt.Errorf("route for the gateway %s could not be found: %v", gw, err)
	}

	var linkIndex int
	for _, gwRoute := range gwRoutes {
		if gwRoute.Gw == nil {
			linkIndex = gwRoute.LinkIndex
			break
		}
	}

	if linkIndex == 0 {
		return fmt.Errorf("Direct route for the gateway %s could not be found", gw)
	}

	if isAdd {
		return n.nlHandle.RouteAdd(&netlink.Route{
			Scope:     netlink.SCOPE_UNIVERSE,
			LinkIndex: linkIndex,
			Gw:        gw,
		})
	}

	return n.nlHandle.RouteDel(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: linkIndex,
		Gw:        gw,
	})
}

// Program a route in to the namespace routing table.
func (n *Namespace) programRoute(path string, dest *net.IPNet, nh net.IP) error {
	gwRoutes, err := n.nlHandle.RouteGet(nh)
	if err != nil {
		return fmt.Errorf("route for the next hop %s could not be found: %v", nh, err)
	}

	return n.nlHandle.RouteAdd(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: gwRoutes[0].LinkIndex,
		Gw:        nh,
		Dst:       dest,
	})
}

// Delete a route from the namespace routing table.
func (n *Namespace) removeRoute(path string, dest *net.IPNet, nh net.IP) error {
	gwRoutes, err := n.nlHandle.RouteGet(nh)
	if err != nil {
		return fmt.Errorf("route for the next hop could not be found: %v", err)
	}

	return n.nlHandle.RouteDel(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: gwRoutes[0].LinkIndex,
		Gw:        nh,
		Dst:       dest,
	})
}

// SetGatewayIPv6 sets the default IPv6 gateway for the sandbox.
func (n *Namespace) SetGatewayIPv6(gwv6 net.IP) error {
	// Silently return if the gateway is empty
	if len(gwv6) == 0 {
		return nil
	}

	err := n.programGateway(gwv6, true)
	if err == nil {
		n.setGatewayIPv6(gwv6)
	}

	return err
}

// UnsetGatewayIPv6 unsets the previously set default IPv6 gateway in the sandbox.
func (n *Namespace) UnsetGatewayIPv6() error {
	gwv6 := n.GatewayIPv6()

	// Silently return if the gateway is empty
	if len(gwv6) == 0 {
		return nil
	}

	err := n.programGateway(gwv6, false)
	if err == nil {
		n.Lock()
		n.gwv6 = net.IP{}
		n.Unlock()
	}

	return err
}

// AddStaticRoute adds a static route to the sandbox.
func (n *Namespace) AddStaticRoute(r *types.StaticRoute) error {
	err := n.programRoute(n.nsPath(), r.Destination, r.NextHop)
	if err == nil {
		n.Lock()
		n.staticRoutes = append(n.staticRoutes, r)
		n.Unlock()
	}
	return err
}

// RemoveStaticRoute removes a static route from the sandbox.
func (n *Namespace) RemoveStaticRoute(r *types.StaticRoute) error {
	err := n.removeRoute(n.nsPath(), r.Destination, r.NextHop)
	if err == nil {
		n.Lock()
		lastIndex := len(n.staticRoutes) - 1
		for i, v := range n.staticRoutes {
			if v == r {
				// Overwrite the route we're removing with the last element
				n.staticRoutes[i] = n.staticRoutes[lastIndex]
				// Shorten the slice to trim the extra element
				n.staticRoutes = n.staticRoutes[:lastIndex]
				break
			}
		}
		n.Unlock()
	}
	return err
}

package osl

import (
	"fmt"
	"net"

	"github.com/docker/docker/libnetwork/types"
	"github.com/vishvananda/netlink"
)

// Gateway returns the IPv4 gateway for the sandbox.
func (n *Namespace) Gateway() net.IP {
	n.mu.Lock()
	defer n.mu.Unlock()

	return n.gw
}

// GatewayIPv6 returns the IPv6 gateway for the sandbox.
func (n *Namespace) GatewayIPv6() net.IP {
	n.mu.Lock()
	defer n.mu.Unlock()

	return n.gwv6
}

// StaticRoutes returns additional static routes for the sandbox. Note that
// directly connected routes are stored on the particular interface they
// refer to.
func (n *Namespace) StaticRoutes() []*types.StaticRoute {
	n.mu.Lock()
	defer n.mu.Unlock()

	routes := make([]*types.StaticRoute, len(n.staticRoutes))
	for i, route := range n.staticRoutes {
		r := route.GetCopy()
		routes[i] = r
	}

	return routes
}

// SetGateway sets the default IPv4 gateway for the sandbox. It is a no-op
// if the given gateway is empty.
func (n *Namespace) SetGateway(gw net.IP) error {
	if len(gw) == 0 {
		return nil
	}

	if err := n.programGateway(gw, true); err != nil {
		return err
	}
	n.mu.Lock()
	n.gw = gw
	n.mu.Unlock()
	return nil
}

// UnsetGateway the previously set default IPv4 gateway in the sandbox.
// It is a no-op if no gateway was set.
func (n *Namespace) UnsetGateway() error {
	gw := n.Gateway()
	if len(gw) == 0 {
		return nil
	}

	if err := n.programGateway(gw, false); err != nil {
		return err
	}
	n.mu.Lock()
	n.gw = net.IP{}
	n.mu.Unlock()
	return nil
}

func (n *Namespace) programGateway(gw net.IP, isAdd bool) error {
	gwRoutes, err := n.nlHandle.RouteGet(gw)
	if err != nil {
		return fmt.Errorf("route for the gateway %s could not be found: %v", gw, err)
	}

	var linkIndex int
	for _, gwRoute := range gwRoutes {
		if gwRoute.Gw == nil {
			linkIndex = gwRoute.LinkIndex
			break
		}
	}

	if linkIndex == 0 {
		return fmt.Errorf("direct route for the gateway %s could not be found", gw)
	}

	if isAdd {
		return n.nlHandle.RouteAdd(&netlink.Route{
			Scope:     netlink.SCOPE_UNIVERSE,
			LinkIndex: linkIndex,
			Gw:        gw,
		})
	}

	return n.nlHandle.RouteDel(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: linkIndex,
		Gw:        gw,
	})
}

// Program a route in to the namespace routing table.
func (n *Namespace) programRoute(dest *net.IPNet, nh net.IP) error {
	gwRoutes, err := n.nlHandle.RouteGet(nh)
	if err != nil {
		return fmt.Errorf("route for the next hop %s could not be found: %v", nh, err)
	}

	return n.nlHandle.RouteAdd(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: gwRoutes[0].LinkIndex,
		Gw:        nh,
		Dst:       dest,
	})
}

// Delete a route from the namespace routing table.
func (n *Namespace) removeRoute(dest *net.IPNet, nh net.IP) error {
	gwRoutes, err := n.nlHandle.RouteGet(nh)
	if err != nil {
		return fmt.Errorf("route for the next hop could not be found: %v", err)
	}

	return n.nlHandle.RouteDel(&netlink.Route{
		Scope:     netlink.SCOPE_UNIVERSE,
		LinkIndex: gwRoutes[0].LinkIndex,
		Gw:        nh,
		Dst:       dest,
	})
}

// SetGatewayIPv6 sets the default IPv6 gateway for the sandbox. It is a no-op
// if the given gateway is empty.
func (n *Namespace) SetGatewayIPv6(gwv6 net.IP) error {
	if len(gwv6) == 0 {
		return nil
	}

	if err := n.programGateway(gwv6, true); err != nil {
		return err
	}

	n.mu.Lock()
	n.gwv6 = gwv6
	n.mu.Unlock()
	return nil
}

// UnsetGatewayIPv6 unsets the previously set default IPv6 gateway in the sandbox.
// It is a no-op if no gateway was set.
func (n *Namespace) UnsetGatewayIPv6() error {
	gwv6 := n.GatewayIPv6()
	if len(gwv6) == 0 {
		return nil
	}

	if err := n.programGateway(gwv6, false); err != nil {
		return err
	}

	n.mu.Lock()
	n.gwv6 = net.IP{}
	n.mu.Unlock()
	return nil
}

// AddStaticRoute adds a static route to the sandbox.
func (n *Namespace) AddStaticRoute(r *types.StaticRoute) error {
	if err := n.programRoute(r.Destination, r.NextHop); err != nil {
		return err
	}

	n.mu.Lock()
	n.staticRoutes = append(n.staticRoutes, r)
	n.mu.Unlock()
	return nil
}

// RemoveStaticRoute removes a static route from the sandbox.
func (n *Namespace) RemoveStaticRoute(r *types.StaticRoute) error {
	if err := n.removeRoute(r.Destination, r.NextHop); err != nil {
		return err
	}

	n.mu.Lock()
	lastIndex := len(n.staticRoutes) - 1
	for i, v := range n.staticRoutes {
		if v == r {
			// Overwrite the route we're removing with the last element
			n.staticRoutes[i] = n.staticRoutes[lastIndex]
			// Shorten the slice to trim the extra element
			n.staticRoutes = n.staticRoutes[:lastIndex]
			break
		}
	}
	n.mu.Unlock()
	return nil
}

Yeah, locking is a big mess in libnetwork (too many locks in one place, and no locks in places where it _actually_ matters). That's (see my other comment) also a reason I'm un-exporting the mutexes, which also makes it easier to find where locking happens.