func (d *windowsDataplane) CleanUpNamespace(args *skel.CmdArgs) error {
	d.logger.Infof("Cleaning up endpoint")
	n, _, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}
	epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
	d.logger.Infof("Attempting to delete HNS endpoint name : %s for container", epName)
	if cri.IsDockershimV1(args.Netns) {
		err = hns.DeprovisionEndpoint(epName, args.Netns, args.ContainerID)
		if err != nil && strings.Contains(err.Error(), "not found") {
			d.logger.WithError(err).Warn("Endpoint not found during delete, assuming it's already been cleaned up")
			return nil
		}
	} else {
		err = hns.RemoveHcnEndpoint(epName)
		if err != nil {
			d.logger.WithError(err).Warn("Failed to find or delete endpoint, assuming it's already been cleaned up")
			return nil
		}
	}
	return err
}
CleanUpNamespace deletes the devices in the network namespace.
package windows
import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"
	"github.com/Microsoft/hcsshim"
	"github.com/Microsoft/hcsshim/hcn"
	"github.com/buger/jsonparser"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/containernetworking/plugins/pkg/hns"
	"github.com/juju/clock"
	"github.com/juju/errors"
	"github.com/juju/mutex"
	"github.com/projectcalico/cni-plugin/internal/pkg/utils/cri"
	"github.com/projectcalico/cni-plugin/internal/pkg/utils/winpol"
	"github.com/projectcalico/cni-plugin/pkg/types"
	api "github.com/projectcalico/libcalico-go/lib/apis/v3"
	calicoclient "github.com/projectcalico/libcalico-go/lib/clientv3"
	"github.com/projectcalico/libcalico-go/lib/options"
	"github.com/rakelkar/gonetsh/netsh"
	"github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/util/wait"
	utilexec "k8s.io/utils/exec"
)
const (
	DefaultVNI = 4096
)
type windowsDataplane struct {
	conf   types.NetConf
	logger *logrus.Entry
}
func NewWindowsDataplane(conf types.NetConf, logger *logrus.Entry) *windowsDataplane {
	return &windowsDataplane{
		conf:   conf,
		logger: logger,
	}
}
func loadNetConf(bytes []byte) (*hns.NetConf, string, error) {
	n := &hns.NetConf{}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, "", fmt.Errorf("failed to load netconf: %v", err)
	}
	return n, n.CNIVersion, nil
}
func acquireLock() (mutex.Releaser, error) {
	spec := mutex.Spec{
		Name:    "TigeraCalicoCNINetworkMutex",
		Clock:   clock.WallClock,
		Delay:   50 * time.Millisecond,
		Timeout: 90000 * time.Millisecond,
	}
	logrus.Infof("Trying to acquire lock %v", spec)
	m, err := mutex.Acquire(spec)
	if err != nil {
		logrus.Errorf("Error acquiring lock %v", spec)
		return nil, err
	}
	logrus.Infof("Acquired lock %v", spec)
	return m, nil
}
func SetupL2bridgeNetwork(networkName string, subNet *net.IPNet, logger *logrus.Entry) (*hcsshim.HNSNetwork, error) {
	hnsNetwork, err := EnsureNetworkExists(networkName, subNet, logger)
	if err != nil {
		logger.Errorf("Unable to create hns network %s", networkName)
		return nil, err
	}
	epName := networkName + "_ep"
	hnsEndpoint, err := CreateAndAttachHostEP(epName, hnsNetwork, subNet, logger)
	if err != nil {
		logger.Errorf("Unable to create host hns endpoint %s", epName)
		return nil, err
	}
	hnsNetwork, err = chkMgmtIPandEnableForwarding(networkName, hnsEndpoint, logger)
	if err != nil {
		logger.Errorf("Failed to enable forwarding : %v", err)
		return nil, err
	}
	return hnsNetwork, err
}
func SetupVxlanNetwork(networkName string, subNet *net.IPNet, vni uint64, logger *logrus.Entry) (*hcsshim.HNSNetwork, error) {
	hnsNetwork, err := ensureVxlanNetworkExists(networkName, subNet, vni, logger)
	if err != nil {
		logger.Errorf("Unable to create hns network %s", networkName)
		return nil, err
	}
	epName := networkName + "_ep"
	_, err = createAndAttachVxlanHostEP(epName, hnsNetwork, subNet, logger)
	if err != nil {
		logger.Errorf("Unable to create host hns endpoint %s", epName)
		return nil, err
	}
	return hnsNetwork, err
}
func (d *windowsDataplane) DoNetworking(
	ctx context.Context,
	calicoClient calicoclient.Interface,
	args *skel.CmdArgs,
	result *current.Result,
	desiredVethName string,
	routes []*net.IPNet,
	endpoint *api.WorkloadEndpoint,
	annotations map[string]string,
) (hostVethName, contVethMAC string, err error) {
	hostVethName = desiredVethName
	if len(routes) > 0 {
		logrus.WithField("routes", routes).Debug("Ignoring in-container routes; not supported on Windows.")
	}
	podIP, subNet, _ := net.ParseCIDR(result.IPs[0].Address.String())
	n, _, err := loadNetConf(args.StdinData)
	if err != nil {
		d.logger.Errorf("Error loading args")
		return "", "", err
	}
	if len(d.conf.RuntimeConfig.DNS.Nameservers) >= 1 {
		result.DNS.Nameservers = d.conf.RuntimeConfig.DNS.Nameservers
		result.DNS.Domain = d.conf.RuntimeConfig.DNS.Domain
		result.DNS.Search = d.conf.RuntimeConfig.DNS.Search
		result.DNS.Options = d.conf.RuntimeConfig.DNS.Options
	} else {
		result.DNS = n.DNS
	}
	allIPAMPools, natOutgoing, err := lookupIPAMPools(ctx, podIP, calicoClient)
	if err != nil {
		d.logger.WithError(err).Error("Failed to look up IPAM pools")
		return "", "", err
	}
	m, err := acquireLock()
	if err != nil {
		d.logger.Errorf("Unable to acquire lock")
		return "", "", err
	}
	defer m.Release()
	var networkName string
	if d.conf.WindowsUseSingleNetwork {
		d.logger.WithField("name", d.conf.Name).Info(
			"Overriding network name, only a single IPAM block will be supported on this host")
		networkName = d.conf.Name
	} else {
		networkName = CreateNetworkName(n.Name, subNet)
	}
	var hnsNetwork *hcsshim.HNSNetwork
	if d.conf.Mode == "vxlan" {
		hnsNetwork, err = SetupVxlanNetwork(networkName, subNet, d.conf.VXLANVNI, d.logger)
	} else {
		hnsNetwork, err = SetupL2bridgeNetwork(networkName, subNet, d.logger)
	}
	if err != nil {
		d.logger.Errorf("Unable to create hns network %s", networkName)
		return "", "", err
	}
	hnsEndpointCont, hcsEndpoint, err := d.createAndAttachContainerEP(args, hnsNetwork, subNet, allIPAMPools, natOutgoing, result, n)
	if err != nil {
		epName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
		d.logger.Errorf("Unable to create container hns endpoint %s", epName)
		return "", "", err
	}
	if d.conf.WindowsDisableDefaultDenyAllPolicy == false {
		var err error
		if cri.IsDockershimV1(args.Netns) {
			defaultDenyAllACL := &hcsshim.ACLPolicy{
				Id:        "CNIDefaultDenyAllPolicy",
				Type:      hcsshim.ACL,
				RuleType:  hcsshim.Switch,
				Action:    hcsshim.Block,
				Direction: hcsshim.In,
				Protocol:  256,
				Priority:  65500,
			}
			err = hnsEndpointCont.ApplyACLPolicy(defaultDenyAllACL)
		} else {
			aclPolicySettings := hcn.AclPolicySetting{
				RuleType:  hcn.RuleTypeSwitch,
				Action:    hcn.ActionTypeBlock,
				Direction: hcn.DirectionTypeIn,
				Protocols: "256",
				Priority:  65500,
			}
			policyJSON, err := json.Marshal(aclPolicySettings)
			if err != nil {
				d.logger.WithError(err).Error("Failed to marshal ACL policy")
				return "", "", err
			}
			defaultDenyAllACL := hcn.PolicyEndpointRequest{
				Policies: []hcn.EndpointPolicy{
					{
						Type:     hcn.ACL,
						Settings: json.RawMessage(policyJSON),
					},
				},
			}
			err = hcsEndpoint.ApplyPolicy(hcn.RequestTypeUpdate, defaultDenyAllACL)
		}
		if err != nil {
			d.logger.Errorf("Error applying ACL policy DenyAll")
			return "", "", err
		}
	}
	if cri.IsDockershimV1(args.Netns) {
		contVethMAC = hnsEndpointCont.MacAddress
	} else {
		contVethMAC = hcsEndpoint.MacAddress
	}
	return hostVethName, contVethMAC, err
}
func lookupIPAMPools(
	ctx context.Context, podIP net.IP, calicoClient calicoclient.Interface,
) (
	cidrs []*net.IPNet,
	natOutgoing bool,
	err error,
) {
	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	pools, err := calicoClient.IPPools().List(ctx, options.ListOptions{})
	if err != nil {
		return
	}
	natOutgoing = true
	for _, p := range pools.Items {
		_, ipNet, err := net.ParseCIDR(p.Spec.CIDR)
		if err != nil {
			logrus.WithError(err).WithField("rawCIDR", p.Spec.CIDR).Warn("IP pool contained bad CIDR, ignoring")
			continue
		}
		cidrs = append(cidrs, ipNet)
		if ipNet.Contains(podIP) {
			logrus.WithField("pool", p.Spec).Debug("Found pool containing pod IP")
			natOutgoing = p.Spec.NATOutgoing
		}
	}
	return
}
func ensureVxlanNetworkExists(networkName string, subNet *net.IPNet, vni uint64, logger *logrus.Entry) (*hcsshim.HNSNetwork, error) {
	var err error
	createNetwork := true
	expectedAddressPrefix := subNet.String()
	expectedGW := getNthIP(subNet, 1)
	var expectedVNI uint64
	expectedNetwork := &hcsshim.HNSNetwork{
		Name:    networkName,
		Type:    "Overlay",
		Subnets: make([]hcsshim.Subnet, 0, 1),
	}
	if vni == 0 {
		expectedVNI = DefaultVNI
	} else if vni < DefaultVNI {
		return nil, fmt.Errorf("Windows does not support VXLANVNI < 4096")
	} else {
		expectedVNI = vni
	}
	existingNetwork, _ := hcsshim.GetHNSNetworkByName(networkName)
	if existingNetwork != nil {
		if existingNetwork.Type == expectedNetwork.Type {
			for _, subnet := range existingNetwork.Subnets {
				if subnet.AddressPrefix == expectedAddressPrefix && subnet.GatewayAddress == expectedGW.String() {
					createNetwork = false
					logger.Infof("Found existing HNS network [%+v]", existingNetwork)
					break
				}
			}
		}
	}
	if createNetwork {
		if existingNetwork != nil {
			if _, err := existingNetwork.Delete(); err != nil {
				logger.Errorf("Unable to delete existing network [%v], error: %v", existingNetwork.Name, err)
				return nil, err
			}
			logger.Infof("Deleted stale HNS network [%v]")
		}
		expectedNetwork.Subnets = append(expectedNetwork.Subnets, hcsshim.Subnet{
			AddressPrefix:  expectedAddressPrefix,
			GatewayAddress: expectedGW.String(),
			Policies: []json.RawMessage{
				[]byte(fmt.Sprintf(`{"Type":"VSID","VSID":%d}`, expectedVNI)),
			},
		})
		jsonRequest, err := json.Marshal(expectedNetwork)
		if err != nil {
			return nil, errors.Annotatef(err, "failed to marshal %+v", expectedNetwork)
		}
		logger.Infof("Attempting to create HNSNetwork %s", string(jsonRequest))
		newNetwork, err := hcsshim.HNSNetworkRequest("POST", "", string(jsonRequest))
		if err != nil {
			return nil, errors.Annotatef(err, "failed to create HNSNetwork %s", networkName)
		}
		var waitErr, lastErr error
		logger.Infof("Waiting to get ManagementIP from HNSNetwork %s", networkName)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			newNetwork, lastErr = hcsshim.HNSNetworkRequest("GET", newNetwork.Id, "")
			return newNetwork != nil && len(newNetwork.ManagementIP) != 0, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get management IP from HNSNetwork %s", networkName)
		}
		netshHelper := netsh.New(utilexec.New())
		logger.Infof("Waiting to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		waitErr = wait.Poll(500*time.Millisecond, 5*time.Second, func() (done bool, err error) {
			_, lastErr = netshHelper.GetInterfaceByIP(newNetwork.ManagementIP)
			return lastErr == nil, nil
		})
		if waitErr == wait.ErrWaitTimeout {
			return nil, errors.Annotatef(lastErr, "timeout, failed to get net interface for HNSNetwork %s (%s)", networkName, newNetwork.ManagementIP)
		}
		logger.Infof("Created HNSNetwork %s", networkName)
		existingNetwork = newNetwork
	}
	existingNetworkV2, err := hcn.GetNetworkByID(existingNetwork.Id)
	if err != nil {
		return nil, errors.Annotatef(err, "Could not find vxlan0 in V2")
	}
	addHostRoute := true
	for _, policy := range existingNetworkV2.Policies {
		if policy.Type == hcn.HostRoute {
			addHostRoute = false
		}
	}
	if addHostRoute {
		hostRoutePolicy := hcn.NetworkPolicy{
			Type:     hcn.HostRoute,
			Settings: []byte("{}"),
		}
		networkRequest := hcn.PolicyNetworkRequest{
			Policies: []hcn.NetworkPolicy{hostRoutePolicy},
		}
		err := existingNetworkV2.AddPolicy(networkRequest)
		if err != nil {
			logger.Warnf("Error adding policy to network : %v", err)
		}
	}
	return existingNetwork, nil
}
func EnsureNetworkExists(networkName string, subNet *net.IPNet, logger *logrus.Entry) (*hcsshim.HNSNetwork, error) {
	var err error
	createNetwork := true
	addressPrefix := subNet.String()
	gatewayAddress := getNthIP(subNet, 1)
	hnsNetwork, _ := hcsshim.GetHNSNetworkByName(networkName)
	if hnsNetwork != nil {
		for _, subnet := range hnsNetwork.Subnets {
			if subnet.AddressPrefix == addressPrefix && subnet.GatewayAddress == gatewayAddress.String() {
				createNetwork = false
				logger.Infof("Found existing HNS network [%+v]", hnsNetwork)
				break
			}
		}
	}
	if createNetwork {
		if hnsNetwork != nil {
			if _, err := hnsNetwork.Delete(); err != nil {
				logger.Errorf("Unable to delete existing network [%v], error: %v", hnsNetwork.Name, err)
				return nil, err
			}
			logger.Infof("Deleted stale HNS network [%v]")
		}
		req := map[string]interface{}{
			"Name": networkName,
			"Type": "L2Bridge",
			"Subnets": []interface{}{
				map[string]interface{}{
					"AddressPrefix":  addressPrefix,
					"GatewayAddress": gatewayAddress,
				},
			},
		}
		reqStr, err := json.Marshal(req)
		if err != nil {
			logger.Errorf("Error in converting to json format")
			return nil, err
		}
		logger.Infof("Attempting to create HNS network, request: %v", string(reqStr))
		if hnsNetwork, err = hcsshim.HNSNetworkRequest("POST", "", string(reqStr)); err != nil {
			logger.Errorf("unable to create network [%v], error: %v", networkName, err)
			return nil, err
		}
		logger.Infof("Created HNS network [%v] as %+v", networkName, hnsNetwork)
	}
	return hnsNetwork, err
}
func EnsureVXLANTunnelAddr(ctx context.Context, calicoClient calicoclient.Interface, nodeName string, ipNet *net.IPNet, networkName string) error {
	logrus.Debug("Checking the node's VXLAN tunnel address")
	var updateRequired bool
	node, err := calicoClient.Nodes().Get(ctx, nodeName, options.GetOptions{})
	if err != nil {
		return err
	}
	expectedIP := getNthIP(ipNet, 1).String()
	if node.Spec.IPv4VXLANTunnelAddr != expectedIP {
		logrus.WithField("ip", expectedIP).Debug("VXLAN tunnel IP to be updated")
		updateRequired = true
	}
	mac, err := GetDRMACAddr(networkName, ipNet)
	if err != nil {
		return err
	}
	expectedMAC := mac.String()
	if node.Spec.VXLANTunnelMACAddr != expectedMAC {
		logrus.WithField("mac", expectedMAC).Debug("VXLAN tunnel MAC to be updated")
		updateRequired = true
	}
	if updateRequired == false {
		return nil
	}
	node.Spec.IPv4VXLANTunnelAddr = expectedIP
	node.Spec.VXLANTunnelMACAddr = expectedMAC
	_, err = calicoClient.Nodes().Update(ctx, node, options.SetOptions{})
	return err
}
func createAndAttachVxlanHostEP(epName string, hnsNetwork *hcsshim.HNSNetwork, subNet *net.IPNet, logger *logrus.Entry) (*hcsshim.HNSEndpoint, error) {
	var err error
	endpointAddress := getNthIP(subNet, 2)
	existingEndpoint, err := hcsshim.GetHNSEndpointByName(epName)
	if err == nil && existingEndpoint.VirtualNetwork == hnsNetwork.Id {
		targetType := "PA"
		for _, policy := range existingEndpoint.Policies {
			policyType, _ := jsonparser.GetUnsafeString(policy, "Type")
			if policyType == targetType {
				actualPaIP, _ := jsonparser.GetUnsafeString(policy, targetType)
				if actualPaIP == hnsNetwork.ManagementIP {
					logger.Infof("Found existing remote HNSEndpoint %s", epName)
					return existingEndpoint, nil
				}
			}
		}
	}
	if existingEndpoint != nil {
		if _, err := existingEndpoint.Delete(); err != nil {
			return nil, errors.Annotatef(err, "failed to delete existing remote HNSEndpoint %s", epName)
		}
		logger.Infof("Deleted stale HNSEndpoint %s", epName)
	}
	macAddr := GetMacAddr(hnsNetwork.ManagementIP)
	newEndpoint := &hcsshim.HNSEndpoint{
		Name:             epName,
		IPAddress:        endpointAddress,
		MacAddress:       macAddr,
		VirtualNetwork:   hnsNetwork.Id,
		IsRemoteEndpoint: true,
		Policies: []json.RawMessage{
			[]byte(fmt.Sprintf(`{"Type":"PA","PA":"%s"}`, hnsNetwork.ManagementIP)),
		},
	}
	if _, err := newEndpoint.Create(); err != nil {
		return nil, errors.Annotatef(err, "failed to create remote HNSEndpoint %s", epName)
	}
	logger.Infof("Created HNSEndpoint %s", epName)
	return newEndpoint, nil
}
func CreateAndAttachHostEP(epName string, hnsNetwork *hcsshim.HNSNetwork, subNet *net.IPNet, logger *logrus.Entry) (*hcsshim.HNSEndpoint, error) {
	var err error
	endpointAddress := getNthIP(subNet, 2)
	attachEndpoint := true
	hnsEndpoint, _ := hcsshim.GetHNSEndpointByName(epName)
	if hnsEndpoint != nil {
		if !hnsEndpoint.IPAddress.Equal(endpointAddress) {
			if _, err = hnsEndpoint.Delete(); err != nil {
				logger.Errorf("Unable to delete existing bridge endpoint [%v], error: %v", epName, err)
				return nil, err
			}
			logger.Infof("Deleted stale bridge endpoint [%v]")
			hnsEndpoint = nil
		} else if strings.ToUpper(hnsEndpoint.VirtualNetwork) == strings.ToUpper(hnsNetwork.Id) {
			attachEndpoint = false
		} else {
			logger.Errorf("HnsEndpoint virtual network %s not matching ID %s",
				hnsEndpoint.VirtualNetwork, hnsNetwork.Id)
		}
	}
	if hnsEndpoint == nil {
		hnsEndpoint = &hcsshim.HNSEndpoint{
			Name:           epName,
			IPAddress:      endpointAddress,
			VirtualNetwork: hnsNetwork.Id,
		}
		logger.Infof("Attempting to create bridge endpoint [%+v]", hnsEndpoint)
		hnsEndpoint, err = hnsEndpoint.Create()
		if err != nil {
			logger.Errorf("Unable to create bridge endpoint [%v], error: %v", epName, err)
			return nil, err
		}
		logger.Infof("Created bridge endpoint [%v] as %+v", epName, hnsEndpoint)
	}
	if attachEndpoint {
		if err = hnsEndpoint.HostAttach(1); err != nil {
			logger.Errorf("Unable to hot attach bridge endpoint [%v] to host compartment, error: %v", epName, err)
			return nil, err
		}
		logger.Infof("Attached bridge endpoint [%v] to host", epName)
	}
	return hnsEndpoint, err
}
func chkMgmtIPandEnableForwarding(networkName string, hnsEndpoint *hcsshim.HNSEndpoint, logger *logrus.Entry) (network *hcsshim.HNSNetwork, err error) {
	netHelper := netsh.New(nil)
	startTime := time.Now()
	logCxt := logger.WithField("network", networkName)
	for {
		network, err = hcsshim.GetHNSNetworkByName(networkName)
		if err != nil {
			logger.Errorf("Unable to get hns network %s after creation, error: %v", networkName, err)
			return nil, err
		}
		if time.Since(startTime) > 30*time.Second {
			return nil, fmt.Errorf(
				"timed out waiting for interface matching the management IP (%v) of network %s",
				network.ManagementIP, networkName)
		}
		if len(network.ManagementIP) == 0 {
			logCxt.Info("Waiting for management IP...")
			time.Sleep(1 * time.Second)
			continue
		}
		if mgmtIface, err := netHelper.GetInterfaceByIP(network.ManagementIP); err != nil {
			logCxt.WithField("ip", network.ManagementIP).WithError(err).Warn(
				"Waiting for interface matching management IP...")
			time.Sleep(1 * time.Second)
			continue
		} else {
			err := enableForwarding(netHelper, mgmtIface, logger)
			if err != nil {
				return nil, err
			}
		}
		break
	}
	ourEpAddr := hnsEndpoint.IPAddress.String()
	netInterface, err := netHelper.GetInterfaceByIP(ourEpAddr)
	if err != nil {
		logger.WithError(err).Errorf("Unable to find interface matching our host endpoint [%v]", ourEpAddr)
		return nil, err
	}
	logger.Infof("Found Interface with IP[%s]: %v", ourEpAddr, netInterface)
	err = enableForwarding(netHelper, netInterface, logger)
	if err != nil {
		return nil, err
	}
	return network, nil
}
func enableForwarding(netHelper netsh.Interface, netInterface netsh.Ipv4Interface, logger *logrus.Entry) error {
	interfaceIdx := strconv.Itoa(netInterface.Idx)
	if err := netHelper.EnableForwarding(interfaceIdx); err != nil {
		logger.WithError(err).Errorf("Unable to enable forwarding on [%v] index [%v]",
			netInterface.Name, interfaceIdx)
		return err
	}
	logger.Infof("Enabled forwarding on [%v] index [%v]", netInterface.Name, interfaceIdx)
	return nil
}
func (d *windowsDataplane) createAndAttachContainerEP(args *skel.CmdArgs,
	hnsNetwork *hcsshim.HNSNetwork,
	affineBlockSubnet *net.IPNet,
	allIPAMPools []*net.IPNet,
	natOutgoing bool,
	result *current.Result,
	n *hns.NetConf) (*hcsshim.HNSEndpoint, *hcn.HostComputeEndpoint, error) {
	var gatewayAddress string
	if d.conf.Mode == "vxlan" {
		gatewayAddress = getNthIP(affineBlockSubnet, 1).String()
	} else {
		gatewayAddress = getNthIP(affineBlockSubnet, 2).String()
	}
	natExclusions := allIPAMPools
	mgmtIP := net.ParseIP(hnsNetwork.ManagementIP)
	if len(mgmtIP) == 0 {
		return nil, nil, fmt.Errorf("HNS network lost its management IP")
	}
	v1pols, v2pols, err := winpol.CalculateEndpointPolicies(n, natExclusions, natOutgoing, mgmtIP, d.logger)
	if err != nil {
		return nil, nil, err
	}
	endpointName := hns.ConstructEndpointName(args.ContainerID, args.Netns, n.Name)
	epIP := result.IPs[0].Address.IP
	epIPBytes := epIP.To4()
	macAddr := ""
	if d.conf.Mode == "vxlan" {
		vxlanMACPrefix := d.conf.VXLANMacPrefix
		if len(vxlanMACPrefix) != 0 {
			if len(vxlanMACPrefix) != 5 || vxlanMACPrefix[2] != '-' {
				return nil, nil, fmt.Errorf("endpointMacPrefix [%v] is invalid, value must be of the format xx-xx", vxlanMACPrefix)
			}
		} else {
			vxlanMACPrefix = "0E-2A"
		}
		macAddr = fmt.Sprintf("%v-%02x-%02x-%02x-%02x", vxlanMACPrefix, epIPBytes[0], epIPBytes[1], epIPBytes[2], epIPBytes[3])
		v1pols = append(v1pols, []json.RawMessage{
			[]byte(fmt.Sprintf(`{"Type":"PA","PA":"%s"}`, hnsNetwork.ManagementIP)),
		}...)
		hcnPol := hcn.EndpointPolicy{
			Type: hcn.NetworkProviderAddress,
			Settings: json.RawMessage(
				fmt.Sprintf(`{"ProviderAddress":"%s"}`, hnsNetwork.ManagementIP),
			),
		}
		v2pols = append(v2pols, hcnPol)
	} else {
		dict := map[string]interface{}{
			"Type":              "ROUTE",
			"DestinationPrefix": mgmtIP.String() + "/32",
			"NeedEncap":         true,
		}
		encoded, err := json.Marshal(dict)
		if err != nil {
			return nil, nil, fmt.Errorf("failed to add route encap policy")
		}
		v1pols = append(v1pols, json.RawMessage(encoded))
		hcnPol := hcn.EndpointPolicy{
			Type: hcn.SDNRoute,
			Settings: json.RawMessage(
				fmt.Sprintf(`{"DestinationPrefix": "%s", "NeedEncap": true}`, mgmtIP.String()+"/32"),
			),
		}
		v2pols = append(v2pols, hcnPol)
	}
	isDockerV1 := cri.IsDockershimV1(args.Netns)
	attempts := 3
	for {
		var hnsEndpointCont *hcsshim.HNSEndpoint
		var hcsEndpoint *hcn.HostComputeEndpoint
		var err error
		if isDockerV1 {
			d.logger.Infof("Attempting to create HNS endpoint name: %s for container", endpointName)
			_, err = hns.ProvisionEndpoint(endpointName, hnsNetwork.Id, args.ContainerID, args.Netns, func() (*hcsshim.HNSEndpoint, error) {
				hnsEP := &hcsshim.HNSEndpoint{
					Name:           endpointName,
					VirtualNetwork: hnsNetwork.Id,
					DNSServerList:  strings.Join(result.DNS.Nameservers, ","),
					DNSSuffix:      strings.Join(result.DNS.Search, ","),
					GatewayAddress: gatewayAddress,
					IPAddress:      epIP,
					MacAddress:     macAddr,
					Policies:       v1pols,
				}
				return hnsEP, nil
			})
			if err == nil {
				hnsEndpointCont, err = hcsshim.GetHNSEndpointByName(endpointName)
				d.logger.Infof("Endpoint to container created! %v", hnsEndpointCont)
			}
		} else {
			d.logger.Infof("Attempting to create HostComputeEndpoint: %s for container", endpointName)
			hcsEndpoint, err = hns.AddHcnEndpoint(endpointName, hnsNetwork.Id, args.Netns, func() (*hcn.HostComputeEndpoint, error) {
				hce := &hcn.HostComputeEndpoint{
					Name:               endpointName,
					HostComputeNetwork: hnsNetwork.Id,
					Dns: hcn.Dns{
						Domain:     result.DNS.Domain,
						Search:     result.DNS.Search,
						ServerList: result.DNS.Nameservers,
						Options:    result.DNS.Options,
					},
					MacAddress: macAddr,
					Routes: []hcn.Route{
						{
							NextHop:           gatewayAddress,
							DestinationPrefix: "0.0.0.0/0",
						},
					},
					IpConfigurations: []hcn.IpConfig{
						{
							IpAddress: epIP.String(),
						},
					},
					SchemaVersion: hcn.SchemaVersion{
						Major: 2,
					},
					Policies: v2pols,
				}
				return hce, nil
			})
			if err == nil {
				d.logger.Infof("Endpoint to container created! %v", hcsEndpoint)
			}
		}
		if err != nil {
			d.logger.WithError(err).Error("Error provisioning endpoint, checking if we need to clean it up.")
			cleanupErr := cleanUpEndpointByIP(epIP, d.logger, isDockerV1)
			if cleanupErr != nil {
				d.logger.WithError(err).Error("Failed to clean up (by IP) after failure.")
			} else {
				d.logger.Info("Cleanup (by IP) succeeded.")
			}
			cleanupErr = cleanUpEndpointByName(endpointName, d.logger, isDockerV1)
			if cleanupErr != nil {
				d.logger.WithError(err).Error("Failed to clean up (by name) after failure.")
			} else {
				d.logger.Info("Cleanup (by name) succeeded.")
			}
			if attempts > 0 {
				d.logger.Info("Retrying...")
				attempts--
				time.Sleep(time.Second)
				continue
			}
			return nil, nil, err
		}
		return hnsEndpointCont, hcsEndpoint, nil
	}
}
func cleanUpEndpointByIP(IP net.IP, logger *logrus.Entry, isDockerV1 bool) error {
	if isDockerV1 {
		endpoints, err := hcsshim.HNSListEndpointRequest()
		if err != nil {
			logger.WithError(err).Error("Failed to list endpoints")
			return err
		}
		for _, ep := range endpoints {
			if ep.IPAddress.Equal(IP) {
				logger.WithField("conflictingEndpoint", ep).Error("Found pre-existing conflicting endpoint.")
				_, err := ep.Delete()
				if err != nil {
					logger.WithError(err).Error("Failed to delete old endpoint")
				}
				return err  
			}
		}
	} else {
		endpoints, err := hcn.ListEndpoints()
		if err != nil {
			logger.WithError(err).Error("Failed to list endpoints")
			return err
		}
		for _, ep := range endpoints {
			for _, ipConf := range ep.IpConfigurations {
				if ipConf.IpAddress == IP.String() {
					logger.WithField("conflictingEndpoint", ep).Error("Found pre-existing conflicting host compute endpoint.")
					err := ep.Delete()
					if err != nil {
						logger.WithError(err).Error("Failed to delete old host compute endpoint")
					}
					return err  
				}
			}
		}
	}
	return nil
}
func cleanUpEndpointByName(endpointName string, logger *logrus.Entry, isDockerV1 bool) error {
	if isDockerV1 {
		hnsEndpoint, err := hcsshim.GetHNSEndpointByName(endpointName)
		if hcsshim.IsNotExist(err) {
			logger.Debug("Endpoint already gone.  Nothing to do.")
			return nil
		}
		if err != nil {
			logger.WithError(err).Error("Failed to get endpoint for cleanup.")
			return err
		}
		_, err = hnsEndpoint.Delete()
		return err
	} else {
		hceEndpoint, err := hcn.GetEndpointByName(endpointName)
		if hcn.IsNotFoundError(err) {
			logger.Debug("Endpoint already gone.  Nothing to do.")
			return nil
		}
		if err != nil {
			logger.WithError(err).Error("Failed to get endpoint for cleanup.")
			return err
		}
		err = hceEndpoint.Delete()
		return err
	}
}
func lookupManagementAddr(mgmtIP net.IP, logger *logrus.Entry) (*net.IPNet, error) {
	interfaces, err := net.Interfaces()
	if err != nil {
		logger.WithError(err).Error("Failed to look up host interfaces")
		return nil, err
	}
	for _, iface := range interfaces {
		addrs, err := iface.Addrs()
		if err != nil {
			logger.WithError(err).WithField("iface", iface.Name).Error(
				"Failed to look up host interface addresses")
			return nil, err
		}
		for _, addr := range addrs {
			if ipAddr, ok := addr.(*net.IPNet); ok {
				if ipAddr.Contains(mgmtIP) {
					return ipAddr, nil
				}
			}
		}
	}
	return nil, fmt.Errorf("couldn't find an interface matching management IP %s", mgmtIP.String())
}
func getNthIP(PodCIDR *net.IPNet, n int) net.IP {
	gwaddr := PodCIDR.IP.To4()
	buffer := make([]byte, len(gwaddr))
	copy(buffer, gwaddr)
	buffer[3] += byte(n)
	return buffer
}
func CreateNetworkName(netName string, subnet *net.IPNet) string {
	str := subnet.IP.String()
	network := strings.Replace(str, ".", "-", -1)
	name := netName + "-" + network
	return name
}
func SetupRoutes(hostVeth interface{}, result *current.Result) error {
	for _, ipAddr := range result.IPs {
		logrus.WithFields(logrus.Fields{"interface": hostVeth, "IP": ipAddr.Address}).Debugf("STUB: CNI adding route")
	}
	return nil
}