func (c *IPAMContext) tryUnassignCidrsFromAll() {
	_, over, warmTargetDefined := c.datastoreTargetState()
	if !warmTargetDefined {
		over = c.computeExtraPrefixesOverWarmTarget()
	}
	if over > 0 {
		eniInfos := c.dataStore.GetENIInfos()
		for eniID := range eniInfos.ENIs {
			cidrs := c.dataStore.FindFreeableCidrs(eniID)
			if cidrs == nil {
				log.Errorf("Error finding unassigned IPs for ENI %s", eniID)
				return
			}
			numFreeable := min(over, len(cidrs))
			cidrs = cidrs[:numFreeable]
			if len(cidrs) == 0 {
				continue
			}
			var deletedCidrs []datastore.CidrInfo
			for _, toDelete := range cidrs {
				err := c.dataStore.DelIPv4CidrFromStore(eniID, toDelete.Cidr, false  )
				if err != nil {
					log.Warnf("Failed to delete Cidr %s on ENI %s from datastore: %s", toDelete, eniID, err)
					ipamdErrInc("decreaseIPPool")
					continue
				} else {
					deletedCidrs = append(deletedCidrs, toDelete)
				}
			}
			c.DeallocCidrs(eniID, deletedCidrs)
		}
	}
}
tryUnassignIPsorPrefixesFromAll determines if there are IPs to free when we have extra IPs beyond the target and warmIPTargetDefined
is enabled, deallocate extra IP addresses
package ipamd
import (
	"context"
	"fmt"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd/datastore"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
)
const (
	ipPoolMonitorInterval       = 5 * time.Second
	maxRetryCheckENI            = 5
	eniAttachTime               = 10 * time.Second
	nodeIPPoolReconcileInterval = 60 * time.Second
	decreaseIPPoolInterval      = 30 * time.Second
	ipReconcileCooldown = 60 * time.Second
	envWarmIPTarget = "WARM_IP_TARGET"
	noWarmIPTarget  = 0
	envMinimumIPTarget = "MINIMUM_IP_TARGET"
	noMinimumIPTarget  = 0
	envWarmENITarget     = "WARM_ENI_TARGET"
	defaultWarmENITarget = 1
	envMaxENI     = "MAX_ENI"
	defaultMaxENI = -1
	envCustomNetworkCfg = "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG"
	eniNoManageTagKey = "node.k8s.amazonaws.com/no_manage"
	envDisableENIProvisioning = "DISABLE_NETWORK_RESOURCE_PROVISIONING"
	envBackingStorePath     = "AWS_VPC_K8S_CNI_BACKING_STORE"
	defaultBackingStorePath = "/var/run/aws-node/ipam.json"
	envEnablePodENI = "ENABLE_POD_ENI"
	envNodeName = "MY_NODE_NAME"
	vpcENIConfigLabel = "vpc.amazonaws.com/eniConfig"
	envEnableIpv4PrefixDelegation = "ENABLE_PREFIX_DELEGATION"
	envWarmPrefixTarget     = "WARM_PREFIX_TARGET"
	defaultWarmPrefixTarget = 0
	envEnableIPv4 = "ENABLE_IPv4"
	envEnableIPv6 = "ENABLE_IPv6"
	ipV4AddrFamily = "4"
	ipV6AddrFamily = "6"
	insufficientCidrErrorCooldown = 120 * time.Second
	envManageUntaggedENI = "MANAGE_UNTAGGED_ENI"
	eniNodeTagKey = "node.k8s.amazonaws.com/instance_id"
	envAnnotatePodIP = "ANNOTATE_POD_IP"
)
var log = logger.Get()
var (
	ipamdErr = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "awscni_ipamd_error_count",
			Help: "The number of errors encountered in ipamd",
		},
		[]string{"fn"},
	)
	ipamdActionsInprogress = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "awscni_ipamd_action_inprogress",
			Help: "The number of ipamd actions in progress",
		},
		[]string{"fn"},
	)
	enisMax = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "awscni_eni_max",
			Help: "The maximum number of ENIs that can be attached to the instance, accounting for unmanaged ENIs",
		},
	)
	ipMax = prometheus.NewGauge(
		prometheus.GaugeOpts{
			Name: "awscni_ip_max",
			Help: "The maximum number of IP addresses that can be allocated to the instance",
		},
	)
	reconcileCnt = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "awscni_reconcile_count",
			Help: "The number of times ipamd reconciles on ENIs and IP/Prefix addresses",
		},
		[]string{"fn"},
	)
	addIPCnt = prometheus.NewCounter(
		prometheus.CounterOpts{
			Name: "awscni_add_ip_req_count",
			Help: "The number of add IP address requests",
		},
	)
	delIPCnt = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "awscni_del_ip_req_count",
			Help: "The number of delete IP address requests",
		},
		[]string{"reason"},
	)
	podENIErr = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Name: "awscni_pod_eni_error_count",
			Help: "The number of errors encountered for pod ENIs",
		},
		[]string{"fn"},
	)
	prometheusRegistered = false
)
type IPAMContext struct {
	awsClient            awsutils.APIs
	dataStore            *datastore.DataStore
	rawK8SClient         client.Client
	cachedK8SClient      client.Client
	enableIPv4           bool
	enableIPv6           bool
	useCustomNetworking  bool
	networkClient        networkutils.NetworkAPIs
	maxIPsPerENI         int
	maxENI               int
	maxPrefixesPerENI    int
	unmanagedENI         int
	warmENITarget        int
	warmIPTarget         int
	minimumIPTarget      int
	warmPrefixTarget     int
	primaryIP            map[string]string  
	lastNodeIPPoolAction time.Time
	lastDecreaseIPPool   time.Time
	reconcileCooldownCache    ReconcileCooldownCache
	terminating               int32  
	disableENIProvisioning    bool
	enablePodENI              bool
	myNodeName                string
	enablePrefixDelegation    bool
	lastInsufficientCidrError time.Time
	enableManageUntaggedMode  bool
	enablePodIPAnnotation     bool
}
func (c *IPAMContext) setUnmanagedENIs(tagMap map[string]awsutils.TagMap) {
	if len(tagMap) == 0 {
		return
	}
	var unmanagedENIlist []string
	for eniID, tags := range tagMap {
		if _, found := tags[eniNoManageTagKey]; found {
			if tags[eniNoManageTagKey] != "true" {
				continue
			}
		} else if _, found := tags[eniNodeTagKey]; found && tags[eniNodeTagKey] == c.awsClient.GetInstanceID() {
			continue
		} else if c.enableManageUntaggedMode {
			continue
		}
		if eniID == c.awsClient.GetPrimaryENI() {
			log.Debugf("Ignoring primary ENI %s since it is always managed", eniID)
		} else {
			log.Debugf("Marking ENI %s as being unmanaged", eniID)
			unmanagedENIlist = append(unmanagedENIlist, eniID)
		}
	}
	c.awsClient.SetUnmanagedENIs(unmanagedENIlist)
}
type ReconcileCooldownCache struct {
	sync.RWMutex
	cache map[string]time.Time
}
func (r *ReconcileCooldownCache) Add(cidr string) {
	r.Lock()
	defer r.Unlock()
	expiry := time.Now().Add(ipReconcileCooldown)
	r.cache[cidr] = expiry
}
func (r *ReconcileCooldownCache) Remove(cidr string) {
	r.Lock()
	defer r.Unlock()
	log.Debugf("Removing %s from cooldown cache.", cidr)
	delete(r.cache, cidr)
}
func (r *ReconcileCooldownCache) RecentlyFreed(cidr string) (found, recentlyFreed bool) {
	r.Lock()
	defer r.Unlock()
	now := time.Now()
	if expiry, ok := r.cache[cidr]; ok {
		log.Debugf("Checking if CIDR %s has been recently freed. Cooldown expires at: %s. (Cooldown: %v)", cidr, expiry, now.Sub(expiry) < 0)
		return true, now.Sub(expiry) < 0
	}
	return false, false
}
func prometheusRegister() {
	if !prometheusRegistered {
		prometheus.MustRegister(ipamdErr)
		prometheus.MustRegister(ipamdActionsInprogress)
		prometheus.MustRegister(enisMax)
		prometheus.MustRegister(ipMax)
		prometheus.MustRegister(reconcileCnt)
		prometheus.MustRegister(addIPCnt)
		prometheus.MustRegister(delIPCnt)
		prometheus.MustRegister(podENIErr)
		prometheusRegistered = true
	}
}
func containsInsufficientCidrBlocksError(err error) bool {
	var awsErr awserr.Error
	if errors.As(err, &awsErr) {
		return awsErr.Code() == "InsufficientCidrBlocks"
	}
	return false
}
func (c *IPAMContext) inInsufficientCidrCoolingPeriod() bool {
	return time.Since(c.lastInsufficientCidrError) <= insufficientCidrErrorCooldown
}
func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContext, error) {
	prometheusRegister()
	c := &IPAMContext{}
	c.rawK8SClient = rawK8SClient
	c.cachedK8SClient = cachedK8SClient
	c.networkClient = networkutils.New()
	c.useCustomNetworking = UseCustomNetworkCfg()
	c.enablePrefixDelegation = usePrefixDelegation()
	c.enableIPv4 = isIPv4Enabled()
	c.enableIPv6 = isIPv6Enabled()
	c.disableENIProvisioning = disablingENIProvisioning()
	client, err := awsutils.New(c.useCustomNetworking, c.disableENIProvisioning, c.enableIPv4, c.enableIPv6)
	if err != nil {
		return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface")
	}
	c.awsClient = client
	c.primaryIP = make(map[string]string)
	c.reconcileCooldownCache.cache = make(map[string]time.Time)
	c.warmENITarget = getWarmENITarget()
	c.warmIPTarget = getWarmIPTarget()
	c.minimumIPTarget = getMinimumIPTarget()
	c.warmPrefixTarget = getWarmPrefixTarget()
	c.enablePodENI = enablePodENI()
	c.enableManageUntaggedMode = enableManageUntaggedMode()
	c.enablePodIPAnnotation = enablePodIPAnnotation()
	err = c.awsClient.FetchInstanceTypeLimits()
	if err != nil {
		log.Errorf("Failed to get ENI limits from file:vpc_ip_limits or EC2 for %s", c.awsClient.GetInstanceType())
		return nil, err
	}
	if !c.isConfigValid() {
		return nil, fmt.Errorf("ipamd: failed to validate configuration")
	}
	c.awsClient.InitCachedPrefixDelegation(c.enablePrefixDelegation)
	c.myNodeName = os.Getenv("MY_NODE_NAME")
	checkpointer := datastore.NewJSONFile(dsBackingStorePath())
	c.dataStore = datastore.NewDataStore(log, checkpointer, c.enablePrefixDelegation)
	err = c.nodeInit()
	if err != nil {
		return nil, err
	}
	mac := c.awsClient.GetPrimaryENImac()
	if c.enableIPv4 || !c.disableENIProvisioning {
		err = c.awsClient.RefreshSGIDs(mac)
		if err != nil {
			return nil, err
		}
		go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second)
	}
	return c, nil
}
func (c *IPAMContext) nodeInit() error {
	ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1))
	defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1))
	var err error
	var vpcV4CIDRs []string
	ctx := context.TODO()
	log.Debugf("Start node init")
	primaryV4IP := c.awsClient.GetLocalIPv4()
	err = c.initENIAndIPLimits()
	if c.enableIPv4 {
		vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs()
		if err != nil {
			return err
		}
	}
	err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4,
		c.enableIPv6)
	if err != nil {
		return errors.Wrap(err, "ipamd init: failed to set up host network")
	}
	metadataResult, err := c.awsClient.DescribeAllENIs()
	if err != nil {
		return errors.New("ipamd init: failed to retrieve attached ENIs info")
	}
	log.Debugf("DescribeAllENIs success: ENIs: %d, tagged: %d", len(metadataResult.ENIMetadata), len(metadataResult.TagMap))
	c.awsClient.SetCNIUnmanagedENIs(metadataResult.MultiCardENIIDs)
	c.setUnmanagedENIs(metadataResult.TagMap)
	enis := c.filterUnmanagedENIs(metadataResult.ENIMetadata)
	for _, eni := range enis {
		log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID)
		isTrunkENI := eni.ENIID == metadataResult.TrunkENI
		isEFAENI := metadataResult.EFAENIs[eni.ENIID]
		if !isTrunkENI && !c.disableENIProvisioning {
			if err := c.awsClient.TagENI(eni.ENIID, metadataResult.TagMap[eni.ENIID]); err != nil {
				return errors.Wrapf(err, "ipamd init: failed to tag managed ENI %v", eni.ENIID)
			}
		}
		retry := 0
		for {
			retry++
			if err = c.setupENI(eni.ENIID, eni, isTrunkENI, isEFAENI); err == nil {
				log.Infof("ENI %s set up.", eni.ENIID)
				break
			}
			if retry > maxRetryCheckENI {
				log.Warnf("Reached max retry: Unable to discover attached IPs for ENI from metadata service (attempted %d/%d): %v", retry, maxRetryCheckENI, err)
				ipamdErrInc("waitENIAttachedMaxRetryExceeded")
				break
			}
			log.Warnf("Error trying to set up ENI %s: %v", eni.ENIID, err)
			if strings.Contains(err.Error(), "setupENINetwork: failed to find the link which uses MAC address") {
				log.Debug("Unable to match link for this ENI, going to the next one.")
				break
			}
			log.Debugf("Unable to discover IPs for this ENI yet (attempt %d/%d)", retry, maxRetryCheckENI)
			time.Sleep(eniAttachTime)
		}
	}
	if err := c.dataStore.ReadBackingStore(c.enableIPv6); err != nil {
		return err
	}
	if c.enableIPv6 {
		return nil
	}
	if c.enablePrefixDelegation {
		c.tryUnassignIPsFromENIs()
	} else {
		c.tryUnassignPrefixesFromENIs()
	}
	if err = c.configureIPRulesForPods(); err != nil {
		return err
	}
	go wait.Forever(func() {
		vpcV4CIDRs = c.updateCIDRsRulesOnChange(vpcV4CIDRs)
	}, 30*time.Second)
	eniConfigName, err := eniconfig.GetNodeSpecificENIConfigName(ctx, c.cachedK8SClient)
	if err == nil && c.useCustomNetworking && eniConfigName != "default" {
		err := c.SetNodeLabel(ctx, vpcENIConfigLabel, eniConfigName)
		if err != nil {
			log.Errorf("Failed to set eniConfig node label", err)
			podENIErrInc("nodeInit")
			return err
		}
	} else {
		err := c.SetNodeLabel(ctx, vpcENIConfigLabel, "")
		if err != nil {
			log.Errorf("Failed to delete eniConfig node label", err)
			podENIErrInc("nodeInit")
			return err
		}
	}
	if metadataResult.TrunkENI != "" {
		err := c.SetNodeLabel(ctx, "vpc.amazonaws.com/has-trunk-attached", "true")
		if err != nil {
			log.Errorf("Failed to set node label", err)
			podENIErrInc("nodeInit")
			return err
		}
	} else {
		c.askForTrunkENIIfNeeded(ctx)
	}
	if !c.disableENIProvisioning {
		increasedPool, err := c.tryAssignCidrs()
		if err == nil && increasedPool {
			c.updateLastNodeIPPoolAction()
		} else if err != nil {
			if containsInsufficientCidrBlocksError(err) {
				log.Errorf("Unable to attach IPs/Prefixes for the ENI, subnet doesn't seem to have enough IPs/Prefixes. Consider using new subnet or carve a reserved range using create-subnet-cidr-reservation")
				c.lastInsufficientCidrError = time.Now()
				return nil
			}
			return err
		}
	}
	return nil
}
func (c *IPAMContext) configureIPRulesForPods() error {
	rules, err := c.networkClient.GetRuleList()
	if err != nil {
		log.Errorf("During ipamd init: failed to retrieve IP rule list %v", err)
		return nil
	}
	for _, info := range c.dataStore.AllocatedIPs() {
		srcIPNet := net.IPNet{IP: net.ParseIP(info.IP), Mask: net.IPv4Mask(255, 255, 255, 255)}
		err = c.networkClient.UpdateRuleListBySrc(rules, srcIPNet)
		if err != nil {
			log.Warnf("UpdateRuleListBySrc in nodeInit() failed for IP %s: %v", info.IP, err)
		}
	}
	return nil
}
func (c *IPAMContext) updateCIDRsRulesOnChange(oldVPCCIDRs []string) []string {
	newVPCCIDRs, err := c.awsClient.GetVPCIPv4CIDRs()
	if err != nil {
		log.Warnf("skipping periodic update to VPC CIDRs due to error: %v", err)
		return oldVPCCIDRs
	}
	old := sets.NewString(oldVPCCIDRs...)
	new := sets.NewString(newVPCCIDRs...)
	if !old.Equal(new) {
		primaryIP := c.awsClient.GetLocalIPv4()
		err = c.networkClient.UpdateHostIptablesRules(newVPCCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP, c.enableIPv4,
			c.enableIPv6)
		if err != nil {
			log.Warnf("unable to update host iptables rules for VPC CIDRs due to error: %v", err)
		}
	}
	return newVPCCIDRs
}
func (c *IPAMContext) updateIPStats(unmanaged int) {
	ipMax.Set(float64(c.maxIPsPerENI * (c.maxENI - unmanaged)))
	enisMax.Set(float64(c.maxENI - unmanaged))
}
func (c *IPAMContext) StartNodeIPPoolManager() {
	if c.enableIPv6 {
		return
	}
	sleepDuration := ipPoolMonitorInterval / 2
	ctx := context.Background()
	for {
		if !c.disableENIProvisioning {
			time.Sleep(sleepDuration)
			c.updateIPPoolIfRequired(ctx)
		}
		time.Sleep(sleepDuration)
		c.nodeIPPoolReconcile(ctx, nodeIPPoolReconcileInterval)
	}
}
func (c *IPAMContext) updateIPPoolIfRequired(ctx context.Context) {
	c.askForTrunkENIIfNeeded(ctx)
	if c.isDatastorePoolTooLow() {
		c.increaseDatastorePool(ctx)
	} else if c.isDatastorePoolTooHigh() {
		c.decreaseDatastorePool(decreaseIPPoolInterval)
	}
	if c.shouldRemoveExtraENIs() {
		c.tryFreeENI()
	}
}
func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) {
	ipamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Add(float64(1))
	defer ipamdActionsInprogress.WithLabelValues("decreaseDatastorePool").Sub(float64(1))
	now := time.Now()
	timeSinceLast := now.Sub(c.lastDecreaseIPPool)
	if timeSinceLast <= interval {
		log.Debugf("Skipping decrease Datastore pool because time since last %v <= %v", timeSinceLast, interval)
		return
	}
	log.Debugf("Starting to decrease Datastore pool")
	c.tryUnassignCidrsFromAll()
	c.lastDecreaseIPPool = now
	c.lastNodeIPPoolAction = now
	total, used, _, cooldownIPs := c.dataStore.GetStats(ipV4AddrFamily)
	log.Debugf("Successfully decreased IP pool")
	logPoolStats(total, used, cooldownIPs, c.maxIPsPerENI, c.enablePrefixDelegation)
}
func (c *IPAMContext) tryFreeENI() {
	if c.isTerminating() {
		log.Debug("AWS CNI is terminating, not detaching any ENIs")
		return
	}
	eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget, c.warmPrefixTarget)
	if eni == "" {
		return
	}
	log.Debugf("Start freeing ENI %s", eni)
	err := c.awsClient.FreeENI(eni)
	if err != nil {
		ipamdErrInc("decreaseIPPoolFreeENIFailed")
		log.Errorf("Failed to free ENI %s, err: %v", eni, err)
		return
	}
}