func (g *Cloud) DeleteExternalTargetPoolAndChecks(service *v1.Service, name, region, clusterID string, hcNames ...string) error {
	serviceName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
	lbRefStr := fmt.Sprintf("%v(%v)", name, serviceName)
	if err := g.DeleteTargetPool(name, region); err != nil && isHTTPErrorCode(err, http.StatusNotFound) {
		klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Target pool already deleted. Continuing to delete other resources.", lbRefStr)
	} else if err != nil {
		klog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete target pool, got error %s.", lbRefStr, err.Error())
		return err
	}
	for _, hcName := range hcNames {
		if err := func() error {
			isNodesHealthCheck := hcName != name
			if isNodesHealthCheck {
				g.sharedResourceLock.Lock()
				defer g.sharedResourceLock.Unlock()
			}
			klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check %v.", lbRefStr, hcName)
			if err := g.DeleteHTTPHealthCheck(hcName); err != nil {
				if isInUsedByError(err) {
					klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is in used: %v.", lbRefStr, hcName, err)
					return nil
				} else if !isHTTPErrorCode(err, http.StatusNotFound) {
					klog.Warningf("DeleteExternalTargetPoolAndChecks(%v): Failed to delete health check %v: %v.", lbRefStr, hcName, err)
					return err
				}
				klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Health check %v is already deleted.", lbRefStr, hcName)
			}
			fwName := MakeHealthCheckFirewallName(clusterID, hcName, isNodesHealthCheck)
			klog.Infof("DeleteExternalTargetPoolAndChecks(%v): Deleting health check firewall %v.", lbRefStr, fwName)
			if err := ignoreNotFound(g.DeleteFirewall(fwName)); err != nil {
				if isForbidden(err) && g.OnXPN() {
					klog.V(4).Infof("DeleteExternalTargetPoolAndChecks(%v): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName)
					g.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, g.NetworkProjectID()))
					return nil
				}
				return err
			}
			return nil
		}(); err != nil {
			return err
		}
	}
	return nil
}
DeleteExternalTargetPoolAndChecks Deletes an external load balancer pool and verifies the operation
package gce
import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/sets"
	servicehelpers "k8s.io/cloud-provider/service/helpers"
	utilnet "k8s.io/utils/net"
	compute "google.golang.org/api/compute/v1"
	"k8s.io/klog/v2"
)
const (
	errStrLbNoHosts = "cannot EnsureLoadBalancer() with no hosts"
)
func (g *Cloud) ensureExternalLoadBalancer(clusterName string, clusterID string, apiService *v1.Service, existingFwdRule *compute.ForwardingRule, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
	if len(nodes) == 0 {
		return nil, fmt.Errorf(errStrLbNoHosts)
	}
	hostNames := nodeNames(nodes)
	supportsNodesHealthCheck := supportsNodesHealthCheck(nodes)
	hosts, err := g.getInstancesByNames(hostNames)
	if err != nil {
		return nil, err
	}
	loadBalancerName := g.GetLoadBalancerName(context.TODO(), clusterName, apiService)
	requestedIP := apiService.Spec.LoadBalancerIP
	ports := apiService.Spec.Ports
	portStr := []string{}
	for _, p := range apiService.Spec.Ports {
		portStr = append(portStr, fmt.Sprintf("%s/%d", p.Protocol, p.Port))
	}
	serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name}
	lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName)
	klog.V(2).Infof("ensureExternalLoadBalancer(%s, %v, %v, %v, %v, %v)", lbRefStr, g.region, requestedIP, portStr, hostNames, apiService.Annotations)
	netTier, err := g.getServiceNetworkTier(apiService)
	if err != nil {
		klog.Errorf("ensureExternalLoadBalancer(%s): Failed to get the desired network tier: %v.", lbRefStr, err)
		return nil, err
	}
	klog.V(4).Infof("ensureExternalLoadBalancer(%s): Desired network tier %q.", lbRefStr, netTier)
	if _, ok := apiService.Annotations[NetworkTierAnnotationKey]; ok {
		g.deleteWrongNetworkTieredResources(loadBalancerName, lbRefStr, netTier)
	}
	fwdRuleExists, fwdRuleNeedsUpdate, fwdRuleIP, err := g.forwardingRuleNeedsUpdate(loadBalancerName, g.region, requestedIP, ports)
	if err != nil {
		return nil, err
	}
	if !fwdRuleExists {
		klog.V(2).Infof("ensureExternalLoadBalancer(%s): Forwarding rule %v doesn't exist.", lbRefStr, loadBalancerName)
	}
	ipAddressToUse := ""
	isUserOwnedIP := false  
	isSafeToReleaseIP := false
	defer func() {
		if isUserOwnedIP {
			return
		}
		if isSafeToReleaseIP {
			if err := g.DeleteRegionAddress(loadBalancerName, g.region); err != nil && !isNotFound(err) {
				klog.Errorf("ensureExternalLoadBalancer(%s): Failed to release static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err)
			} else if isNotFound(err) {
				klog.V(2).Infof("ensureExternalLoadBalancer(%s): IP address %s is not reserved.", lbRefStr, ipAddressToUse)
			} else {
				klog.Infof("ensureExternalLoadBalancer(%s): Released static IP %s.", lbRefStr, ipAddressToUse)
			}
		} else {
			klog.Warningf("ensureExternalLoadBalancer(%s): Orphaning static IP %s in region %v: %v.", lbRefStr, ipAddressToUse, g.region, err)
		}
	}()
	if requestedIP != "" {
		isUserOwnedIP, err = verifyUserRequestedIP(g, g.region, requestedIP, fwdRuleIP, lbRefStr, netTier)
		if err != nil {
			return nil, err
		}
		ipAddressToUse = requestedIP
	}
	if !isUserOwnedIP {
		ipAddr, existed, err := ensureStaticIP(g, loadBalancerName, serviceName.String(), g.region, fwdRuleIP, netTier)
		if err != nil {
			return nil, fmt.Errorf("failed to ensure a static IP for load balancer (%s): %v", lbRefStr, err)
		}
		klog.Infof("ensureExternalLoadBalancer(%s): Ensured IP address %s (tier: %s).", lbRefStr, ipAddr, netTier)
		isSafeToReleaseIP = !existed
		ipAddressToUse = ipAddr
	}
	sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService)
	if err != nil {
		return nil, err
	}
	firewallExists, firewallNeedsUpdate, err := g.firewallNeedsUpdate(loadBalancerName, serviceName.String(), ipAddressToUse, ports, sourceRanges)
	if err != nil {
		return nil, err
	}
	if firewallNeedsUpdate {
		desc := makeFirewallDescription(serviceName.String(), ipAddressToUse)
		if firewallExists {
			klog.Infof("ensureExternalLoadBalancer(%s): Updating firewall.", lbRefStr)
			if err := g.updateFirewall(apiService, MakeFirewallName(loadBalancerName), desc, sourceRanges, ports, hosts); err != nil {
				return nil, err
			}
			klog.Infof("ensureExternalLoadBalancer(%s): Updated firewall.", lbRefStr)
		} else {
			klog.Infof("ensureExternalLoadBalancer(%s): Creating firewall.", lbRefStr)
			if err := g.createFirewall(apiService, MakeFirewallName(loadBalancerName), desc, sourceRanges, ports, hosts); err != nil {
				return nil, err
			}
			klog.Infof("ensureExternalLoadBalancer(%s): Created firewall.", lbRefStr)
		}
	}
	tpExists, tpNeedsRecreation, err := g.targetPoolNeedsRecreation(loadBalancerName, g.region, apiService.Spec.SessionAffinity)
	if err != nil {
		return nil, err
	}
	if !tpExists {
		klog.Infof("ensureExternalLoadBalancer(%s): Target pool for service doesn't exist.", lbRefStr)
	}
	var hcToCreate, hcToDelete *compute.HttpHealthCheck
	hcLocalTrafficExisting, err := g.GetHTTPHealthCheck(loadBalancerName)
	if err != nil && !isHTTPErrorCode(err, http.StatusNotFound) {
		return nil, fmt.Errorf("error checking HTTP health check for load balancer (%s): %v", lbRefStr, err)
	}
	if path, healthCheckNodePort := servicehelpers.GetServiceHealthCheckPathPort(apiService); path != "" {
		klog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs local traffic health checks on: %d%s.", lbRefStr, healthCheckNodePort, path)
		if hcLocalTrafficExisting == nil {
			klog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from nodes health checks to local traffic health checks.", lbRefStr)
			if supportsNodesHealthCheck {
				hcToDelete = makeHTTPHealthCheck(MakeNodesHealthCheckName(clusterID), GetNodesHealthCheckPath(), GetNodesHealthCheckPort())
			}
			tpNeedsRecreation = true
		}
		hcToCreate = makeHTTPHealthCheck(loadBalancerName, path, healthCheckNodePort)
	} else {
		klog.V(4).Infof("ensureExternalLoadBalancer(%s): Service needs nodes health checks.", lbRefStr)
		if hcLocalTrafficExisting != nil {
			klog.V(2).Infof("ensureExternalLoadBalancer(%s): Updating from local traffic health checks to nodes health checks.", lbRefStr)
			hcToDelete = hcLocalTrafficExisting
			tpNeedsRecreation = true
		}
		if supportsNodesHealthCheck {
			hcToCreate = makeHTTPHealthCheck(MakeNodesHealthCheckName(clusterID), GetNodesHealthCheckPath(), GetNodesHealthCheckPort())
		}
	}
	if fwdRuleExists && (fwdRuleNeedsUpdate || tpNeedsRecreation) {
		isSafeToReleaseIP = false
		if err := g.DeleteRegionForwardingRule(loadBalancerName, g.region); err != nil && !isNotFound(err) {
			return nil, fmt.Errorf("failed to delete existing forwarding rule for load balancer (%s) update: %v", lbRefStr, err)
		}
		klog.Infof("ensureExternalLoadBalancer(%s): Deleted forwarding rule.", lbRefStr)
	}
	if err := g.ensureTargetPoolAndHealthCheck(tpExists, tpNeedsRecreation, apiService, loadBalancerName, clusterID, ipAddressToUse, hosts, hcToCreate, hcToDelete); err != nil {
		return nil, err
	}
	if tpNeedsRecreation || fwdRuleNeedsUpdate {
		klog.Infof("ensureExternalLoadBalancer(%s): Creating forwarding rule, IP %s (tier: %s).", lbRefStr, ipAddressToUse, netTier)
		if err := createForwardingRule(g, loadBalancerName, serviceName.String(), g.region, ipAddressToUse, g.targetPoolURL(loadBalancerName), ports, netTier); err != nil {
			return nil, fmt.Errorf("failed to create forwarding rule for load balancer (%s): %v", lbRefStr, err)
		}
		isSafeToReleaseIP = true
		klog.Infof("ensureExternalLoadBalancer(%s): Created forwarding rule, IP %s.", lbRefStr, ipAddressToUse)
	}
	status := &v1.LoadBalancerStatus{}
	status.Ingress = []v1.LoadBalancerIngress{{IP: ipAddressToUse}}
	return status, nil
}
func (g *Cloud) updateExternalLoadBalancer(clusterName string, service *v1.Service, nodes []*v1.Node) error {
	hosts, err := g.getInstancesByNames(nodeNames(nodes))
	if err != nil {
		return err
	}
	loadBalancerName := g.GetLoadBalancerName(context.TODO(), clusterName, service)
	return g.updateTargetPool(loadBalancerName, hosts)
}
func (g *Cloud) ensureExternalLoadBalancerDeleted(clusterName, clusterID string, service *v1.Service) error {
	loadBalancerName := g.GetLoadBalancerName(context.TODO(), clusterName, service)
	serviceName := types.NamespacedName{Namespace: service.Namespace, Name: service.Name}
	lbRefStr := fmt.Sprintf("%v(%v)", loadBalancerName, serviceName)
	var hcNames []string
	if path, _ := servicehelpers.GetServiceHealthCheckPathPort(service); path != "" {
		hcToDelete, err := g.GetHTTPHealthCheck(loadBalancerName)
		if err != nil && !isHTTPErrorCode(err, http.StatusNotFound) {
			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Failed to retrieve health check:%v.", lbRefStr, err)
			return err
		}
		if err == nil {
			hcNames = append(hcNames, hcToDelete.Name)
		}
	} else {
		hcNames = append(hcNames, loadBalancerName)
		hcNames = append(hcNames, MakeNodesHealthCheckName(clusterID))
	}
	errs := utilerrors.AggregateGoroutines(
		func() error {
			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting firewall rule.", lbRefStr)
			fwName := MakeFirewallName(loadBalancerName)
			err := ignoreNotFound(g.DeleteFirewall(fwName))
			if isForbidden(err) && g.OnXPN() {
				klog.V(4).Infof("ensureExternalLoadBalancerDeleted(%s): Do not have permission to delete firewall rule %v (on XPN). Raising event.", lbRefStr, fwName)
				g.raiseFirewallChangeNeededEvent(service, FirewallToGCloudDeleteCmd(fwName, g.NetworkProjectID()))
				return nil
			}
			return err
		},
		func() error {
			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting IP address.", lbRefStr)
			return ignoreNotFound(g.DeleteRegionAddress(loadBalancerName, g.region))
		},
		func() error {
			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting forwarding rule.", lbRefStr)
			if err := ignoreNotFound(g.DeleteRegionForwardingRule(loadBalancerName, g.region)); err != nil {
				return err
			}
			klog.Infof("ensureExternalLoadBalancerDeleted(%s): Deleting target pool.", lbRefStr)
			if err := g.DeleteExternalTargetPoolAndChecks(service, loadBalancerName, g.region, clusterID, hcNames...); err != nil {
				return err
			}
			return nil
		},
	)
	if errs != nil {
		return utilerrors.Flatten(errs)
	}
	return nil
}