func changeRequestMatches(change *servicequotas.RequestedServiceQuotaChange, quota float64) bool {
	if *change.ServiceCode != vCPUServiceCode {
		return false
	}
	if *change.QuotaCode != vCPUQuotaCode {
		return false
	}
	if *change.DesiredValue != quota {
		return false
	}
	return true
}
changeRequestMatches returns true if the QuotaCode, ServiceCode and desired value match
package account
import (
	"errors"
	"fmt"
	"strconv"
	"time"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/servicequotas"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/go-logr/logr"
	awsv1alpha1 "github.com/openshift/aws-account-operator/pkg/apis/aws/v1alpha1"
	"github.com/openshift/aws-account-operator/pkg/awsclient"
	controllerutils "github.com/openshift/aws-account-operator/pkg/controller/utils"
	retry "github.com/avast/retry-go"
)
type regionInitializationError struct {
	ErrorMsg string
	Region 	 string
}
func (r *ReconcileAccount) InitializeSupportedRegions(reqLogger logr.Logger, account *awsv1alpha1.Account, regions []awsv1alpha1.AwsRegions, creds *sts.AssumeRoleOutput, regionAMIs map[string]awsv1alpha1.AmiSpec) {
	ec2Notifications, ec2Errors := make(chan string), make(chan regionInitializationError)
	defer close(ec2Notifications)
	defer close(ec2Errors)
	vCPUQuota, _ := r.getDesiredVCPUValue(reqLogger)
	reqLogger.Info("retrieved desired vCPU quota value from configMap", "quota.vcpu", vCPUQuota)
	managedTags := r.getManagedTags(reqLogger)
	customerTags := r.getCustomTags(reqLogger, account)
	for _, region := range regions {
		go r.InitializeRegion(reqLogger, account, region.Name, regionAMIs[region.Name], vCPUQuota, ec2Notifications, ec2Errors, creds, managedTags, customerTags)  
	}
	var regionInitFailedRegion []string
	regionInitFailed := false
	for i := 0; i < len(regions); i++ {
		select {
		case msg := <-ec2Notifications:
			reqLogger.Info(msg)
		case errMsg := <-ec2Errors:
			regionInitFailed = true
			reqLogger.Error(errors.New(errMsg.ErrorMsg), errMsg.ErrorMsg)
			regionInitFailedRegion = append(regionInitFailedRegion, errMsg.Region)
		}
	}
	if regionInitFailed && len(regions) == 1 {
		controllerutils.SetAccountStatus(
			account,
			fmt.Sprintf("Account %s failed to initialize expected region %v", account.Name, regionInitFailedRegion),
			awsv1alpha1.AccountInitializingRegions,
			AccountFailed,
		)
	} else {
		reqLogger.Info("Successfully completed initializing desired regions")
	}
}
func (r *ReconcileAccount) InitializeRegion(
	reqLogger logr.Logger,
	account *awsv1alpha1.Account,
	region string,
	instanceInfo awsv1alpha1.AmiSpec,
	vCPUQuota float64,
	ec2Notifications chan string,
	ec2Errors chan regionInitializationError,
	creds *sts.AssumeRoleOutput,
	managedTags []awsclient.AWSTag,
	customerTags []awsclient.AWSTag,
) error {
	var quotaIncreaseRequired bool
	var caseID string
	awsClient, err := r.awsClientBuilder.GetClient(controllerName, r.Client, awsclient.NewAwsClientInput{
		AwsCredsSecretIDKey:     *creds.Credentials.AccessKeyId,
		AwsCredsSecretAccessKey: *creds.Credentials.SecretAccessKey,
		AwsToken:                *creds.Credentials.SessionToken,
		AwsRegion:               region,
	})
	if err != nil {
		connErr := fmt.Sprintf("unable to connect to region %s when attempting to initialize it", region)
		reqLogger.Error(err, connErr)
		ec2Errors <- regionInitializationError{ErrorMsg: connErr, Region: region}
		return err
	}
	reqLogger.Info("initializing region", "region", region)
	cleaned, err := cleanRegion(awsClient, reqLogger, account.Name, region)
	if err != nil {
		cleanErr := fmt.Sprintf("Error while attempting to clean region: %v", err.Error())
		ec2Errors <- regionInitializationError{ErrorMsg: cleanErr, Region: region}
		return err
	}
	if cleaned {
		ec2Notifications <- fmt.Sprintf("Region %s was already innitialized", region)
		return nil
	}
	if vCPUQuota != 0 {
		quotaIncreaseRequired, err = vCPUQuotaNeedsIncrease(awsClient, vCPUQuota)
		if err != nil {
			reqLogger.Error(err, "failed retrieving current vCPU quota from AWS")
		}
	}
	if quotaIncreaseRequired {
		reqLogger.Info("vCPU quota increase required", "region", region)
		caseID, err = checkQuotaRequestHistory(awsClient, vCPUQuota)
		if err != nil {
			reqLogger.Error(err, "failed retrieving quota change history")
		}
		if caseID != "" {
			reqLogger.Info("found matching quota change request", "caseID", caseID)
		}
		if caseID == "" && err == nil {
			reqLogger.Info("submitting vCPU quota increase request", "region", region)
			caseID, err = setVCPUQuota(awsClient, vCPUQuota)
			if err != nil {
				reqLogger.Error(err, "failed requesting vCPU quota increase")
			}
		}
		if caseID != "" {
			reqLogger.Info("quota increase request submitted successfully", "region", region, "caseID", caseID)
		}
	}
	err = r.BuildAndDestroyEC2Instances(reqLogger, account, awsClient, instanceInfo, managedTags, customerTags)
	if err != nil {
		createErr := fmt.Sprintf("Unable to create instance in region: %s", region)
		controllerutils.LogAwsError(reqLogger, createErr, nil, err)
		ec2Errors <- regionInitializationError{ErrorMsg: createErr, Region: region}
		return err
	}
	successMsg := fmt.Sprintf("EC2 instance created and terminated successfully in region: %s", region)
	ec2Notifications <- successMsg
	return nil
}
func (r *ReconcileAccount) BuildAndDestroyEC2Instances(reqLogger logr.Logger, account *awsv1alpha1.Account, awsClient awsclient.Client, instanceInfo awsv1alpha1.AmiSpec, managedTags []awsclient.AWSTag, customerTags []awsclient.AWSTag) error {
	instanceID, err := CreateEC2Instance(reqLogger, account, awsClient, instanceInfo, managedTags, customerTags)
	if err != nil {
		if instanceID != "" {
			reqLogger.Error(err, fmt.Sprintf("Early termination of instance with ID: %s", instanceID))
			termErr := TerminateEC2Instance(reqLogger, awsClient, instanceID)
			if termErr != nil {
				controllerutils.LogAwsError(reqLogger, "AWS error while attempting to terminate instance", nil, termErr)
			}
		}
		return err
	}
	var DescError error
	totalWait := controllerutils.WaitTime * 60
	currentWait := 1
	for totalWait > 0 {
		currentWait = currentWait * 2
		if currentWait > totalWait {
			currentWait = totalWait
		}
		totalWait -= currentWait
		time.Sleep(time.Duration(currentWait) * time.Second)
		var code int
		code, DescError = DescribeEC2Instances(reqLogger, awsClient, instanceID)
		if code == 16 {  
			reqLogger.Info(fmt.Sprintf("EC2 Instance: %s Running", instanceID))
			break
		} else if code == 401 {  
			reqLogger.Error(DescError, fmt.Sprintf("Missing required permissions for account %s", account.Name))
			return err
		}
	}
	if DescError != nil {
		DescErrorMsg := fmt.Sprintf("Could not get EC2 instance state, terminating instance %s", instanceID)
		if DescError, ok := err.(awserr.Error); ok {
			DescErrorMsg = fmt.Sprintf("Could not get EC2 instance state: %s, terminating instance %s", DescError.Code(), instanceID)
		}
		reqLogger.Error(DescError, DescErrorMsg)
	}
	reqLogger.Info(fmt.Sprintf("Terminating EC2 Instance: %s", instanceID))
	err = TerminateEC2Instance(reqLogger, awsClient, instanceID)
	if err != nil {
		return err
	}
	reqLogger.Info(fmt.Sprintf("EC2 Instance: %s Terminated", instanceID))
	return nil
}
func CreateEC2Instance(reqLogger logr.Logger, account *awsv1alpha1.Account, client awsclient.Client, instanceInfo awsv1alpha1.AmiSpec, managedTags []awsclient.AWSTag, customerTags []awsclient.AWSTag) (string, error) {
	var timeoutInstanceID string
	totalWait := controllerutils.WaitTime * 60
	currentWait := 1
	for totalWait > 0 {
		currentWait = currentWait * 2
		if currentWait > totalWait {
			currentWait = totalWait
		}
		totalWait -= currentWait
		time.Sleep(time.Duration(currentWait) * time.Second)
		tags := awsclient.AWSTags.BuildTags(account, managedTags, customerTags).GetEC2Tags()
		runResult, runErr := client.RunInstances(&ec2.RunInstancesInput{
			ImageId:      aws.String(instanceInfo.Ami),
			InstanceType: aws.String(instanceInfo.InstanceType),
			MinCount:     aws.Int64(1),
			MaxCount:     aws.Int64(1),
			TagSpecifications: []*ec2.TagSpecification{
				{
					ResourceType: &awsv1alpha1.InstanceResourceType,
					Tags:         tags,
				},
				{
					ResourceType: &awsv1alpha1.VolumeResourceType,
					Tags:         tags,
				},
			},
		})
		if runErr != nil {
			if aerr, ok := runErr.(awserr.Error); ok {
				if len(runResult.Instances) > 0 {
					timeoutInstanceID = *runResult.Instances[0].InstanceId
				}
				switch aerr.Code() {
				case "PendingVerification", "OptInRequired":
					continue
				default:
					controllerutils.LogAwsError(reqLogger, "Failed while trying to create EC2 instance", runErr, runErr)
					return timeoutInstanceID, runErr
				}
			}
			return timeoutInstanceID, awsv1alpha1.ErrFailedAWSTypecast
		}
		return *runResult.Instances[0].InstanceId, nil
	}
	return timeoutInstanceID, awsv1alpha1.ErrCreateEC2Instance
}
func DescribeEC2Instances(reqLogger logr.Logger, client awsclient.Client, instanceID string) (int, error) {
	result, err := client.DescribeInstanceStatus(&ec2.DescribeInstanceStatusInput{
		InstanceIds: aws.StringSlice([]string{instanceID}),
	})
	if err != nil {
		controllerutils.LogAwsError(reqLogger, "New AWS Error while describing EC2 instance", nil, err)
		if aerr, ok := err.(awserr.Error); ok {
			if aerr.Code() == "UnauthorizedOperation" {
				return 401, err
			}
		}
		return 0, err
	}
	if len(result.InstanceStatuses) > 1 {
		return 0, errors.New("more than one EC2 instance found")
	}
	if len(result.InstanceStatuses) == 0 {
		return 0, errors.New("no EC2 instances found")
	}
	return int(*result.InstanceStatuses[0].InstanceState.Code), nil
}
func TerminateEC2Instance(reqLogger logr.Logger, client awsclient.Client, instanceID string) error {
	_, err := client.TerminateInstances(&ec2.TerminateInstancesInput{
		InstanceIds: aws.StringSlice([]string{instanceID}),
	})
	if err != nil {
		controllerutils.LogAwsError(reqLogger, "New AWS Error while terminating EC2 instance", nil, err)
		return err
	}
	return nil
}
func ListEC2InstanceStatus(reqLogger logr.Logger, client awsclient.Client) (*ec2.DescribeInstanceStatusOutput, error) {
	result, err := client.DescribeInstanceStatus(nil)
	if err != nil {
		controllerutils.LogAwsError(reqLogger, "New AWS Error Listing EC2 instance status", nil, err)
		return nil, err
	}
	return result, nil
}
func (r *ReconcileAccount) getDesiredVCPUValue(reqLogger logr.Logger) (float64, error) {
	var err error
	var vCPUQuota float64
	configMap, err := controllerutils.GetOperatorConfigMap(r.Client)
	v, ok := configMap.Data["quota.vcpu"]
	if !ok {
		err = awsv1alpha1.ErrInvalidConfigMap
	}
	if err != nil {
		reqLogger.Info("Failed getting desired vCPU quota from configmap data, defaulting quota to 0")
		return vCPUQuota, err
	}
	vCPUQuota, err = strconv.ParseFloat(v, 64)
	if err != nil {
		reqLogger.Info("Failed converting vCPU quota from configmap string to float64, defaulting quota to 0")
		return vCPUQuota, err
	}
	return vCPUQuota, nil
}
func vCPUQuotaNeedsIncrease(client awsclient.Client, desiredQuota float64) (bool, error) {
	var result *servicequotas.GetServiceQuotaOutput
	retry.DefaultDelay = 3 * time.Second
	retry.DefaultAttempts = uint(5)
	err := retry.Do(
		func() (err error) {
			result, err = client.GetServiceQuota(
				&servicequotas.GetServiceQuotaInput{
					QuotaCode:   aws.String(vCPUQuotaCode),
					ServiceCode: aws.String(vCPUServiceCode),
				},
			)
			return err
		},
		retry.RetryIf(func(err error) bool {
			if aerr, ok := err.(awserr.Error); ok {
				switch aerr.Code() {
				case "AccessDeniedException":
					return true
				case "ServiceException":
					return true
				case "TooManyRequestsException":
					return true
				case "UnrecognizedClientException":
					return true
				}
			}
			return false
		}),
	)
	if result.Quota != nil {
		if *result.Quota.Value < desiredQuota {
			return true, err
		}
	}
	return false, err
}
func setVCPUQuota(client awsclient.Client, desiredQuota float64) (string, error) {
	var result *servicequotas.RequestServiceQuotaIncreaseOutput
	var alreadySubmitted bool
	retry.DefaultDelay = 3 * time.Second
	retry.DefaultAttempts = uint(5)
	err := retry.Do(
		func() (err error) {
			result, err = client.RequestServiceQuotaIncrease(
				&servicequotas.RequestServiceQuotaIncreaseInput{
					DesiredValue: aws.Float64(desiredQuota),
					ServiceCode:  aws.String(vCPUServiceCode),
					QuotaCode:    aws.String(vCPUQuotaCode),
				})
			if err != nil {
				if aerr, ok := err.(awserr.Error); ok {
					if aerr.Code() == "ResourceAlreadyExistsException" {
						alreadySubmitted = true
						return nil
					}
				}
			}
			return err
		},
		retry.RetryIf(func(err error) bool {
			if aerr, ok := err.(awserr.Error); ok {
				switch aerr.Code() {
				case "AccessDeniedException":
					return true
				case "TooManyRequestsException":
					return true
				case "ServiceException":
					return true
				case "UnrecognizedClientException":
					return true
				}
			}
			return false
		}),
	)
	if alreadySubmitted {
		return "RequestAlreadyExists", nil
	}
	if err != nil {
		return "", err
	}
	if (servicequotas.RequestServiceQuotaIncreaseOutput{}) == *result {
		err := fmt.Errorf("returned RequestServiceQuotaIncreaseOutput is nil")
		return "", err
	}
	if (servicequotas.RequestedServiceQuotaChange{}) == *result.RequestedQuota {
		err := fmt.Errorf("returned RequestedServiceQuotasIncreaseOutput field RequestedServiceQuotaChange is nil")
		return "", err
	}
	err = retry.Do(
		func() (err error) {
			var nilString *string
			if result.RequestedQuota.CaseId == nilString {
				err := fmt.Errorf("returned CaseID is nil")
				return err
			}
			return nil
		},
	)
	if err != nil {
		return "", err
	}
	return *result.RequestedQuota.CaseId, nil
}
func checkQuotaRequestHistory(awsClient awsclient.Client, vCPUQuota float64) (string, error) {
	var err error
	var nextToken *string
	var caseID string
	retry.DefaultDelay = 3 * time.Second
	retry.DefaultAttempts = uint(5)
	for {
		var result *servicequotas.ListRequestedServiceQuotaChangeHistoryByQuotaOutput
		var err error
		var submitted bool
		err = retry.Do(
			func() (err error) {
				result, err = awsClient.ListRequestedServiceQuotaChangeHistoryByQuota(
					&servicequotas.ListRequestedServiceQuotaChangeHistoryByQuotaInput{
						NextToken:   nextToken,
						ServiceCode: aws.String(vCPUServiceCode),
						QuotaCode:   aws.String(vCPUQuotaCode),
					},
				)
				return err
			},
			retry.RetryIf(func(err error) bool {
				if aerr, ok := err.(awserr.Error); ok {
					switch aerr.Code() {
					case "AccessDeniedException":
						return true
					case "ServiceException":
						return true
					case "TooManyRequestsException":
						return true
					case "UnrecognizedClientException":
						return true
					}
				}
				return false
			}),
		)
		if err != nil {
			return "", err
		}
		for _, change := range result.RequestedQuotas {
			if changeRequestMatches(change, vCPUQuota) {
				submitted = true
				caseID = *change.CaseId
				break
			}
		}
		if submitted {
			break
		}
		if result.NextToken == nil {
			break
		}
		nextToken = result.NextToken
	}
	return caseID, err
}