<commit_before>#!/usr/bin/env python

import argparse
import logging
import os
import sys
import time

import boto3

from ecsUpdate import ApplyECS

logging.basicConfig(format="%(asctime)s %(levelname)s [%(threadName)s] - %(message)s", stream=sys.stdout,
                    level=logging.INFO)


class ApplyCF:
    def __init__(self):
        self.success_status = ['CREATE_COMPLETE','UPDATE_COMPLETE']
        self.failed_status = ['CREATE_FAILED', 'ROLLBACK_IN_PROGRESS','ROLLBACK_FAILED','ROLLBACK_COMPLETE','UPDATE_ROLLBACK_IN_PROGRESS','UPDATE_ROLLBACK_FAILED','UPDATE_ROLLBACK_COMPLETE']

    def catfile(self, fn):
      with open(fn) as f:
        print f.read()

    def main(self, dir_, cf_params):

        file_dir = os.path.dirname(os.path.realpath(__file__))
        job_path = os.path.join(file_dir, dir_)

        self.load(job_path, cf_params)

    def load(self, job_path, cf_params):
        env = cf_params['env']
        cluster = cf_params['cluster']
        region = cf_params['region']
        elb_name = 'ecs-elb-' + cluster
        cf_client = boto3.client('cloudformation', region_name=region)

        self.extract_common_ecs_params(cf_params, cluster, elb_name, env, region)

        for subdir, dirs, files in os.walk(job_path):
            for fn in files:
                filename = os.path.join(subdir, fn)

                # Skip non-cf files
                ext = filename.split('.')[-1]
                if ext != 'template' and ext != 'yml':
                    continue
                name = filename.split('/')[-1].split('.')[0]
                cf_params['name'] = name
                logging.info("Processing CloudFormation Template: " + filename)
                parameters = [{'ParameterKey': 'name', 'ParameterValue': name}]

                if name is None or name in filename:
                    with open(filename, 'r') as f_h:
                        try:
                            cf_template = f_h.read()
                        except:
                            logging.exception("Error reading file %s " % filename)
                            self.catfile(filename)
                            raise
                        validate_response = {}
                        try:
                            validate_response = cf_client.validate_template(TemplateBody=cf_template)
                            logging.info("CloudFormation template validated")
                        except Exception as e:
                            logging.error("Error validating file: %s" % filename)
                            logging.error(validate_response)
                            logging.exception(e)
                            sys.exit(1)

                        try:
                            for cf_parameter in validate_response['Parameters']:
                                if cf_parameter['ParameterKey'] not in cf_params:
                                    logging.warning("Parameter: %s is specified by template in %s but not specified after --cfparams" % (filename, cf_parameter['ParameterKey']))
                                parameters.append({'ParameterKey': cf_parameter['ParameterKey'],
                                                   'ParameterValue': cf_params[cf_parameter['ParameterKey']]})
                            service_name = "%s-%s-%s" % (env, name, cluster)
                            existing_stacks = cf_client.list_stacks()
                            existing_stack_id = None
                            cf_command = cf_client.create_stack
                            for stack in existing_stacks['StackSummaries']:
                                if stack['StackName'] == service_name and stack['StackStatus'] != 'DELETE_COMPLETE':
                                    existing_stack_id = stack['StackId']
                                    cf_command = cf_client.update_stack
                                    break
                            logging.info("Updating CloudFormation Stack: " + service_name)
                            cf_response = cf_command(StackName=service_name, TemplateBody=cf_template, Parameters=parameters, Capabilities=["CAPABILITY_IAM"])
                            creating_stack_id = cf_response['StackId']
                            stack_status = self.wait_for_stack_creation(cf_client, creating_stack_id, service_name)
                            if existing_stack_id is not None:
                                print("Registering new task defintion to restart services/deploy new containers")
                                self.restart_tasks(cf_client, existing_stack_id, region, cf_params['cluster'])
                        except Exception as e:
                            logging.error("Error executing CloudFormation: %s" % filename)
                            logging.exception(e)
                            sys.exit(1)

    def restart_tasks(self, cf_client, existing_stack_id, region, cluster):
        stack_resources = cf_client.describe_stack_resources(StackName=existing_stack_id)
        for resource in stack_resources['StackResources']:
            if resource['ResourceType'] == "AWS::ECS::TaskDefinition":
                task_definition_arn = resource['PhysicalResourceId']
            if resource['ResourceType'] == "AWS::ECS::Service":
                service_arn = resource['PhysicalResourceId']
<commit_msg>Dont update priority if already set on the stack.  Don't return failure if there is nothing to update.<commit_after>#!/usr/bin/env python

import argparse
import logging
import os
import sys
import time
import botocore
import boto3

from ecsUpdate import ApplyECS

logging.basicConfig(format="%(asctime)s %(levelname)s [%(threadName)s] - %(message)s", stream=sys.stdout,
                    level=logging.INFO)


class ApplyCF:
    def __init__(self):
        self.success_status = ['CREATE_COMPLETE','UPDATE_COMPLETE']
        self.failed_status = ['CREATE_FAILED', 'ROLLBACK_IN_PROGRESS','ROLLBACK_FAILED','ROLLBACK_COMPLETE','UPDATE_ROLLBACK_IN_PROGRESS','UPDATE_ROLLBACK_FAILED','UPDATE_ROLLBACK_COMPLETE']

    def catfile(self, fn):
      with open(fn) as f:
        print f.read()

    def main(self, dir_, cf_params):

        file_dir = os.path.dirname(os.path.realpath(__file__))
        job_path = os.path.join(file_dir, dir_)

        self.load(job_path, cf_params)

    def load(self, job_path, cf_params):
        env = cf_params['env']
        cluster = cf_params['cluster']
        region = cf_params['region']
        elb_name = 'ecs-elb-' + cluster
        cf_client = boto3.client('cloudformation', region_name=region)

        self.extract_common_ecs_params(cf_params, cluster, elb_name, env, region)

        for subdir, dirs, files in os.walk(job_path):
            for fn in files:
                filename = os.path.join(subdir, fn)

                # Skip non-cf files
                ext = filename.split('.')[-1]
                if ext != 'template' and ext != 'yml':
                    continue
                name = filename.split('/')[-1].split('.')[0]
                cf_params['name'] = name
                logging.info("Processing CloudFormation Template: " + filename)
                parameters = [{'ParameterKey': 'name', 'ParameterValue': name}]

                if name is None or name in filename:
                    with open(filename, 'r') as f_h:
                        try:
                            cf_template = f_h.read()
                        except:
                            logging.exception("Error reading file %s " % filename)
                            self.catfile(filename)
                            raise
                        validate_response = {}
                        try:
                            validate_response = cf_client.validate_template(TemplateBody=cf_template)
                            logging.info("CloudFormation template validated")
                        except Exception as e:
                            logging.error("Error validating file: %s" % filename)
                            logging.error(validate_response)
                            logging.exception(e)
                            sys.exit(1)

                        try:

                            service_name = "%s-%s-%s" % (env, name, cluster)
                            existing_stacks = cf_client.list_stacks()
                            existing_stack_id = None
                            cf_command = cf_client.create_stack
                            for stack in existing_stacks['StackSummaries']:
                                if stack['StackName'] == service_name and stack['StackStatus'] != 'DELETE_COMPLETE':
                                    existing_stack_id = stack['StackId']
                                    cf_command = cf_client.update_stack
                                    cf_params.pop('priority')  # don't need to set priority if this is an update operation
                                    break
                            for cf_parameter in validate_response['Parameters']:
                                if cf_parameter['ParameterKey'] not in cf_params:
                                    logging.warning("Parameter: %s is specified by template in %s but not specified after --cfparams" % (filename, cf_parameter['ParameterKey']))
                                    if existing_stack_id is not None:
                                        parameters.append({'ParameterKey': cf_parameter['ParameterKey'], 'UsePreviousValue': True})
                                else:
                                    parameters.append({'ParameterKey': cf_parameter['ParameterKey'], 'ParameterValue': cf_params[cf_parameter['ParameterKey']]})
                            logging.info("Updating CloudFormation Stack: " + service_name)
                            try:
                                cf_response = cf_command(StackName=service_name, TemplateBody=cf_template, Parameters=parameters, Capabilities=["CAPABILITY_IAM"])
                                creating_stack_id = cf_response['StackId']
                                stack_status = self.wait_for_stack_creation(cf_client, creating_stack_id, service_name)
                            except botocore.exceptions.ClientError as e:
                                if e.response["Error"]["Message"] == 'No updates are to be performed.':
                                    logging.info("No updates to be performed, CF update succeeded.  Restarting tasks.")
                                else:
                                    raise
                            if existing_stack_id is not None:
                                print("Registering new task defintion to restart services/deploy new containers")
                                self.restart_tasks(cf_client, existing_stack_id, region, cf_params['cluster'])
                        except Exception as e:
                            logging.error("Error executing CloudFormation: %s" % filename)
                            logging.exception(e)
                            sys.exit(1)

    def restart_tasks(self, cf_client, existing_stack_id, region, cluster):
        stack_resources = cf_client.describe_stack_resources(StackName=existing_stack_id)
        for resource in stack_resources['StackResources']:
            if resource['ResourceType'] == "AWS::ECS::TaskDefinition":
                task_definition_arn = resource['PhysicalResourceId']
            if resource['ResourceType'] == "AWS::ECS::Service":
                service_arn = resource['PhysicalResourceId']

{'fraction_non_alphanumeric': 0.09011652999568408, 'fraction_numerical': 0.0012084592145015106, 'mean_word_length': 1.6248300860897145, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1485413', 'n_tokens_mistral': 2889, 'n_tokens_neox': 2787, 'n_words': 618}