<commit_before>import json
import logging
import os
import time
from string import Template

from libsubmit.error import *
from libsubmit.providers.aws.template import template_string
from libsubmit.providers.provider_base import ExecutionProvider

logger = logging.getLogger(__name__)

try:
    import boto3
    from botocore.exceptions import ClientError

except ImportError:
    _boto_enabled = False
else:
    _boto_enabled = True

translate_table = {
    'pending': 'PENDING',
    'running': 'RUNNING',
    'terminated': 'COMPLETED',
    'shutting-down': 'COMPLETED',  # (configuring),
    'stopping': 'COMPLETED',  # We shouldn't really see this state
    'stopped': 'COMPLETED',  # We shouldn't really see this state
}


class EC2Provider(ExecutionProvider):
    '''
    Here's a sample config for the EC2 provider:

    .. code-block:: python

         { "auth" : { # Definition of authentication method for AWS. One of 3 methods are required to authenticate
                      # with AWS: keyfile, profile or env_variables. If keyfile or profile is not set, Boto3 will
                      # look for the following env variables:
                      # AWS_ACCESS_KEY_ID : The access key for your AWS account.
                      # AWS_SECRET_ACCESS_KEY : The secret key for your AWS account.
                      # AWS_SESSION_TOKEN : The session key for your AWS account.

              "keyfile"    : #{Description: Path to json file that contains 'AWSAccessKeyId' and 'AWSSecretKey'
                             # Type: String,
                             # Required: False},

              "profile"    : #{Description: Specify the profile to be used from the standard aws config file
                             # ~/.aws/config.
                             # Type: String,
                             # Expected: "default", # Use the 'default' aws profile
                             # Required: False},

            },

           "execution" : { # Definition of all execution aspects of a site

              "executor"   : #{Description: Define the executor used as task executor,
                             # Type: String,
                             # Expected: "ipp",
                             # Required: True},

              "provider"   : #{Description: The provider name, in this case ec2
                             # Type: String,
                             # Expected: "aws",
                             # Required:  True },

              "block" : { # Definition of a block

                  "nodes"      : #{Description: # of nodes to provision per block
                                 # Type: Integer,
                                 # Default: 1},

                  "taskBlocks" : #{Description: # of workers to launch per block
                                 # as either an number or as a bash expression.
                                 # for eg, "1" , "$(($CORES / 2))"
                                 # Type: String,
                                 # Default: "1" },

                  "walltime"  :  #{Description: Walltime requested per block in HH:MM:SS
                                 # Type: String,
                                 # Default: "00:20:00" },

                  "initBlocks" : #{Description: # of blocks to provision at the start of
                                 # the DFK
                                 # Type: Integer
                                 # Default: ?
                                 # Required:    },

                  "minBlocks" :  #{Description: Minimum # of blocks outstanding at any time
                                 # WARNING :: Not Implemented
                                 # Type: Integer
                                 # Default: 0 },

                  "maxBlocks" :  #{Description: Maximum # Of blocks outstanding at any time
                                 # WARNING :: Not Implemented
                                 # Type: Integer
                                 # Default: ? },

                  "options"   : {  # Scheduler specific options


                      "instanceType" : #{Description: Instance type t2.small|t2...
                                       # Type: String,
                                       # Required: False
                                       # Default: t2.small },

                      "imageId"      : #{"Description: The ID of the AMI
                                       # Type: String,
                                       # Required: True },

                      "region"       : #{"Description: AWS region to launch machines in
                                       # in the submit script to the scheduler
                                       # Type: String,
                                       # Default: 'us-east-2',
                                       # Required: False },

                      "keyName"      : #{"Description: Name of the AWS private key (.pem file)
                                       # that is usually generated on the console to allow ssh access
                                       # to the EC2 instances, mostly for debugging.
                                       # in the submit script to the scheduler
                                       # Type: String,
                                       # Required: True },

                      "spotMaxBid"   : #{"Description: If requesting spot market machines, specify
                                       # the max Bid price.
                                       # Type: Float,
                                       # Required: False },
                      "overrides"    : #{"Description : String to append to the Userdata script executed
                                       # in the cloudinit phase of instance initialization
                                       # Type : String,
                                       # Default : ''
                                       # Required : False },

                  }
              }
            }
         }
    '''

    def __repr__(self):
        return "<EC2 Execution Provider for site:{0}>".format(self.sitename)

    def __init__(self, config, channel=None):
        ''' Initialize the EC2Provider class

        Args:
             - Config (dict): Dictionary with all the config options.

        KWargs:
             - Channel (None): A channel is not required for EC2.

        '''

        self.channel = channel
        if not _boto_enabled:
            raise OptionalModuleMissing(['boto3'], "AWS Provider requires boto3 module.")

        self.config = config
        self.sitename = config['site']
        self.current_blocksize = 0
        self.resources = {}
        # atexit.register(self.goodbye)
        self.config = config
        options = self.config["execution"]["block"]["options"]
        logger.warn("Options %s", options)
        self.overrides = options.get("overrides", '')
        self.instance_type = options.get("instanceType", "t2.small")
        self.image_id = options["imageId"]
        self.key_name = options["keyName"]
        self.region = options.get("region", 'us-east-2')
        self.max_nodes = (
            self.config["execution"]["block"].get("maxBlocks", 1) *
            self.config["execution"]["block"].get("nodes", 1)
        )

        self.spot_max_bid = options.get("spotMaxBid", 0)

        try:
            self.initialize_boto_client()
        except Exception as e:
            logger.error("Site:[{0}] Failed to initialize".format(self))
            raise e

        try:
            self.statefile = self.config["execution"]["block"]["options"].get(
                "stateFile", '.ec2site_{0}.json'.format(self.sitename)
            )
            self.read_state_file(self.statefile)

        except Exception as e:
            self.create_vpc().id
            logger.info("No State File. Cannot load previous options. Creating new infrastructure")
            self.write_state_file()
<commit_msg>Moving infra creation for #46<commit_after>import json
import logging
import os
import time
from string import Template

from libsubmit.error import *
from libsubmit.providers.aws.template import template_string
from libsubmit.providers.provider_base import ExecutionProvider

logger = logging.getLogger(__name__)

try:
    import boto3
    from botocore.exceptions import ClientError

except ImportError:
    _boto_enabled = False
else:
    _boto_enabled = True

translate_table = {
    'pending': 'PENDING',
    'running': 'RUNNING',
    'terminated': 'COMPLETED',
    'shutting-down': 'COMPLETED',  # (configuring),
    'stopping': 'COMPLETED',  # We shouldn't really see this state
    'stopped': 'COMPLETED',  # We shouldn't really see this state
}


class EC2Provider(ExecutionProvider):
    '''
    Here's a sample config for the EC2 provider:

    .. code-block:: python

         { "auth" : { # Definition of authentication method for AWS. One of 3 methods are required to authenticate
                      # with AWS: keyfile, profile or env_variables. If keyfile or profile is not set, Boto3 will
                      # look for the following env variables:
                      # AWS_ACCESS_KEY_ID : The access key for your AWS account.
                      # AWS_SECRET_ACCESS_KEY : The secret key for your AWS account.
                      # AWS_SESSION_TOKEN : The session key for your AWS account.

              "keyfile"    : #{Description: Path to json file that contains 'AWSAccessKeyId' and 'AWSSecretKey'
                             # Type: String,
                             # Required: False},

              "profile"    : #{Description: Specify the profile to be used from the standard aws config file
                             # ~/.aws/config.
                             # Type: String,
                             # Expected: "default", # Use the 'default' aws profile
                             # Required: False},

            },

           "execution" : { # Definition of all execution aspects of a site

              "executor"   : #{Description: Define the executor used as task executor,
                             # Type: String,
                             # Expected: "ipp",
                             # Required: True},

              "provider"   : #{Description: The provider name, in this case ec2
                             # Type: String,
                             # Expected: "aws",
                             # Required:  True },

              "block" : { # Definition of a block

                  "nodes"      : #{Description: # of nodes to provision per block
                                 # Type: Integer,
                                 # Default: 1},

                  "taskBlocks" : #{Description: # of workers to launch per block
                                 # as either an number or as a bash expression.
                                 # for eg, "1" , "$(($CORES / 2))"
                                 # Type: String,
                                 # Default: "1" },

                  "walltime"  :  #{Description: Walltime requested per block in HH:MM:SS
                                 # Type: String,
                                 # Default: "00:20:00" },

                  "initBlocks" : #{Description: # of blocks to provision at the start of
                                 # the DFK
                                 # Type: Integer
                                 # Default: ?
                                 # Required:    },

                  "minBlocks" :  #{Description: Minimum # of blocks outstanding at any time
                                 # WARNING :: Not Implemented
                                 # Type: Integer
                                 # Default: 0 },

                  "maxBlocks" :  #{Description: Maximum # Of blocks outstanding at any time
                                 # WARNING :: Not Implemented
                                 # Type: Integer
                                 # Default: ? },

                  "options"   : {  # Scheduler specific options


                      "instanceType" : #{Description: Instance type t2.small|t2...
                                       # Type: String,
                                       # Required: False
                                       # Default: t2.small },

                      "imageId"      : #{"Description: The ID of the AMI
                                       # Type: String,
                                       # Required: True },

                      "region"       : #{"Description: AWS region to launch machines in
                                       # in the submit script to the scheduler
                                       # Type: String,
                                       # Default: 'us-east-2',
                                       # Required: False },

                      "keyName"      : #{"Description: Name of the AWS private key (.pem file)
                                       # that is usually generated on the console to allow ssh access
                                       # to the EC2 instances, mostly for debugging.
                                       # in the submit script to the scheduler
                                       # Type: String,
                                       # Required: True },

                      "spotMaxBid"   : #{"Description: If requesting spot market machines, specify
                                       # the max Bid price.
                                       # Type: Float,
                                       # Required: False },
                      "overrides"    : #{"Description : String to append to the Userdata script executed
                                       # in the cloudinit phase of instance initialization
                                       # Type : String,
                                       # Default : ''
                                       # Required : False },

                  }
              }
            }
         }
    '''

    def __repr__(self):
        return "<EC2 Execution Provider for site:{0}>".format(self.sitename)

    def __init__(self, config, channel=None):
        ''' Initialize the EC2Provider class

        Args:
             - Config (dict): Dictionary with all the config options.

        KWargs:
             - Channel (None): A channel is not required for EC2.

        '''

        self.channel = channel
        if not _boto_enabled:
            raise OptionalModuleMissing(['boto3'], "AWS Provider requires boto3 module.")

        self.config = config
        self.sitename = config['site']
        self.current_blocksize = 0
        self.resources = {}
        # atexit.register(self.goodbye)
        self.config = config
        options = self.config["execution"]["block"]["options"]
        logger.warn("Options %s", options)
        self.overrides = options.get("overrides", '')
        self.instance_type = options.get("instanceType", "t2.small")
        self.image_id = options["imageId"]
        self.key_name = options["keyName"]
        self.region = options.get("region", 'us-east-2')
        self.max_nodes = (
            self.config["execution"]["block"].get("maxBlocks", 1) *
            self.config["execution"]["block"].get("nodes", 1)
        )

        self.spot_max_bid = options.get("spotMaxBid", 0)

        try:
            self.initialize_boto_client()
        except Exception as e:
            logger.error("Site:[{0}] Failed to initialize".format(self))
            raise e

        state_file_exists = False
        try:
            self.statefile = self.config["execution"]["block"]["options"].get(
                "stateFile", '.ec2site_{0}.json'.format(self.sitename)
            )
            self.read_state_file(self.statefile)
            state_file_exists = True
        except Exception as e:
            logger.info("No State File. Cannot load previous options. Creating new infrastructure")

        if not state_file_exists:
            try :
                self.create_vpc().id
            except Exception as e:
                logger.info("Failed to create ec2 infrastructure : {0}".format(e))
                raise
            else:
                # If infrastructure creation worked write state file
                self.write_state_file()


{'fraction_non_alphanumeric': 0.08533089356511131, 'fraction_numerical': 0.004757548032936871, 'mean_word_length': 1.1996243627582506, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 120, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2205453', 'n_tokens_mistral': 3854, 'n_tokens_neox': 3664, 'n_words': 1151}