<commit_before>import logging
import datetime
import time
import boto3
import base64

logger = logging.getLogger(__name__)

STATUS_FULFILLED = 'fulfilled'
STATUS_PRICE_TOO_LOW = 'price-too-low'
REQUEST_TIMEOUT = 300


class EC2Client(object):

    def __init__(self, timeout=REQUEST_TIMEOUT):
        self._client = boto3.client('ec2')
        self._resource = boto3.resource('ec2')
        self._timeout = timeout  # request timeout in seconds
        self._price_index = 1.5
        self._wait_interval = 5  # seconds

    def _request_spot_instance(self, instance_type, image_id, region_name, network_if, spot_price, key_pair='',
                               user_data='', count=1, duration=0, request_type='one-time'):
        """
        Create a spot instance request
        :return: list of request id-s
        """
        params = dict(DryRun=False,
                      InstanceCount=count,
                      Type=request_type,
                      SpotPrice=str(spot_price),
                      LaunchSpecification={'ImageId': image_id,
                                           'InstanceType': instance_type,
                                           'NetworkInterfaces': network_if,
                                           },
                      ValidUntil=datetime.datetime.now() + datetime.timedelta(minutes=self._timeout/60 + 5)
                      )
        if not duration:
            params.update({'AvailabilityZoneGroup': region_name})
        else:
            params.update({'BlockDurationMinutes': int(duration)})
        if key_pair:
            params['LaunchSpecification'].update({'KeyName': key_pair})
        if user_data:
            params['LaunchSpecification'].update({'UserData': self._encode_user_data(user_data)})

        logger.info('Sending spot request with params: %s', params)
        resp = self._client.request_spot_instances(**params)

        request_ids = [req['SpotInstanceRequestId'] for req in resp['SpotInstanceRequests']]
        logger.debug('Spot requests: %s', request_ids)
        return request_ids

    def _request_spot_fleet(self, instance_type, image_id, region_name, network_if, key_pair='', user_data='', count=3):
        spot_price = self._get_spot_price(instance_type)
        fleet_config = {'LaunchSpecifications':
                        [
                            {'ImageId': image_id,
                             'InstanceType': instance_type,
                             'NetworkInterfaces': network_if,
                             'Placement': {'AvailabilityZone': region_name},
                             },
                        ],
                        'IamFleetRole': 'arn:aws:iam::797456418907:role/aws-ec2-spot-fleet-role',
                        'SpotPrice': str(spot_price['desired']),
                        'TargetCapacity': count,
                        }
        if key_pair:
            fleet_config['LaunchSpecifications'][0].update({'KeyName': key_pair})
        if user_data:
            fleet_config['LaunchSpecifications'][0].update({'UserData': self._encode_user_data(user_data)})

        logger.info('Sending spot fleet request with params: %s', fleet_config)
        resp = self._client.request_spot_fleet(DryRun=False,
                                               SpotFleetRequestConfig=fleet_config)

        request_id = resp['SpotFleetRequestId']
        logger.debug('Spot fleet request: %s', request_id)
        return request_id

    def _get_spot_price(self, instance_type, region_name=''):
        """
        Calculate spot price for bidding
        :return: spot bid price
        """
        logger.info('Calculating spot price for bidding')
        history = self._client.describe_spot_price_history(InstanceTypes=[instance_type], AvailabilityZone=region_name)
        prices = [float(item['SpotPrice']) for item in history['SpotPriceHistory']]
        price_avg = round(sum(prices) / len(prices), 4)
        price_desired = (price_avg + min(prices)) / 4
        price_desired = round(price_desired, 4)
        price = (dict(min=min(prices), max=max(prices), avg=price_avg, desired=price_desired))
        logger.info('Spot bid price: %s', price)
        return price

    def _is_request_fulfilled(self, request_ids):
        """
        Check request status
        """
        resp = self._client.describe_spot_instance_requests(SpotInstanceRequestIds=request_ids)
        for req in resp['SpotInstanceRequests']:
            if req['Status']['Code'] != STATUS_FULFILLED or req['State'] != 'active':
                if req['Status']['Code'] == STATUS_PRICE_TOO_LOW:
                    return False, req['Status']['Code']
                return False, resp
        return True, resp

    def _wait_for_request_done(self, request_ids):
        """
        Wait for spot requests fulfilled
        :param request_ids: spot request id-s
        :return: list of spot instance id-s
        """
        logger.info('Waiting for spot instances...')
        timeout = 0
        status = False<commit_msg>ec2 client: check price history exists (spot is unavailable for some instance types)<commit_after>import logging
import datetime
import time
import boto3
import base64

logger = logging.getLogger(__name__)

STATUS_FULFILLED = 'fulfilled'
STATUS_PRICE_TOO_LOW = 'price-too-low'
REQUEST_TIMEOUT = 300


class EC2Client(object):

    def __init__(self, timeout=REQUEST_TIMEOUT):
        self._client = boto3.client('ec2')
        self._resource = boto3.resource('ec2')
        self._timeout = timeout  # request timeout in seconds
        self._price_index = 1.5
        self._wait_interval = 5  # seconds

    def _request_spot_instance(self, instance_type, image_id, region_name, network_if, spot_price, key_pair='',
                               user_data='', count=1, duration=0, request_type='one-time'):
        """
        Create a spot instance request
        :return: list of request id-s
        """
        params = dict(DryRun=False,
                      InstanceCount=count,
                      Type=request_type,
                      SpotPrice=str(spot_price),
                      LaunchSpecification={'ImageId': image_id,
                                           'InstanceType': instance_type,
                                           'NetworkInterfaces': network_if,
                                           },
                      ValidUntil=datetime.datetime.now() + datetime.timedelta(minutes=self._timeout/60 + 5)
                      )
        if not duration:
            params.update({'AvailabilityZoneGroup': region_name})
        else:
            params.update({'BlockDurationMinutes': int(duration)})
        if key_pair:
            params['LaunchSpecification'].update({'KeyName': key_pair})
        if user_data:
            params['LaunchSpecification'].update({'UserData': self._encode_user_data(user_data)})

        logger.info('Sending spot request with params: %s', params)
        resp = self._client.request_spot_instances(**params)

        request_ids = [req['SpotInstanceRequestId'] for req in resp['SpotInstanceRequests']]
        logger.debug('Spot requests: %s', request_ids)
        return request_ids

    def _request_spot_fleet(self, instance_type, image_id, region_name, network_if, key_pair='', user_data='', count=3):
        spot_price = self._get_spot_price(instance_type)
        fleet_config = {'LaunchSpecifications':
                        [
                            {'ImageId': image_id,
                             'InstanceType': instance_type,
                             'NetworkInterfaces': network_if,
                             'Placement': {'AvailabilityZone': region_name},
                             },
                        ],
                        'IamFleetRole': 'arn:aws:iam::797456418907:role/aws-ec2-spot-fleet-role',
                        'SpotPrice': str(spot_price['desired']),
                        'TargetCapacity': count,
                        }
        if key_pair:
            fleet_config['LaunchSpecifications'][0].update({'KeyName': key_pair})
        if user_data:
            fleet_config['LaunchSpecifications'][0].update({'UserData': self._encode_user_data(user_data)})

        logger.info('Sending spot fleet request with params: %s', fleet_config)
        resp = self._client.request_spot_fleet(DryRun=False,
                                               SpotFleetRequestConfig=fleet_config)

        request_id = resp['SpotFleetRequestId']
        logger.debug('Spot fleet request: %s', request_id)
        return request_id

    def _get_spot_price(self, instance_type, region_name=''):
        """
        Calculate spot price for bidding
        :return: spot bid price
        """
        logger.info('Calculating spot price for bidding')
        history = self._client.describe_spot_price_history(InstanceTypes=[instance_type], AvailabilityZone=region_name)
        if 'SpotPriceHistory' not in history or not history['SpotPriceHistory']:
            raise Exception("Failed getting spot price history for instance type %s", instance_type)
        prices = [float(item['SpotPrice']) for item in history['SpotPriceHistory']]
        price_avg = round(sum(prices) / len(prices), 4)
        price_desired = (price_avg + min(prices)) / 4
        price_desired = round(price_desired, 4)
        price = (dict(min=min(prices), max=max(prices), avg=price_avg, desired=price_desired))
        logger.info('Spot bid price: %s', price)
        return price

    def _is_request_fulfilled(self, request_ids):
        """
        Check request status
        """
        resp = self._client.describe_spot_instance_requests(SpotInstanceRequestIds=request_ids)
        for req in resp['SpotInstanceRequests']:
            if req['Status']['Code'] != STATUS_FULFILLED or req['State'] != 'active':
                if req['Status']['Code'] == STATUS_PRICE_TOO_LOW:
                    return False, req['Status']['Code']
                return False, resp
        return True, resp

    def _wait_for_request_done(self, request_ids):
        """
        Wait for spot requests fulfilled
        :param request_ids: spot request id-s
        :return: list of spot instance id-s
        """
        logger.info('Waiting for spot instances...')
        timeout = 0
        status = False
{'fraction_non_alphanumeric': 0.10360490963564319, 'fraction_numerical': 0.007635063303372958, 'mean_word_length': 2.145288753799392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7513829', 'n_tokens_mistral': 2842, 'n_tokens_neox': 2713, 'n_words': 578}