<commit_before>import logging
import urlparse

from dateutil.parser import parse as dateutil_parse
import requests
import requests.exceptions

from autoscaler.config import Config
from autoscaler.autoscaling_groups import AutoScalingGroup
import autoscaler.errors as errors
import autoscaler.utils as utils

logger = logging.getLogger(__name__)

_DEFAULT_TAG_VALUE = 'default'


class AzureClient(object):
    def __init__(self, region='us-south-central'):
        self.region = region

    def _url(self, path):
        return urlparse.urljoin('http://azure-{}.{}/'.format(self.region, Config.NAMESPACE), path)

    def list_instances(self):
        try:
            req = requests.get(self._url('instances'))
            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def create_instances(self, instance_type, number, tags):
        url = self._url('instances')
        data = {
            'instance_type': instance_type,
            'number': number,
            'tags': tags
        }

        logger.debug('POST %s (data=%s)', url, data)

        try:
            req = requests.post(url, json=data)

            logger.debug('response: %s', req.text)

            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def delete_instances(self, instance_id):
        try:
            req = requests.delete(self._url('instances/{}'.format(instance_id)))
            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def get_tags(self):
        req = requests.get(self._url('allowed_launch_parameters'))
        req.raise_for_status()
        return req.json()


class AzureGroups(object):
    def __init__(self, regions):
        self.regions = regions

    def get_all_groups(self, kube_nodes):
        groups = []
        for region in self.regions:
            client = AzureClient(region)

            tags = client.get_tags()
            instances = client.list_instances()

            if 'error' in instances:
                logger.warn('Failed to get instances in %s. Skipping.', region)
                continue

            for tag_set in tags['parameter_sets']:
                instance_type = tag_set['instance_type']
                tags = tag_set['tags']
                group_instances = [
                    AzureInstance(inst) for inst in instances['instances']
                    if inst['instance_type'] == instance_type and all(inst['tags'][k] == tags[k] for k in tags.keys())]
                for tag in tag_set.get('arbitrary_value_tags', []):
                    tags[tag] = _DEFAULT_TAG_VALUE
                group = AzureGroup(client, instance_type, tags, group_instances, kube_nodes)
                groups.append(group)

        return groups


class AzureGroup(AutoScalingGroup):
    provider = 'azure'

    def __init__(self, client, instance_type, tags, instances, kube_nodes):
        self.client = client
        self.instance_type = instance_type
        self.tags = tags
        self.name = instance_type
        self.desired_capacity = len(instances)

        self.region = client.region

        self.selectors = dict(tags)<commit_msg>more error handling<commit_after>import logging
import urlparse

from dateutil.parser import parse as dateutil_parse
import requests
import requests.exceptions

from autoscaler.config import Config
from autoscaler.autoscaling_groups import AutoScalingGroup
import autoscaler.errors as errors
import autoscaler.utils as utils

logger = logging.getLogger(__name__)

_DEFAULT_TAG_VALUE = 'default'


class AzureClient(object):
    def __init__(self, region='us-south-central'):
        self.region = region

    def _url(self, path):
        return urlparse.urljoin('http://azure-{}.{}/'.format(self.region, Config.NAMESPACE), path)

    def list_instances(self):
        try:
            req = requests.get(self._url('instances'))
            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def create_instances(self, instance_type, number, tags):
        url = self._url('instances')
        data = {
            'instance_type': instance_type,
            'number': number,
            'tags': tags
        }

        logger.debug('POST %s (data=%s)', url, data)

        try:
            req = requests.post(url, json=data)

            logger.debug('response: %s', req.text)

            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def delete_instances(self, instance_id):
        try:
            req = requests.delete(self._url('instances/{}'.format(instance_id)))
            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }

        return req.json()

    def get_tags(self):
        try:
            req = requests.get(self._url('allowed_launch_parameters'))
            req.raise_for_status()
        except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
            errors.capture_exception()
            return {
                'error': str(e)
            }
        return req.json()


class AzureGroups(object):
    def __init__(self, regions):
        self.regions = regions

    def get_all_groups(self, kube_nodes):
        groups = []
        for region in self.regions:
            client = AzureClient(region)

            tags = client.get_tags()
            instances = client.list_instances()

            if 'error' in instances or 'error' in tags:
                logger.warn('Failed to get instances in %s. Skipping.', region)
                continue

            for tag_set in tags['parameter_sets']:
                instance_type = tag_set['instance_type']
                tags = tag_set['tags']
                group_instances = [
                    AzureInstance(inst) for inst in instances['instances']
                    if inst['instance_type'] == instance_type and all(inst['tags'][k] == tags[k] for k in tags.keys())]
                for tag in tag_set.get('arbitrary_value_tags', []):
                    tags[tag] = _DEFAULT_TAG_VALUE
                group = AzureGroup(client, instance_type, tags, group_instances, kube_nodes)
                groups.append(group)

        return groups


class AzureGroup(AutoScalingGroup):
    provider = 'azure'

    def __init__(self, client, instance_type, tags, instances, kube_nodes):
        self.client = client
        self.instance_type = instance_type
        self.tags = tags
        self.name = instance_type
        self.desired_capacity = len(instances)

        self.region = client.region

        self.selectors = dict(tags)
{'fraction_non_alphanumeric': 0.10110439127004996, 'fraction_numerical': 0.0, 'mean_word_length': 2.133031301482702, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1158635', 'n_tokens_mistral': 2127, 'n_tokens_neox': 2048, 'n_words': 467}