<commit_before># vim: expandtab:tabstop=4:shiftwidth=4

import subprocess
import os
import json
import re

class AwsUtil(object):
    def __init__(self):
        self.host_type_aliases = {
                'legacy-openshift-broker': ['broker', 'ex-srv'],
                         'openshift-node': ['node', 'ex-node'],
                   'openshift-messagebus': ['msg'],
            'openshift-customer-database': ['mongo'],
                'openshift-website-proxy': ['proxy'],
            'openshift-community-website': ['drupal'],
                         'package-mirror': ['mirror'],
        }

        self.alias_lookup = {}
        for key, values in self.host_type_aliases.iteritems():
            for value in values:
                self.alias_lookup[value] = key

        self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)))
        self.multi_ec2_path = os.path.realpath(os.path.join(self.file_path, '..','inventory','multi_ec2.py'))

    def get_inventory(self,args=[]):
        cmd = [self.multi_ec2_path]

        if args:
            cmd.extend(args)

        env = os.environ

        p = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                         stdout=subprocess.PIPE, env=env)

        out,err = p.communicate()

        if p.returncode != 0:
            raise RuntimeError(err)

        return json.loads(out.strip())

    def get_environments(self):
        pattern = re.compile(r'^tag_environment_(.*)')

        envs = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                envs.append(m.group(1))

        envs.sort()
        return envs

    def get_host_types(self):
        pattern = re.compile(r'^tag_host-type_(.*)')

        host_types = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                host_types.append(m.group(1))

        host_types.sort()
        return host_types

    def get_security_groups(self):
        pattern = re.compile(r'^security_group_(.*)')

        groups = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                groups.append(m.group(1))

        groups.sort()
        return groups

    def build_host_dict_by_env(self, args=[]):
        inv = self.get_inventory(args)

        inst_by_env = {}
        for dns, host in inv['_meta']['hostvars'].items():
            if host['ec2_tag_environment'] not in inst_by_env:
                inst_by_env[host['ec2_tag_environment']] = {}
            host_id = "%s:%s" % (host['ec2_tag_Name'],host['ec2_id'])
            inst_by_env[host['ec2_tag_environment']][host_id] = host

        return inst_by_env

    # Display host_types
    def print_host_types(self):
        host_types = self.get_host_types()
        ht_format_str = "%35s"
        alias_format_str = "%-20s"
        combined_format_str = ht_format_str + "    " + alias_format_str

        print
        print combined_format_str % ('Host Types', 'Aliases')
        print combined_format_str % ('----------', '-------')

        for ht in host_types:
            aliases = []
            if ht in self.host_type_aliases:
                aliases = self.host_type_aliases[ht]<commit_msg>fixed bug in AwsUtil that was causing it to blow up on hosts without an environment set.<commit_after># vim: expandtab:tabstop=4:shiftwidth=4

import subprocess
import os
import json
import re

class AwsUtil(object):
    def __init__(self):
        self.host_type_aliases = {
                'legacy-openshift-broker': ['broker', 'ex-srv'],
                         'openshift-node': ['node', 'ex-node'],
                   'openshift-messagebus': ['msg'],
            'openshift-customer-database': ['mongo'],
                'openshift-website-proxy': ['proxy'],
            'openshift-community-website': ['drupal'],
                         'package-mirror': ['mirror'],
        }

        self.alias_lookup = {}
        for key, values in self.host_type_aliases.iteritems():
            for value in values:
                self.alias_lookup[value] = key

        self.file_path = os.path.join(os.path.dirname(os.path.realpath(__file__)))
        self.multi_ec2_path = os.path.realpath(os.path.join(self.file_path, '..','inventory','multi_ec2.py'))

    def get_inventory(self,args=[]):
        cmd = [self.multi_ec2_path]

        if args:
            cmd.extend(args)

        env = os.environ

        p = subprocess.Popen(cmd, stderr=subprocess.PIPE,
                         stdout=subprocess.PIPE, env=env)

        out,err = p.communicate()

        if p.returncode != 0:
            raise RuntimeError(err)

        return json.loads(out.strip())

    def get_environments(self):
        pattern = re.compile(r'^tag_environment_(.*)')

        envs = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                envs.append(m.group(1))

        envs.sort()
        return envs

    def get_host_types(self):
        pattern = re.compile(r'^tag_host-type_(.*)')

        host_types = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                host_types.append(m.group(1))

        host_types.sort()
        return host_types

    def get_security_groups(self):
        pattern = re.compile(r'^security_group_(.*)')

        groups = []
        inv = self.get_inventory()
        for key in inv.keys():
            m = pattern.match(key)
            if m:
                groups.append(m.group(1))

        groups.sort()
        return groups

    def build_host_dict_by_env(self, args=[]):
        inv = self.get_inventory(args)

        inst_by_env = {}
        for dns, host in inv['_meta']['hostvars'].items():
            # If you don't have an environment tag, we're going to ignore you
            if not host.has_key('ec2_tag_environment'):
                continue

            if host['ec2_tag_environment'] not in inst_by_env:
                inst_by_env[host['ec2_tag_environment']] = {}
            host_id = "%s:%s" % (host['ec2_tag_Name'],host['ec2_id'])
            inst_by_env[host['ec2_tag_environment']][host_id] = host

        return inst_by_env

    # Display host_types
    def print_host_types(self):
        host_types = self.get_host_types()
        ht_format_str = "%35s"
        alias_format_str = "%-20s"
        combined_format_str = ht_format_str + "    " + alias_format_str

        print
        print combined_format_str % ('Host Types', 'Aliases')
        print combined_format_str % ('----------', '-------')

        for ht in host_types:
            aliases = []
            if ht in self.host_type_aliases:
                aliases = self.host_type_aliases[ht]
{'fraction_non_alphanumeric': 0.1300565463244889, 'fraction_numerical': 0.0053646512976656515, 'mean_word_length': 1.9093209616195699, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7269105', 'n_tokens_mistral': 2210, 'n_tokens_neox': 2107, 'n_words': 401}