<commit_before>import sys


class Cloud:
    ROLE_CONTROLLER = 'controller'
    ROLE_UCSM = 'ucsm'
    ROLE_NETWORK = 'network'
    ROLE_COMPUTE = 'compute'
    ROLE_MEDIATOR = 'mediator'

    def __init__(self, cloud, user, password, tenant, end_point=None):
        self._openstack_bin = 'openstack'
        self._neutron_bin = 'neutron'

        self._create_server_cmd = self._openstack_bin + ' server create -f shell '
        self._create_subnet_cmd = self._neutron_bin + ' subnet-create -f shell '
        self._create_network_cmd = self._openstack_bin + ' network create -f shell '
        self._create_port_cmd = self._neutron_bin + ' port-create -f shell '

        self._delete_server_cmd = self._openstack_bin + ' server delete '
        self._delete_keypair_cmd = self._openstack_bin + ' keypair delete '
        self._delete_network_cmd = self._openstack_bin + ' network delete '
        self._delete_subnet_cmd = self._neutron_bin + ' subnet-delete '
        self._delete_port_cmd = self._neutron_bin + ' port-delete '
        self._delete_user_cmd = self._openstack_bin + ' user delete '
        self._delete_project_cmd = self._openstack_bin + ' project delete '

        self._show_server_cmd = self._openstack_bin + ' server show -f shell '
        self._show_subnet_cmd = self._neutron_bin + ' subnet-show -f shell '
        
        self._list_network_cmd = self._openstack_bin + ' network list -f json'
        self._list_subnet_cmd = self._openstack_bin + ' subnet list -f json'
        self._list_router_cmd = self._openstack_bin + ' router list -f json'
        self._list_port_cmd = self._openstack_bin + ' port list -f json'
        self._list_server_cmd = self._openstack_bin + ' server list -f json --all'
        self._list_keypair_cmd = self._openstack_bin + ' keypair list -f json'
        self._list_image_cmd = self._openstack_bin + ' image list -f json'
        self._list_user_cmd = self._openstack_bin + ' user list -f json'
        self._list_project_cmd = self._openstack_bin + ' project list -f json'

        self._fip_network = 'Default Value Set In Cloud.__init__()'
        self._provider_physical_network = 'Default Value Set In Cloud.__init__()'
        self._openstack_version = 'Default Value Set In Cloud.__init__()'

        self.name = cloud
        self.user = user
        self.tenant = tenant
        self.password = password
        self.info = {'controller': [], 'ucsm': [], 'network': [], 'compute': []}
        self.mac_2_ip = {}
        self.hostname_2_ip = {}
        self.end_point = end_point
        self._dns = '192.168.3.11'
        self._unique_pattern_in_name = 'sqe-test'
        self._instance_counter = 0  # this counter is used to count how many instances are created via this class



    def __repr__(self):
        return '--os-username {u} --os-tenant-name {t} --os-password {p} --os-auth-url {a}'.format(u=self.user, t=self.tenant, p=self.password, a=self.get_end_point())

    def get_end_point(self):
        return self.end_point

    def get_first(self, role, parameter):
        """
            :param role: controller, network, compute, ucsm
            :param parameter: ip, mac, hostname
            :return: the first value for given parameter of given role
        """
        values = self.get(role=role, parameter=parameter)
        if values:
            return values[0]
        else:
            return 'NoValueFor' + role + parameter

    def add_server(self, config_name, server):
        """ Set all parameters for the given server
        :param config_name:
        :param server:
        """
        if config_name.startswith('aio'):
            role = self.ROLE_CONTROLLER
        else:
            role = None
            for x in self.info.keys():
                if x in config_name:
                    role = x
                    break
            if role is None:
                raise RuntimeError('Failed to deduce cloud role for server {0}'.format(server))

        self.hostname_2_ip[server.hostname] = server.ip
        self.mac_2_ip[server.ip_mac] = server.ip

        _info = {'ip': server.ip, 'mac': server.ip_mac, 'username': server.username, 'hostname': server.hostname, 'password': <PASSWORD>}
        self.info[role].append(_info)

    def add_service_end_point(self, service, url, end_point):
        self.service_end_points[service].update({url: end_point})

    def get_service_end_point(self, service, url):
        return self.service_end_points[service][url]

    def create_open_rc(self):
        """ Creates open_rc for the given cloud"""
        open_rc = """
export OS_USERNAME={user}
export OS_TENANT_NAME={tenant}
export OS_PASSWORD={password}
export OS_AUTH_URL={end_point}
"""
        return open_rc.format(user=self.user, tenant=self.tenant, password=<PASSWORD>, end_point=self.get_end_point())

    def cmd(self, cmd):
        import json
        import subprocess
        import sys

        sys.stdout.writelines("Call command: " + cmd)
        venv = '. ' + sys.prefix + '/bin/activate && ' if hasattr(sys, 'real_prefix') else ''
        ans = subprocess.check_output('{venv}{cmd} {creds}'.format(venv=venv, cmd=cmd, creds=self), shell=True)
        sys.stdout.writelines("Output:" + str(ans))
        if '-f csv' in cmd:
            return self._process_csv_output(ans)
        elif '-f json' in cmd:
            return json.loads(ans)
        elif '-f shell' in cmd:
            return self._process_shell_output(ans)
        else:
            return ans

    @staticmethod
    def _filter(minus_c, flt):
        return '{c} {g}'.format(c='-c ' + minus_c if minus_c else '', g=' | grep ' + flt if flt else '')

    @staticmethod
    def _process_csv_output(answer):
        output_list = []
        if answer:
            lines = answer.split('\n')
            keys = []
            for i, line in enumerate(lines):
                line = line.strip('\r')
                if i == 0:  # it's line of keys like: "ID","Name" or "ID","Name","Subnets"
                    keys = map(lambda x: x.strip('" '), line.split(','))
                else:
                    output_list.append(dict(zip(keys, map(lambda x: x.strip('" '), line.split(',')))))
        return output_list

    @staticmethod
    def _process_shell_output(answer):
        output_dict = {}
        if answer:
            lines = answer.split('\n')
            for line in lines:
                if '=' not in line:  # might be lines not in form a=b
                    continue
                line = line.strip('\r')
                key, value = line.split('=', 1)
                output_dict[key.strip('" ')] = value.strip('" ')
        return output_dict

    @staticmethod
    def _names_from_answer(ans):
        return filter(lambda x: x not in ['name', '|', '+--------+'], ans.split())

    def get_fip_network(self):
        if not self._fip_network:
            ans = self.cmd(self._neutron_bin + ' net-list --router:external=True -c name')
            net_names = self._names_from_answer(ans)
            if not net_names:
                return
            self._fip_network = net_names[0]
            ans = self.cmd(self._neutron_bin + ' net-list -c provider:physical_network --name {0}'.format(self._fip_network))
            self._provider_physical_network = filter(lambda x: x not in ['provider:physical_network', '|', '+---------------------------+'], ans.split())[0]
        return self._fip_network

    @staticmethod
    def _parse_cli_output(output_lines):
        """Borrowed from tempest-lib. Parse single table from cli output.
        :param output_lines:
        :returns: dict with list of column names in 'headers' key and rows in 'values' key.
        """

        if not isinstance(output_lines, list):
            output_lines = output_lines.split('\n')

        if not output_lines[-1]:
            output_lines = output_lines[:-1]  # skip last line if empty (just newline at the end)

        columns = Cloud._table_columns(output_lines[2])
        is_2_values = len(columns) == 2

        def line_to_row(input_line):
            return [input_line[col[0]:col[1]].strip() for col in columns]

        headers = None
        d2 = {}
        output_list = []
        for line in output_lines[3:-1]:
            row = line_to_row(line)
            if is_2_values:
                d2[row[0]] = row[1]
            else:
                headers = headers or line_to_row(output_lines[1])
                output_list.append({headers[i]: row[i] for i in xrange(len(columns))})

        return output_list if output_list else [d2]

    @staticmethod
    def _table_columns(first_table_row):
        """Borrowed from tempest-lib. Find column ranges in output line.
        :returns: list of tuples (start,end) for each column detected by plus (+) characters in delimiter line.
        """
        positions = []
        start = 1  # there is '+' at 0
        while start < len(first_table_row):
            end = first_table_row.find('+', start)
            if end == -1:
                break
            positions.append((start, end))
            start = end + 1
        return positions

    @staticmethod
    def get_cidrs4(class_a, how_many):
        import random
        from netaddr import IPNetwork

        return map(lambda number: IPNetwork('{a}.{b}.{c}.0/24'.format(a=class_a, b=random.randint(0, 99), c=number)), xrange(1, how_many+1))

    def get_net_names(self, common_part_of_name, how_many):
        return map(lambda number: '{sqe_pref}-{name}-net-{number}'.format(sqe_pref=self._unique_pattern_in_name, name=common_part_of_name, number=number), xrange(1, how_many+1))

    def get_net_subnet_lines(self, common_part_of_name, class_a, how_many, vlan=None, is_dhcp=True):<commit_msg>CReate networks with IP XX.10<XX<99.XX.XX<commit_after>import sys


class Cloud:
    ROLE_CONTROLLER = 'controller'
    ROLE_UCSM = 'ucsm'
    ROLE_NETWORK = 'network'
    ROLE_COMPUTE = 'compute'
    ROLE_MEDIATOR = 'mediator'

    def __init__(self, cloud, user, password, tenant, end_point=None):
        self._openstack_bin = 'openstack'
        self._neutron_bin = 'neutron'

        self._create_server_cmd = self._openstack_bin + ' server create -f shell '
        self._create_subnet_cmd = self._neutron_bin + ' subnet-create -f shell '
        self._create_network_cmd = self._openstack_bin + ' network create -f shell '
        self._create_port_cmd = self._neutron_bin + ' port-create -f shell '

        self._delete_server_cmd = self._openstack_bin + ' server delete '
        self._delete_keypair_cmd = self._openstack_bin + ' keypair delete '
        self._delete_network_cmd = self._openstack_bin + ' network delete '
        self._delete_subnet_cmd = self._neutron_bin + ' subnet-delete '
        self._delete_port_cmd = self._neutron_bin + ' port-delete '
        self._delete_user_cmd = self._openstack_bin + ' user delete '
        self._delete_project_cmd = self._openstack_bin + ' project delete '

        self._show_server_cmd = self._openstack_bin + ' server show -f shell '
        self._show_subnet_cmd = self._neutron_bin + ' subnet-show -f shell '
        
        self._list_network_cmd = self._openstack_bin + ' network list -f json'
        self._list_subnet_cmd = self._openstack_bin + ' subnet list -f json'
        self._list_router_cmd = self._openstack_bin + ' router list -f json'
        self._list_port_cmd = self._openstack_bin + ' port list -f json'
        self._list_server_cmd = self._openstack_bin + ' server list -f json --all'
        self._list_keypair_cmd = self._openstack_bin + ' keypair list -f json'
        self._list_image_cmd = self._openstack_bin + ' image list -f json'
        self._list_user_cmd = self._openstack_bin + ' user list -f json'
        self._list_project_cmd = self._openstack_bin + ' project list -f json'

        self._fip_network = 'Default Value Set In Cloud.__init__()'
        self._provider_physical_network = 'Default Value Set In Cloud.__init__()'
        self._openstack_version = 'Default Value Set In Cloud.__init__()'

        self.name = cloud
        self.user = user
        self.tenant = tenant
        self.password = password
        self.info = {'controller': [], 'ucsm': [], 'network': [], 'compute': []}
        self.mac_2_ip = {}
        self.hostname_2_ip = {}
        self.end_point = end_point
        self._dns = '192.168.3.11'
        self._unique_pattern_in_name = 'sqe-test'
        self._instance_counter = 0  # this counter is used to count how many instances are created via this class



    def __repr__(self):
        return '--os-username {u} --os-tenant-name {t} --os-password {p} --os-auth-url {a}'.format(u=self.user, t=self.tenant, p=self.password, a=self.get_end_point())

    def get_end_point(self):
        return self.end_point

    def get_first(self, role, parameter):
        """
            :param role: controller, network, compute, ucsm
            :param parameter: ip, mac, hostname
            :return: the first value for given parameter of given role
        """
        values = self.get(role=role, parameter=parameter)
        if values:
            return values[0]
        else:
            return 'NoValueFor' + role + parameter

    def add_server(self, config_name, server):
        """ Set all parameters for the given server
        :param config_name:
        :param server:
        """
        if config_name.startswith('aio'):
            role = self.ROLE_CONTROLLER
        else:
            role = None
            for x in self.info.keys():
                if x in config_name:
                    role = x
                    break
            if role is None:
                raise RuntimeError('Failed to deduce cloud role for server {0}'.format(server))

        self.hostname_2_ip[server.hostname] = server.ip
        self.mac_2_ip[server.ip_mac] = server.ip

        _info = {'ip': server.ip, 'mac': server.ip_mac, 'username': server.username, 'hostname': server.hostname, 'password': <PASSWORD>}
        self.info[role].append(_info)

    def add_service_end_point(self, service, url, end_point):
        self.service_end_points[service].update({url: end_point})

    def get_service_end_point(self, service, url):
        return self.service_end_points[service][url]

    def create_open_rc(self):
        """ Creates open_rc for the given cloud"""
        open_rc = """
export OS_USERNAME={user}
export OS_TENANT_NAME={tenant}
export OS_PASSWORD={password}
export OS_AUTH_URL={end_point}
"""
        return open_rc.format(user=self.user, tenant=self.tenant, password=<PASSWORD>, end_point=self.get_end_point())

    def cmd(self, cmd):
        import json
        import subprocess
        import sys

        sys.stdout.writelines("Call command: " + cmd)
        venv = '. ' + sys.prefix + '/bin/activate && ' if hasattr(sys, 'real_prefix') else ''
        ans = subprocess.check_output('{venv}{cmd} {creds}'.format(venv=venv, cmd=cmd, creds=self), shell=True)
        sys.stdout.writelines("Output:" + str(ans))
        if '-f csv' in cmd:
            return self._process_csv_output(ans)
        elif '-f json' in cmd:
            return json.loads(ans)
        elif '-f shell' in cmd:
            return self._process_shell_output(ans)
        else:
            return ans

    @staticmethod
    def _filter(minus_c, flt):
        return '{c} {g}'.format(c='-c ' + minus_c if minus_c else '', g=' | grep ' + flt if flt else '')

    @staticmethod
    def _process_csv_output(answer):
        output_list = []
        if answer:
            lines = answer.split('\n')
            keys = []
            for i, line in enumerate(lines):
                line = line.strip('\r')
                if i == 0:  # it's line of keys like: "ID","Name" or "ID","Name","Subnets"
                    keys = map(lambda x: x.strip('" '), line.split(','))
                else:
                    output_list.append(dict(zip(keys, map(lambda x: x.strip('" '), line.split(',')))))
        return output_list

    @staticmethod
    def _process_shell_output(answer):
        output_dict = {}
        if answer:
            lines = answer.split('\n')
            for line in lines:
                if '=' not in line:  # might be lines not in form a=b
                    continue
                line = line.strip('\r')
                key, value = line.split('=', 1)
                output_dict[key.strip('" ')] = value.strip('" ')
        return output_dict

    @staticmethod
    def _names_from_answer(ans):
        return filter(lambda x: x not in ['name', '|', '+--------+'], ans.split())

    def get_fip_network(self):
        if not self._fip_network:
            ans = self.cmd(self._neutron_bin + ' net-list --router:external=True -c name')
            net_names = self._names_from_answer(ans)
            if not net_names:
                return
            self._fip_network = net_names[0]
            ans = self.cmd(self._neutron_bin + ' net-list -c provider:physical_network --name {0}'.format(self._fip_network))
            self._provider_physical_network = filter(lambda x: x not in ['provider:physical_network', '|', '+---------------------------+'], ans.split())[0]
        return self._fip_network

    @staticmethod
    def _parse_cli_output(output_lines):
        """Borrowed from tempest-lib. Parse single table from cli output.
        :param output_lines:
        :returns: dict with list of column names in 'headers' key and rows in 'values' key.
        """

        if not isinstance(output_lines, list):
            output_lines = output_lines.split('\n')

        if not output_lines[-1]:
            output_lines = output_lines[:-1]  # skip last line if empty (just newline at the end)

        columns = Cloud._table_columns(output_lines[2])
        is_2_values = len(columns) == 2

        def line_to_row(input_line):
            return [input_line[col[0]:col[1]].strip() for col in columns]

        headers = None
        d2 = {}
        output_list = []
        for line in output_lines[3:-1]:
            row = line_to_row(line)
            if is_2_values:
                d2[row[0]] = row[1]
            else:
                headers = headers or line_to_row(output_lines[1])
                output_list.append({headers[i]: row[i] for i in xrange(len(columns))})

        return output_list if output_list else [d2]

    @staticmethod
    def _table_columns(first_table_row):
        """Borrowed from tempest-lib. Find column ranges in output line.
        :returns: list of tuples (start,end) for each column detected by plus (+) characters in delimiter line.
        """
        positions = []
        start = 1  # there is '+' at 0
        while start < len(first_table_row):
            end = first_table_row.find('+', start)
            if end == -1:
                break
            positions.append((start, end))
            start = end + 1
        return positions

    @staticmethod
    def get_cidrs4(class_a, how_many):
        import random
        from netaddr import IPNetwork

        return map(lambda number: IPNetwork('{a}.{b}.{c}.0/24'.format(a=class_a, b=random.randint(10, 99), c=number)), xrange(1, how_many+1))

    def get_net_names(self, common_part_of_name, how_many):
        return map(lambda number: '{sqe_pref}-{name}-net-{number}'.format(sqe_pref=self._unique_pattern_in_name, name=common_part_of_name, number=number), xrange(1, how_many+1))

    def get_net_subnet_lines(self, common_part_of_name, class_a, how_many, vlan=None, is_dhcp=True):
{'fraction_non_alphanumeric': 0.11882153672432377, 'fraction_numerical': 0.00559462095159883, 'mean_word_length': 2.3192504258943782, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6591682', 'n_tokens_mistral': 6068, 'n_tokens_neox': 5920, 'n_words': 1439}