<commit_before># Copyright 2015 Rackspace
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import ConfigParser
import math
import subprocess

import requests

import random

from oslotest import base


class TestCase(base.BaseTestCase):
    """Base test class for all functional tests."""

    def setUp(self):
        """Set the endpoints for ironic and glance.
        Create a config file for arsenal with default values
        set in the `set_config_values` method.
        """
        super(TestCase, self).setUp()
        self.processes_to_kill = []
        self.flavors = 3
        self.port = str(random.randint(2000, 9000))
        self.mimic_endpoint = "http://localhost"
        self.mimic_ironic_url = "{0}:{1}/ironic/v1/nodes/detail".format(
            self.mimic_endpoint,
            self.port)
        self.mimic_glance_url = "{0}:{1}/glance/v2/images".format(
            self.mimic_endpoint,
            self.port)
        self.default_config_file = 'test_default' + self.port + '.conf'
        self.strategy = None

    def tearDown(self):
        """Kill arsenal and mimic processes
        """
        super(TestCase, self).tearDown()

        for each in self.processes_to_kill:
            try:
                each.kill()
            except OSError as e:
                if not ('No such process' in str(e)):
                    raise

    def start_mimic_service(self):
        """Start the mimic service and wait for the service to be started.
        """
        p = subprocess.Popen(['twistd', '-n',
                              '--pidfile=twistd{}.pid'.format(self.port),
                              'mimic', '-l', self.port],
                             stdout=subprocess.PIPE)
        self.processes_to_kill.append(p)
        while True:
            line = p.stdout.readline()
            if ((line == '' and p.poll() is not None) or  # process done
                    "Starting factory <twisted.web.server.Site instance"
                    in line):
                break

    def start_arsenal_service(self, config_file=None,
                              service_status="Started Arsenal service"):
        """Start the arsenal service with the given config file.
        If a config file is not provided, create and the use the default
        config file.
        """
        if not config_file:
            config_file = self.default_config_file
            config_values = self.set_config_values()
            self.create_arsenal_config_file(config_values,
                                            file_name=self.default_config_file)
        a = subprocess.Popen(['arsenal-director', '--config-file', config_file,
                              '-v'],
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        self.processes_to_kill.append(a)
        while True:
            line = a.stdout.readline()
            if ((line == '' and a.poll() is not None) or  # process done
                    service_status in line):
                break

    def generate_config_file_name(self, name='test'):
        """Create a config file name appending the port to it
        """
        return name + str(self.port) + '.conf'

    def set_config_values(self, dry_run=False, interval=1, rate_limit=100,
                          percentage_to_cache=0.5, image_weights=None,
                          default_image_weight=1):
        """Set values for the arsenal config file."""
        mimic_endpoint = self.mimic_endpoint
        port = self.port
        image_weights = (image_weights or
                         {'OnMetal - CentOS 6': 80,
                          'OnMetal - CentOS 7': 80,
                          'OnMetal - CoreOS (Alpha)': 11,
                          'OnMetal - CoreOS (Beta)': 1,
                          'OnMetal - CoreOS (Stable)': 5,
                          'OnMetal - Debian 7 (Wheezy)': 60,
                          'OnMetal - Debian 8 (Jessie)': 14,
                          'OnMetal - Debian Testing (Stretch)': 2,
                          'OnMetal - Debian Unstable (Sid)': 2,
                          'OnMetal - Fedora 21': 1,
                          'OnMetal - Fedora 22': 2,
                          'OnMetal - Ubuntu 12.04 LTS (Precise Pangolin)': 132,
                          'OnMetal - Ubuntu 14.04 LTS (Trusty Tahr)': 163,
                          'OnMetal - Ubuntu 15.04 (Vivid Vervet)': 3})
        return {
            'director':
                {'scout': 'onmetal_scout.OnMetalScout',
                 'dry_run': dry_run,
                 'poll_spacing': interval,
                 'directive_spacing': interval,
                 'cache_directive_rate_limit': rate_limit,
                 'cache_directive_limiting_period': interval,
                 'eject_directive_rate_limit': rate_limit,
                 'eject_directive_limiting_period': interval,
                 'log_statistics': True},
            'client_wrapper':
                {'call_max_retries': 3,
                 'call_retry_interval': 3,
                 'os_tenant_name': 232323,
                 'os_username': 'test-user',
                 'region_name': 'ORD',
                 'service_name': 'cloudServersOpenStack',
                 'auth_system': 'rackspace',
                 'os_api_url': '{0}:{1}/identity/v2.0'.format(
                     mimic_endpoint, port),
                 'os_password': '<PASSWORD>'},
            'nova': {},
            'ironic':
                {'admin_username': 'test-admin',
                 'admin_password': '<PASSWORD>',
                 'admin_tenant_name': 99999,
                 'admin_url': '{0}:{1}/identity/v2.0'.format(
                     mimic_endpoint, port),
                 'api_endpoint': '{0}:{1}/ironic/v1'.format(
                     mimic_endpoint, port)},
            'glance':
                {'api_endpoint': '{0}:{1}/glance/v2'.format(
                    mimic_endpoint, port),
                 'admin_auth_token': '<PASSWORD>'},
            'simple_proportional_strategy':
                {'percentage_to_cache': percentage_to_cache},
            'strategy':
                {'module_class': self.strategy or
                 'simple_proportional_strategy.SimpleProportionalStrategy',
                 'image_weights': image_weights,
                 'default_image_weight': default_image_weight}
        }

    def create_arsenal_config_file(self, config_values,
                                   file_name='test.conf'):
        """Given `config_values` object set the values in the
        arsensal.conf file.
        """
        config = ConfigParser.RawConfigParser()
        for each_key in config_values.keys():
            if not config.has_section(each_key):
                config.add_section(each_key)
            for key, value in config_values[each_key].iteritems():
                config.set(each_key, key, value)
        f = open(file_name, 'w')
        config.write(f)
        f.close()

    def get_all_ironic_nodes(self):
        """Get a list of ironic nodes with details.
        """
        nodes_list_response = requests.get(self.mimic_ironic_url)
        self.assertEqual(nodes_list_response.status_code, 200)
        ironic_nodes_list = nodes_list_response.json()['nodes']
        return ironic_nodes_list

    def get_provisioned_ironic_nodes(self):
        """Get a list of provisioned ironic nodes.
        """
        all_nodes = self.get_all_ironic_nodes()
        return [node for node in all_nodes
                if node.get('instance_uuid')]

    def get_unprovisioned_ironic_nodes(self):
        """Get a list of unprovisioned ironic nodes.
        """
        all_nodes = self.get_all_ironic_nodes()
        return [node for node in all_nodes
                if not node.get('instance_uuid')]

    def get_cached_ironic_nodes(self):
        """Get the cached nodes from the list of nodes in ironic.
        """<commit_msg>Remove files generated by functional testing<commit_after># Copyright 2015 Rackspace
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import ConfigParser
import math
import os
import subprocess

import requests

import random

from oslotest import base


class TestCase(base.BaseTestCase):
    """Base test class for all functional tests."""

    def setUp(self):
        """Set the endpoints for ironic and glance.
        Create a config file for arsenal with default values
        set in the `set_config_values` method.
        """
        super(TestCase, self).setUp()
        self.processes_to_kill = []
        self.flavors = 3
        self.port = str(random.randint(2000, 9000))
        self.mimic_endpoint = "http://localhost"
        self.mimic_ironic_url = "{0}:{1}/ironic/v1/nodes/detail".format(
            self.mimic_endpoint,
            self.port)
        self.mimic_glance_url = "{0}:{1}/glance/v2/images".format(
            self.mimic_endpoint,
            self.port)
        self.default_config_file = 'test_default' + self.port + '.conf'
        self.strategy = None
        self.generated_test_files = []

    def tearDown(self):
        """Kill arsenal and mimic processes
        """
        super(TestCase, self).tearDown()

        for each in self.processes_to_kill:
            try:
                each.kill()
            except OSError as e:
                if not ('No such process' in str(e)):
                    raise

        # Remove files generated by testing.
        for filename in self.generated_test_files:
            os.remove(filename)
        self.generated_test_files = []

    def start_mimic_service(self):
        """Start the mimic service and wait for the service to be started.
        """
        pid_filename = 'twistd{}.pid'.format(self.port)
        p = subprocess.Popen(['twistd', '-n',
                              '--pidfile={}'.format(pid_filename),
                              'mimic', '-l', self.port],
                             stdout=subprocess.PIPE)
        self.generated_test_files.append(pid_filename)
        self.processes_to_kill.append(p)
        while True:
            line = p.stdout.readline()
            if ((line == '' and p.poll() is not None) or  # process done
                    "Starting factory <twisted.web.server.Site instance"
                    in line):
                break

    def start_arsenal_service(self, config_file=None,
                              service_status="Started Arsenal service"):
        """Start the arsenal service with the given config file.
        If a config file is not provided, create and the use the default
        config file.
        """
        if not config_file:
            config_file = self.default_config_file
            config_values = self.set_config_values()
            self.create_arsenal_config_file(config_values,
                                            file_name=self.default_config_file)
        a = subprocess.Popen(['arsenal-director', '--config-file', config_file,
                              '-v'],
                             stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        self.processes_to_kill.append(a)
        while True:
            line = a.stdout.readline()
            if ((line == '' and a.poll() is not None) or  # process done
                    service_status in line):
                break

    def generate_config_file_name(self, name='test'):
        """Create a config file name appending the port to it
        """
        return name + str(self.port) + '.conf'

    def set_config_values(self, dry_run=False, interval=1, rate_limit=100,
                          percentage_to_cache=0.5, image_weights=None,
                          default_image_weight=1):
        """Set values for the arsenal config file."""
        mimic_endpoint = self.mimic_endpoint
        port = self.port
        image_weights = (image_weights or
                         {'OnMetal - CentOS 6': 80,
                          'OnMetal - CentOS 7': 80,
                          'OnMetal - CoreOS (Alpha)': 11,
                          'OnMetal - CoreOS (Beta)': 1,
                          'OnMetal - CoreOS (Stable)': 5,
                          'OnMetal - Debian 7 (Wheezy)': 60,
                          'OnMetal - Debian 8 (Jessie)': 14,
                          'OnMetal - Debian Testing (Stretch)': 2,
                          'OnMetal - Debian Unstable (Sid)': 2,
                          'OnMetal - Fedora 21': 1,
                          'OnMetal - Fedora 22': 2,
                          'OnMetal - Ubuntu 12.04 LTS (Precise Pangolin)': 132,
                          'OnMetal - Ubuntu 14.04 LTS (Trusty Tahr)': 163,
                          'OnMetal - Ubuntu 15.04 (Vivid Vervet)': 3})
        return {
            'director':
                {'scout': 'onmetal_scout.OnMetalScout',
                 'dry_run': dry_run,
                 'poll_spacing': interval,
                 'directive_spacing': interval,
                 'cache_directive_rate_limit': rate_limit,
                 'cache_directive_limiting_period': interval,
                 'eject_directive_rate_limit': rate_limit,
                 'eject_directive_limiting_period': interval,
                 'log_statistics': True},
            'client_wrapper':
                {'call_max_retries': 3,
                 'call_retry_interval': 3,
                 'os_tenant_name': 232323,
                 'os_username': 'test-user',
                 'region_name': 'ORD',
                 'service_name': 'cloudServersOpenStack',
                 'auth_system': 'rackspace',
                 'os_api_url': '{0}:{1}/identity/v2.0'.format(
                     mimic_endpoint, port),
                 'os_password': '<PASSWORD>'},
            'nova': {},
            'ironic':
                {'admin_username': 'test-admin',
                 'admin_password': '<PASSWORD>',
                 'admin_tenant_name': 99999,
                 'admin_url': '{0}:{1}/identity/v2.0'.format(
                     mimic_endpoint, port),
                 'api_endpoint': '{0}:{1}/ironic/v1'.format(
                     mimic_endpoint, port)},
            'glance':
                {'api_endpoint': '{0}:{1}/glance/v2'.format(
                    mimic_endpoint, port),
                 'admin_auth_token': '<PASSWORD>'},
            'simple_proportional_strategy':
                {'percentage_to_cache': percentage_to_cache},
            'strategy':
                {'module_class': self.strategy or
                 'simple_proportional_strategy.SimpleProportionalStrategy',
                 'image_weights': image_weights,
                 'default_image_weight': default_image_weight}
        }

    def create_arsenal_config_file(self, config_values,
                                   file_name='test.conf'):
        """Given `config_values` object set the values in the
        arsensal.conf file.
        """
        config = ConfigParser.RawConfigParser()
        for each_key in config_values.keys():
            if not config.has_section(each_key):
                config.add_section(each_key)
            for key, value in config_values[each_key].iteritems():
                config.set(each_key, key, value)
        f = open(file_name, 'w')
        config.write(f)
        f.close()

        self.generated_test_files.append(file_name)

    def get_all_ironic_nodes(self):
        """Get a list of ironic nodes with details.
        """
        nodes_list_response = requests.get(self.mimic_ironic_url)
        self.assertEqual(nodes_list_response.status_code, 200)
        ironic_nodes_list = nodes_list_response.json()['nodes']
        return ironic_nodes_list

    def get_provisioned_ironic_nodes(self):
        """Get a list of provisioned ironic nodes.
        """
        all_nodes = self.get_all_ironic_nodes()
        return [node for node in all_nodes
                if node.get('instance_uuid')]

    def get_unprovisioned_ironic_nodes(self):
        """Get a list of unprovisioned ironic nodes.
        """
        all_nodes = self.get_all_ironic_nodes()
        return [node for node in all_nodes
                if not node.get('instance_uuid')]

    def get_cached_ironic_nodes(self):
        """Get the cached nodes from the list of nodes in ironic.
        """
{'fraction_non_alphanumeric': 0.09945292254534985, 'fraction_numerical': 0.011862942700835013, 'mean_word_length': 1.7176838810641628, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3845405', 'n_tokens_mistral': 5012, 'n_tokens_neox': 4731, 'n_words': 1185}