Merge "Openstack: Using VSPERF to Test on Openstack."

6 files changed, 536 insertions, 0 deletions

diff --git a/tools/os_deploy_tgen/osclients/__init__.py b/tools/os_deploy_tgen/osclients/__init__.py
new file mode 100644
index 00000000..e73a36c9
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2020 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Openstack Client
+"""
diff --git a/tools/os_deploy_tgen/osclients/glance.py b/tools/os_deploy_tgen/osclients/glance.py
new file mode 100644
index 00000000..f59f0d8d
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/glance.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Glance CLient
+"""
+
+def get_image(glance_client, image_name):
+    """
+    Get the IMage
+    """
+    for image in glance_client.images.list():
+        if image.name == image_name:
+            return image
+    return None
+
+
+def get_supported_versions(glance_client):
+    """
+    Get Supported Version
+    """
+    return set(version['id'] for version in glance_client.versions.list())
diff --git a/tools/os_deploy_tgen/osclients/heat.py b/tools/os_deploy_tgen/osclients/heat.py
new file mode 100755
index 00000000..8681731b
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/heat.py
@@ -0,0 +1,156 @@
+# Copyright 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Heat Client
+"""
+
+#import sys
+import time
+
+from heatclient import exc
+from oslo_log import log as logging
+from timeout_decorator import timeout
+
+LOG = logging.getLogger(__name__)
+
+
+def create_stack(heat_client, stack_name, template, parameters,
+                 environment=None):
+    """
+    Create Stack
+    """
+    stack_params = {
+        'stack_name': stack_name,
+        'template': template,
+        'parameters': parameters,
+        'environment': environment,
+    }
+
+    stack = heat_client.stacks.create(**stack_params)['stack']
+    LOG.info('New stack: %s', stack)
+
+    wait_stack_completion(heat_client, stack['id'])
+
+    return stack['id']
+
+
+def get_stack_status(heat_client, stack_id):
+    """
+    Get Stack Status
+    """
+    # stack.get operation may take long time and run out of time. The reason
+    # is that it resolves all outputs which is done serially. On the other hand
+    # stack status can be retrieved from the list operation. Internally listing
+    # supports paging and every request should not take too long.
+    for stack in heat_client.stacks.list():
+        if stack.id == stack_id:
+            return stack.status, stack.stack_status_reason
+        else:
+            raise exc.HTTPNotFound(message='Stack %s is not found' % stack_id)
+    return None
+
+def get_id_with_name(heat_client, stack_name):
+    """
+    Get Stack ID by name
+    """
+    # This method isn't really necessary since the Heat client accepts
+    # stack_id and stack_name interchangeably. This is provided more as a
+    # safety net to use ids which are guaranteed to be unique and provides
+    # the benefit of keeping the Shaker code consistent and more easily
+    # traceable.
+    stack = heat_client.stacks.get(stack_name)
+    return stack.id
+
+
+def wait_stack_completion(heat_client, stack_id):
+    """
+    Wait for Stack completion
+    """
+    reason = None
+    status = None
+
+    while True:
+        status, reason = get_stack_status(heat_client, stack_id)
+        LOG.debug('Stack status: %s', status)
+        if status not in ['IN_PROGRESS', '']:
+            break
+
+        time.sleep(5)
+
+    if status != 'COMPLETE':
+        resources = heat_client.resources.list(stack_id)
+        for res in resources:
+            if (res.resource_status != 'CREATE_COMPLETE' and
+                    res.resource_status_reason):
+                LOG.error('Heat stack resource %(res)s of type %(type)s '
+                          'failed with %(reason)s',
+                          dict(res=res.logical_resource_id,
+                               type=res.resource_type,
+                               reason=res.resource_status_reason))
+
+        raise exc.StackFailure(stack_id, status, reason)
+
+
+# set the timeout for this method so we don't get stuck polling indefinitely
+# waiting for a delete
+@timeout(600)
+def wait_stack_deletion(heat_client, stack_id):
+    """
+    Wait for stack deletion
+    """
+    try:
+        heat_client.stacks.delete(stack_id)
+        while True:
+            status, reason = get_stack_status(heat_client, stack_id)
+            LOG.debug('Stack status: %s Stack reason: %s', status, reason)
+            if status == 'FAILED':
+                raise exc.StackFailure('Failed to delete stack %s' % stack_id)
+
+            time.sleep(5)
+
+    except TimeoutError:
+        LOG.error('Timed out waiting for deletion of stack %s' % stack_id)
+
+    except exc.HTTPNotFound:
+        # once the stack is gone we can assume it was successfully deleted
+        # clear the exception so it doesn't confuse the logs
+        #if sys.version_info < (3, 0):
+        #    sys.exc_clear()
+        LOG.info('Stack %s was successfully deleted', stack_id)
+
+
+def get_stack_outputs(heat_client, stack_id):
+    """
+    Get Stack Output
+    """
+    # try to use optimized way to retrieve outputs, fallback otherwise
+    if hasattr(heat_client.stacks, 'output_list'):
+        try:
+            output_list = heat_client.stacks.output_list(stack_id)['outputs']
+
+            result = {}
+            for output in output_list:
+                output_key = output['output_key']
+                value = heat_client.stacks.output_show(stack_id, output_key)
+                result[output_key] = value['output']['output_value']
+
+            return result
+        except BaseException as err:
+            LOG.info('Cannot get output list, fallback to old way: %s', err)
+
+    outputs_list = heat_client.stacks.get(stack_id).to_dict()['outputs']
+    return dict((item['output_key'], item['output_value'])
+                for item in outputs_list)
diff --git a/tools/os_deploy_tgen/osclients/neutron.py b/tools/os_deploy_tgen/osclients/neutron.py
new file mode 100644
index 00000000..f75077dc
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/neutron.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Neutron client
+"""
+
+from oslo_log import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+def choose_external_net(neutron_client):
+    """
+    Choose External Network
+    """
+    ext_nets = neutron_client.list_networks(
+        **{'router:external': True})['networks']
+    if not ext_nets:
+        raise Exception('No external networks found')
+    return ext_nets[0]['name']
diff --git a/tools/os_deploy_tgen/osclients/nova.py b/tools/os_deploy_tgen/osclients/nova.py
new file mode 100644
index 00000000..b2baa34f
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/nova.py
@@ -0,0 +1,213 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Nova Client
+"""
+
+import itertools
+import re
+import time
+
+from novaclient import client as nova_client_pkg
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ForbiddenException(nova_client_pkg.exceptions.Forbidden):
+    """
+    Custome Exception
+    """
+
+
+
+def get_available_compute_nodes(nova_client, flavor_name):
+    """
+    Return available compute nodes
+    """
+    try:
+        host_list = [dict(host=svc.host, zone=svc.zone)
+                     for svc in
+                     nova_client.services.list(binary='nova-compute')
+                     if svc.state == 'up' and svc.status == 'enabled']
+
+        # If the flavor has aggregate_instance_extra_specs set then filter
+        # host_list to pick only the hosts matching the chosen flavor.
+        flavor = get_flavor(nova_client, flavor_name)
+
+        if flavor is not None:
+            extra_specs = flavor.get_keys()
+
+            for item in extra_specs:
+                if "aggregate_instance_extra_specs" in item:
+                    LOG.debug('Flavor contains %s, using compute node '
+                              'filtering', extra_specs)
+
+                    # getting the extra spec seting for flavor in the
+                    # standard format of extra_spec:value
+                    extra_spec = item.split(":")[1]
+                    extra_spec_value = extra_specs.get(item)
+
+                    # create a set of aggregate host which match
+                    agg_hosts = set(itertools.chain(
+                        *[agg.hosts for agg in
+                          nova_client.aggregates.list() if
+                          agg.metadata.get(extra_spec) == extra_spec_value]))
+
+                    # update list of available hosts with
+                    # host_aggregate cross-check
+                    host_list = [elem for elem in host_list if
+                                 elem['host'] in agg_hosts]
+
+        LOG.debug('Available compute nodes: %s ', host_list)
+
+        return host_list
+
+    except nova_client_pkg.exceptions.Forbidden as error:
+        msg = 'Forbidden to get list of compute nodes'
+        raise ForbiddenException(msg) from error
+
+
+def does_flavor_exist(nova_client, flavor_name):
+    """
+    Check if flavor exists
+    """
+    for flavor in nova_client.flavors.list():
+        if flavor.name == flavor_name:
+            return True
+    return False
+
+
+def create_flavor(nova_client, **kwargs):
+    """
+    Create a flavor
+    """
+    try:
+        nova_client.flavors.create(**kwargs)
+    except nova_client_pkg.exceptions.Forbidden as error:
+        msg = 'Forbidden to create flavor'
+        raise ForbiddenException(msg) from error
+
+
+def get_server_ip(nova_client, server_name, ip_type):
+    """
+    Get IP of the compute
+    """
+    server = nova_client.servers.find(name=server_name)
+    addresses = server.addresses
+    ips = [v['addr'] for v in itertools.chain(*addresses.values())
+           if v['OS-EXT-IPS:type'] == ip_type]
+    if not ips:
+        raise Exception('Could not get IP address of server: %s' % server_name)
+    if len(ips) > 1:
+        raise Exception('Server %s has more than one IP addresses: %s' %
+                        (server_name, ips))
+    return ips[0]
+
+
+def get_server_host_id(nova_client, server_name):
+    """
+    Get the host id
+    """
+    server = nova_client.servers.find(name=server_name)
+    return server.hostId
+
+
+def check_server_console(nova_client, server_id, len_limit=100):
+    """
+    Check Server console
+    """
+    try:
+        console = (nova_client.servers.get(server_id)
+                   .get_console_output(len_limit))
+    except nova_client_pkg.exceptions.ClientException as exc:
+        LOG.warning('Error retrieving console output: %s. Ignoring', exc)
+        return None
+
+    for line in console.splitlines():
+        if (re.search(r'\[critical\]', line, flags=re.IGNORECASE) or
+                re.search(r'Cloud-init.*Datasource DataSourceNone\.', line)):
+            message = ('Instance %(id)s has critical cloud-init error: '
+                       '%(msg)s. Check metadata service availability' %
+                       dict(id=server_id, msg=line))
+            LOG.error(message)
+            return message
+        if re.search(r'\[error', line, flags=re.IGNORECASE):
+            LOG.error('Error message in instance %(id)s console: %(msg)s',
+                      dict(id=server_id, msg=line))
+        elif re.search(r'warn', line, flags=re.IGNORECASE):
+            LOG.info('Warning message in instance %(id)s console: %(msg)s',
+                     dict(id=server_id, msg=line))
+
+    return None
+
+
+def _poll_for_status(nova_client, server_id, final_ok_states, poll_period=20,
+                     status_field="status"):
+    """
+    Poll for status
+    """
+    LOG.debug('Poll instance %(id)s, waiting for any of statuses %(statuses)s',
+              dict(id=server_id, statuses=final_ok_states))
+    while True:
+        obj = nova_client.servers.get(server_id)
+
+        err_msg = check_server_console(nova_client, server_id)
+        if err_msg:
+            raise Exception('Critical error in instance %s console: %s' %
+                            (server_id, err_msg))
+
+        status = getattr(obj, status_field)
+        if status:
+            status = status.lower()
+
+        LOG.debug('Instance %(id)s has status %(status)s',
+                  dict(id=server_id, status=status))
+
+        if status in final_ok_states:
+            break
+        if status in ('error', 'paused'):
+            raise Exception(obj.fault['message'])
+
+        time.sleep(poll_period)
+
+
+def wait_server_shutdown(nova_client, server_id):
+    """
+    Wait server shutdown
+    """
+    _poll_for_status(nova_client, server_id, ['shutoff'])
+
+
+def wait_server_snapshot(nova_client, server_id):
+    """
+    Wait server snapshot
+    """
+    task_state_field = "OS-EXT-STS:task_state"
+    server = nova_client.servers.get(server_id)
+    if hasattr(server, task_state_field):
+        _poll_for_status(nova_client, server.id, [None, '-', ''],
+                         status_field=task_state_field)
+
+
+def get_flavor(nova_client, flavor_name):
+    """
+    Get the flavor
+    """
+    for flavor in nova_client.flavors.list():
+        if flavor.name == flavor_name:
+            return flavor
+    return None
diff --git a/tools/os_deploy_tgen/osclients/openstack.py b/tools/os_deploy_tgen/osclients/openstack.py
new file mode 100644
index 00000000..58297e6c
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/openstack.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Openstack Client - Main File
+"""
+
+import os_client_config
+from oslo_log import log as logging
+from oslo_utils import importutils
+
+LOG = logging.getLogger(__name__)
+
+
+class OpenStackClientException(Exception):
+    '''
+    Custom Exception
+    '''
+
+
+def init_profiling(os_profile):
+    """
+    Initialize Profiling
+    """
+    if os_profile:
+        osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+
+        if osprofiler_profiler:  # lib is present
+            osprofiler_profiler.init(os_profile)
+            trace_id = osprofiler_profiler.get().get_base_id()
+            LOG.info('Profiling is enabled, trace id: %s', trace_id)
+        else:  # param is set, but lib is not present
+            LOG.warning('Profiling could not be enabled. To enable profiling '
+                        'please install "osprofiler" library')
+
+
+class OpenStackClient():
+    """
+    Client Class
+    """
+    def __init__(self, openstack_params):
+        """
+        Initialize
+        """
+        LOG.debug('Establishing connection to OpenStack')
+
+        init_profiling(openstack_params.get('os_profile'))
+
+        config = os_client_config.OpenStackConfig()
+        cloud_config = config.get_one_cloud(**openstack_params)
+        if openstack_params['os_insecure']:
+            cloud_config.config['verify'] = False
+            cloud_config.config['cacert'] = None
+        self.keystone_session = cloud_config.get_session()
+        self.nova = cloud_config.get_legacy_client('compute')
+        self.neutron = cloud_config.get_legacy_client('network')
+        self.glance = cloud_config.get_legacy_client('image')
+
+        # heat client wants endpoint to be always set
+        endpoint = cloud_config.get_session_endpoint('orchestration')
+        if not endpoint:
+            raise OpenStackClientException(
+                'Endpoint for orchestration service is not found')
+        self.heat = cloud_config.get_legacy_client('orchestration',
+                                                   endpoint=endpoint)
+
+        # Ping OpenStack
+        self.keystone_session.get_token()
+
+        LOG.info('Connection to OpenStack is initialized')