From 9ec3918b56f1e8862fe140455928cdcd87a2554b Mon Sep 17 00:00:00 2001
From: opensource-tnbt <sridhar.rao@spirent.com>
Date: Wed, 11 Nov 2020 22:55:02 +0530
Subject: Openstack: Using VSPERF to Test on Openstack.
This patch will support running VSPERF Tests with Openstack.
This patch adds the following:
1. Provide --openstack parameter.
2. New Configuration file for openstack
3. Deploy Trafficgenerator based on configuration provided
4. Run Tests after Trafficgenerator are deployed on openstack
Update-1: Minor bug-fixes and Documentation Added.
Update-2: Add user-config to heat.
Update-3: Update Python Requirements
Update-4: Add dogpile
Update-5: Update decription of the Hot files.
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Iebec356eb893e0e6726cac6a10537b99e41f67f4
---
tools/os_deploy_tgen/__init__.py | 17 +
tools/os_deploy_tgen/osclients/__init__.py | 17 +
tools/os_deploy_tgen/osclients/glance.py | 34 ++
tools/os_deploy_tgen/osclients/heat.py | 156 +++++++
tools/os_deploy_tgen/osclients/neutron.py | 34 ++
tools/os_deploy_tgen/osclients/nova.py | 213 ++++++++++
tools/os_deploy_tgen/osclients/openstack.py | 82 ++++
tools/os_deploy_tgen/osdt.py | 601 +++++++++++++++++++++++++++
tools/os_deploy_tgen/templates/hotfiles.md | 13 +
tools/os_deploy_tgen/templates/l2.hot | 89 ++++
tools/os_deploy_tgen/templates/l2_1c_1i.yaml | 8 +
tools/os_deploy_tgen/templates/l2_1c_2i.yaml | 10 +
tools/os_deploy_tgen/templates/l2_2c_2i.yaml | 10 +
tools/os_deploy_tgen/templates/l2_old.hot | 93 +++++
tools/os_deploy_tgen/templates/l2fip.hot | 122 ++++++
tools/os_deploy_tgen/templates/l2up.hot | 126 ++++++
tools/os_deploy_tgen/templates/l3.hot | 125 ++++++
tools/os_deploy_tgen/templates/l3_1c_2i.yaml | 11 +
tools/os_deploy_tgen/templates/l3_2c_2i.yaml | 11 +
tools/os_deploy_tgen/templates/scenario.yaml | 44 ++
tools/os_deploy_tgen/utilities/__init__.py | 17 +
tools/os_deploy_tgen/utilities/utils.py | 183 ++++++++
22 files changed, 2016 insertions(+)
create mode 100644 tools/os_deploy_tgen/__init__.py
create mode 100644 tools/os_deploy_tgen/osclients/__init__.py
create mode 100644 tools/os_deploy_tgen/osclients/glance.py
create mode 100755 tools/os_deploy_tgen/osclients/heat.py
create mode 100644 tools/os_deploy_tgen/osclients/neutron.py
create mode 100644 tools/os_deploy_tgen/osclients/nova.py
create mode 100644 tools/os_deploy_tgen/osclients/openstack.py
create mode 100644 tools/os_deploy_tgen/osdt.py
create mode 100644 tools/os_deploy_tgen/templates/hotfiles.md
create mode 100644 tools/os_deploy_tgen/templates/l2.hot
create mode 100644 tools/os_deploy_tgen/templates/l2_1c_1i.yaml
create mode 100644 tools/os_deploy_tgen/templates/l2_1c_2i.yaml
create mode 100644 tools/os_deploy_tgen/templates/l2_2c_2i.yaml
create mode 100644 tools/os_deploy_tgen/templates/l2_old.hot
create mode 100644 tools/os_deploy_tgen/templates/l2fip.hot
create mode 100644 tools/os_deploy_tgen/templates/l2up.hot
create mode 100644 tools/os_deploy_tgen/templates/l3.hot
create mode 100644 tools/os_deploy_tgen/templates/l3_1c_2i.yaml
create mode 100644 tools/os_deploy_tgen/templates/l3_2c_2i.yaml
create mode 100644 tools/os_deploy_tgen/templates/scenario.yaml
create mode 100644 tools/os_deploy_tgen/utilities/__init__.py
create mode 100644 tools/os_deploy_tgen/utilities/utils.py
(limited to 'tools/os_deploy_tgen')
diff --git a/tools/os_deploy_tgen/__init__.py b/tools/os_deploy_tgen/__init__.py
new file mode 100644
index 00000000..1b2d5ea6
--- /dev/null
+++ b/tools/os_deploy_tgen/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2020 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Package to deploy Traffic-generator in Openstack
+"""
diff --git a/tools/os_deploy_tgen/osclients/__init__.py b/tools/os_deploy_tgen/osclients/__init__.py
new file mode 100644
index 00000000..e73a36c9
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2020 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Openstack Client
+"""
diff --git a/tools/os_deploy_tgen/osclients/glance.py b/tools/os_deploy_tgen/osclients/glance.py
new file mode 100644
index 00000000..f59f0d8d
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/glance.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Glance CLient
+"""
+
+def get_image(glance_client, image_name):
+ """
+ Get the IMage
+ """
+ for image in glance_client.images.list():
+ if image.name == image_name:
+ return image
+ return None
+
+
+def get_supported_versions(glance_client):
+ """
+ Get Supported Version
+ """
+ return set(version['id'] for version in glance_client.versions.list())
diff --git a/tools/os_deploy_tgen/osclients/heat.py b/tools/os_deploy_tgen/osclients/heat.py
new file mode 100755
index 00000000..8681731b
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/heat.py
@@ -0,0 +1,156 @@
+# Copyright 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Heat Client
+"""
+
+#import sys
+import time
+
+from heatclient import exc
+from oslo_log import log as logging
+from timeout_decorator import timeout
+
+LOG = logging.getLogger(__name__)
+
+
+def create_stack(heat_client, stack_name, template, parameters,
+ environment=None):
+ """
+ Create Stack
+ """
+ stack_params = {
+ 'stack_name': stack_name,
+ 'template': template,
+ 'parameters': parameters,
+ 'environment': environment,
+ }
+
+ stack = heat_client.stacks.create(**stack_params)['stack']
+ LOG.info('New stack: %s', stack)
+
+ wait_stack_completion(heat_client, stack['id'])
+
+ return stack['id']
+
+
+def get_stack_status(heat_client, stack_id):
+ """
+ Get Stack Status
+ """
+ # stack.get operation may take long time and run out of time. The reason
+ # is that it resolves all outputs which is done serially. On the other hand
+ # stack status can be retrieved from the list operation. Internally listing
+ # supports paging and every request should not take too long.
+ for stack in heat_client.stacks.list():
+ if stack.id == stack_id:
+ return stack.status, stack.stack_status_reason
+ else:
+ raise exc.HTTPNotFound(message='Stack %s is not found' % stack_id)
+ return None
+
+def get_id_with_name(heat_client, stack_name):
+ """
+ Get Stack ID by name
+ """
+ # This method isn't really necessary since the Heat client accepts
+ # stack_id and stack_name interchangeably. This is provided more as a
+ # safety net to use ids which are guaranteed to be unique and provides
+ # the benefit of keeping the Shaker code consistent and more easily
+ # traceable.
+ stack = heat_client.stacks.get(stack_name)
+ return stack.id
+
+
+def wait_stack_completion(heat_client, stack_id):
+ """
+ Wait for Stack completion
+ """
+ reason = None
+ status = None
+
+ while True:
+ status, reason = get_stack_status(heat_client, stack_id)
+ LOG.debug('Stack status: %s', status)
+ if status not in ['IN_PROGRESS', '']:
+ break
+
+ time.sleep(5)
+
+ if status != 'COMPLETE':
+ resources = heat_client.resources.list(stack_id)
+ for res in resources:
+ if (res.resource_status != 'CREATE_COMPLETE' and
+ res.resource_status_reason):
+ LOG.error('Heat stack resource %(res)s of type %(type)s '
+ 'failed with %(reason)s',
+ dict(res=res.logical_resource_id,
+ type=res.resource_type,
+ reason=res.resource_status_reason))
+
+ raise exc.StackFailure(stack_id, status, reason)
+
+
+# set the timeout for this method so we don't get stuck polling indefinitely
+# waiting for a delete
+@timeout(600)
+def wait_stack_deletion(heat_client, stack_id):
+ """
+ Wait for stack deletion
+ """
+ try:
+ heat_client.stacks.delete(stack_id)
+ while True:
+ status, reason = get_stack_status(heat_client, stack_id)
+ LOG.debug('Stack status: %s Stack reason: %s', status, reason)
+ if status == 'FAILED':
+ raise exc.StackFailure('Failed to delete stack %s' % stack_id)
+
+ time.sleep(5)
+
+ except TimeoutError:
+ LOG.error('Timed out waiting for deletion of stack %s' % stack_id)
+
+ except exc.HTTPNotFound:
+ # once the stack is gone we can assume it was successfully deleted
+ # clear the exception so it doesn't confuse the logs
+ #if sys.version_info < (3, 0):
+ # sys.exc_clear()
+ LOG.info('Stack %s was successfully deleted', stack_id)
+
+
+def get_stack_outputs(heat_client, stack_id):
+ """
+ Get Stack Output
+ """
+ # try to use optimized way to retrieve outputs, fallback otherwise
+ if hasattr(heat_client.stacks, 'output_list'):
+ try:
+ output_list = heat_client.stacks.output_list(stack_id)['outputs']
+
+ result = {}
+ for output in output_list:
+ output_key = output['output_key']
+ value = heat_client.stacks.output_show(stack_id, output_key)
+ result[output_key] = value['output']['output_value']
+
+ return result
+ except BaseException as err:
+ LOG.info('Cannot get output list, fallback to old way: %s', err)
+
+ outputs_list = heat_client.stacks.get(stack_id).to_dict()['outputs']
+ return dict((item['output_key'], item['output_value'])
+ for item in outputs_list)
diff --git a/tools/os_deploy_tgen/osclients/neutron.py b/tools/os_deploy_tgen/osclients/neutron.py
new file mode 100644
index 00000000..f75077dc
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/neutron.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2015 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Neutron client
+"""
+
+from oslo_log import log as logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+def choose_external_net(neutron_client):
+ """
+ Choose External Network
+ """
+ ext_nets = neutron_client.list_networks(
+ **{'router:external': True})['networks']
+ if not ext_nets:
+ raise Exception('No external networks found')
+ return ext_nets[0]['name']
diff --git a/tools/os_deploy_tgen/osclients/nova.py b/tools/os_deploy_tgen/osclients/nova.py
new file mode 100644
index 00000000..b2baa34f
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/nova.py
@@ -0,0 +1,213 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Nova Client
+"""
+
+import itertools
+import re
+import time
+
+from novaclient import client as nova_client_pkg
+from oslo_log import log as logging
+
+LOG = logging.getLogger(__name__)
+
+
+class ForbiddenException(nova_client_pkg.exceptions.Forbidden):
+ """
+ Custome Exception
+ """
+
+
+
+def get_available_compute_nodes(nova_client, flavor_name):
+ """
+ Return available compute nodes
+ """
+ try:
+ host_list = [dict(host=svc.host, zone=svc.zone)
+ for svc in
+ nova_client.services.list(binary='nova-compute')
+ if svc.state == 'up' and svc.status == 'enabled']
+
+ # If the flavor has aggregate_instance_extra_specs set then filter
+ # host_list to pick only the hosts matching the chosen flavor.
+ flavor = get_flavor(nova_client, flavor_name)
+
+ if flavor is not None:
+ extra_specs = flavor.get_keys()
+
+ for item in extra_specs:
+ if "aggregate_instance_extra_specs" in item:
+ LOG.debug('Flavor contains %s, using compute node '
+ 'filtering', extra_specs)
+
+ # getting the extra spec seting for flavor in the
+ # standard format of extra_spec:value
+ extra_spec = item.split(":")[1]
+ extra_spec_value = extra_specs.get(item)
+
+ # create a set of aggregate host which match
+ agg_hosts = set(itertools.chain(
+ *[agg.hosts for agg in
+ nova_client.aggregates.list() if
+ agg.metadata.get(extra_spec) == extra_spec_value]))
+
+ # update list of available hosts with
+ # host_aggregate cross-check
+ host_list = [elem for elem in host_list if
+ elem['host'] in agg_hosts]
+
+ LOG.debug('Available compute nodes: %s ', host_list)
+
+ return host_list
+
+ except nova_client_pkg.exceptions.Forbidden as error:
+ msg = 'Forbidden to get list of compute nodes'
+ raise ForbiddenException(msg) from error
+
+
+def does_flavor_exist(nova_client, flavor_name):
+ """
+ Check if flavor exists
+ """
+ for flavor in nova_client.flavors.list():
+ if flavor.name == flavor_name:
+ return True
+ return False
+
+
+def create_flavor(nova_client, **kwargs):
+ """
+ Create a flavor
+ """
+ try:
+ nova_client.flavors.create(**kwargs)
+ except nova_client_pkg.exceptions.Forbidden as error:
+ msg = 'Forbidden to create flavor'
+ raise ForbiddenException(msg) from error
+
+
+def get_server_ip(nova_client, server_name, ip_type):
+ """
+ Get IP of the compute
+ """
+ server = nova_client.servers.find(name=server_name)
+ addresses = server.addresses
+ ips = [v['addr'] for v in itertools.chain(*addresses.values())
+ if v['OS-EXT-IPS:type'] == ip_type]
+ if not ips:
+ raise Exception('Could not get IP address of server: %s' % server_name)
+ if len(ips) > 1:
+ raise Exception('Server %s has more than one IP addresses: %s' %
+ (server_name, ips))
+ return ips[0]
+
+
+def get_server_host_id(nova_client, server_name):
+ """
+ Get the host id
+ """
+ server = nova_client.servers.find(name=server_name)
+ return server.hostId
+
+
+def check_server_console(nova_client, server_id, len_limit=100):
+ """
+ Check Server console
+ """
+ try:
+ console = (nova_client.servers.get(server_id)
+ .get_console_output(len_limit))
+ except nova_client_pkg.exceptions.ClientException as exc:
+ LOG.warning('Error retrieving console output: %s. Ignoring', exc)
+ return None
+
+ for line in console.splitlines():
+ if (re.search(r'\[critical\]', line, flags=re.IGNORECASE) or
+ re.search(r'Cloud-init.*Datasource DataSourceNone\.', line)):
+ message = ('Instance %(id)s has critical cloud-init error: '
+ '%(msg)s. Check metadata service availability' %
+ dict(id=server_id, msg=line))
+ LOG.error(message)
+ return message
+ if re.search(r'\[error', line, flags=re.IGNORECASE):
+ LOG.error('Error message in instance %(id)s console: %(msg)s',
+ dict(id=server_id, msg=line))
+ elif re.search(r'warn', line, flags=re.IGNORECASE):
+ LOG.info('Warning message in instance %(id)s console: %(msg)s',
+ dict(id=server_id, msg=line))
+
+ return None
+
+
+def _poll_for_status(nova_client, server_id, final_ok_states, poll_period=20,
+ status_field="status"):
+ """
+ Poll for status
+ """
+ LOG.debug('Poll instance %(id)s, waiting for any of statuses %(statuses)s',
+ dict(id=server_id, statuses=final_ok_states))
+ while True:
+ obj = nova_client.servers.get(server_id)
+
+ err_msg = check_server_console(nova_client, server_id)
+ if err_msg:
+ raise Exception('Critical error in instance %s console: %s' %
+ (server_id, err_msg))
+
+ status = getattr(obj, status_field)
+ if status:
+ status = status.lower()
+
+ LOG.debug('Instance %(id)s has status %(status)s',
+ dict(id=server_id, status=status))
+
+ if status in final_ok_states:
+ break
+ if status in ('error', 'paused'):
+ raise Exception(obj.fault['message'])
+
+ time.sleep(poll_period)
+
+
+def wait_server_shutdown(nova_client, server_id):
+ """
+ Wait server shutdown
+ """
+ _poll_for_status(nova_client, server_id, ['shutoff'])
+
+
+def wait_server_snapshot(nova_client, server_id):
+ """
+ Wait server snapshot
+ """
+ task_state_field = "OS-EXT-STS:task_state"
+ server = nova_client.servers.get(server_id)
+ if hasattr(server, task_state_field):
+ _poll_for_status(nova_client, server.id, [None, '-', ''],
+ status_field=task_state_field)
+
+
+def get_flavor(nova_client, flavor_name):
+ """
+ Get the flavor
+ """
+ for flavor in nova_client.flavors.list():
+ if flavor.name == flavor_name:
+ return flavor
+ return None
diff --git a/tools/os_deploy_tgen/osclients/openstack.py b/tools/os_deploy_tgen/osclients/openstack.py
new file mode 100644
index 00000000..58297e6c
--- /dev/null
+++ b/tools/os_deploy_tgen/osclients/openstack.py
@@ -0,0 +1,82 @@
+# Copyright (c) 2020 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Openstack Client - Main File
+"""
+
+import os_client_config
+from oslo_log import log as logging
+from oslo_utils import importutils
+
+LOG = logging.getLogger(__name__)
+
+
+class OpenStackClientException(Exception):
+ '''
+ Custom Exception
+ '''
+
+
+def init_profiling(os_profile):
+ """
+ Initialize Profiling
+ """
+ if os_profile:
+ osprofiler_profiler = importutils.try_import("osprofiler.profiler")
+
+ if osprofiler_profiler: # lib is present
+ osprofiler_profiler.init(os_profile)
+ trace_id = osprofiler_profiler.get().get_base_id()
+ LOG.info('Profiling is enabled, trace id: %s', trace_id)
+ else: # param is set, but lib is not present
+ LOG.warning('Profiling could not be enabled. To enable profiling '
+ 'please install "osprofiler" library')
+
+
+class OpenStackClient():
+ """
+ Client Class
+ """
+ def __init__(self, openstack_params):
+ """
+ Initialize
+ """
+ LOG.debug('Establishing connection to OpenStack')
+
+ init_profiling(openstack_params.get('os_profile'))
+
+ config = os_client_config.OpenStackConfig()
+ cloud_config = config.get_one_cloud(**openstack_params)
+ if openstack_params['os_insecure']:
+ cloud_config.config['verify'] = False
+ cloud_config.config['cacert'] = None
+ self.keystone_session = cloud_config.get_session()
+ self.nova = cloud_config.get_legacy_client('compute')
+ self.neutron = cloud_config.get_legacy_client('network')
+ self.glance = cloud_config.get_legacy_client('image')
+
+ # heat client wants endpoint to be always set
+ endpoint = cloud_config.get_session_endpoint('orchestration')
+ if not endpoint:
+ raise OpenStackClientException(
+ 'Endpoint for orchestration service is not found')
+ self.heat = cloud_config.get_legacy_client('orchestration',
+ endpoint=endpoint)
+
+ # Ping OpenStack
+ self.keystone_session.get_token()
+
+ LOG.info('Connection to OpenStack is initialized')
diff --git a/tools/os_deploy_tgen/osdt.py b/tools/os_deploy_tgen/osdt.py
new file mode 100644
index 00000000..0aad8597
--- /dev/null
+++ b/tools/os_deploy_tgen/osdt.py
@@ -0,0 +1,601 @@
+# Copyright 2020 Spirent Communications, Mirantis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Code to deploy Trafficgenerator on Openstack.
+This Code is based on Openstack Shaker.
+"""
+
+
+import collections
+import functools
+import random
+#import sys
+import os
+import copy
+import logging
+#import json
+import jinja2
+#import shutil
+#import datetime
+#import time
+
+#from conf import merge_spec
+from conf import settings as S
+
+from tools.os_deploy_tgen.utilities import utils
+from tools.os_deploy_tgen.osclients import heat
+from tools.os_deploy_tgen.osclients import neutron
+from tools.os_deploy_tgen.osclients import nova
+from tools.os_deploy_tgen.osclients import openstack
+
+LOG = logging.getLogger(__name__)
+_CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+
+class DeploymentException(Exception):
+ """ Exception Handling """
+
+
+def prepare_for_cross_az(compute_nodes, zones):
+ """
+ Deployment across Availability Zones
+ """
+ if len(zones) != 2:
+ LOG.info('cross_az is specified, but len(zones) is not 2')
+ return compute_nodes
+
+ masters = []
+ slaves = []
+ for node in compute_nodes:
+ if node['zone'] == zones[0]:
+ masters.append(node)
+ else:
+ slaves.append(node)
+
+ res = []
+ for i in range(min(len(masters), len(slaves))):
+ res.append(masters[i])
+ res.append(slaves[i])
+
+ return res
+
+
+def generate_agents(compute_nodes, accommodation, unique):
+ """
+ Generate TestVNF Instances
+ """
+ print('Number of compute nodes')
+ print(compute_nodes)
+ density = accommodation.get('density') or 1
+
+ zones = accommodation.get('zones')
+ if zones:
+ compute_nodes = [
+ c for c in compute_nodes if c['zone'] in zones or
+ ':'.join(filter(None, [c['zone'], c['host']])) in zones]
+ if 'cross_az' in accommodation:
+ compute_nodes = prepare_for_cross_az(compute_nodes, zones)
+
+ best_effort = accommodation.get('best_effort', False)
+ compute_nodes_requested = accommodation.get('compute_nodes')
+ if compute_nodes_requested:
+ if compute_nodes_requested > len(compute_nodes):
+ print(str(len(compute_nodes)))
+ if best_effort:
+ LOG.info('Allowing best_effort accommodation:')
+ else:
+ raise DeploymentException(
+ 'Exception Not enough compute nodes %(cn)s for requested '
+ 'instance accommodation %(acc)s' %
+ dict(cn=compute_nodes, acc=accommodation))
+ else:
+ compute_nodes = random.sample(compute_nodes,
+ compute_nodes_requested)
+
+ cn_count = len(compute_nodes)
+ iterations = cn_count * density
+ ite = 0
+ if 'single_room' in accommodation and 'pair' in accommodation:
+ # special case to allow pair, single_room on single compute node
+ if best_effort and iterations == 1:
+ LOG.info('Allowing best_effort accommodation: '
+ 'single_room, pair on one compute node')
+ else:
+ iterations //= 2
+ node_formula = lambda x: compute_nodes[x % cn_count]
+
+ agents = {}
+
+ for ite in range(iterations):
+ if 'pair' in accommodation:
+ master_id = '%s_master_%s' % (unique, ite)
+ slave_id = '%s_slave_%s' % (unique, ite)
+ master = dict(id=master_id, mode='master', slave_id=slave_id)
+ slave = dict(id=slave_id, mode='slave', master_id=master_id)
+
+ if 'single_room' in accommodation:
+ master_formula = lambda x: ite * 2
+ slave_formula = lambda x: ite * 2 + 1
+ elif 'double_room' in accommodation:
+ master_formula = lambda x: ite
+ slave_formula = lambda x: ite
+ else: # mixed_room
+ master_formula = lambda x: ite
+ slave_formula = lambda x: ite + 1
+
+ mas = node_formula(master_formula(ite))
+ master['node'], master['zone'] = mas['host'], mas['zone']
+ sla = node_formula(slave_formula(ite))
+ slave['node'], slave['zone'] = sla['host'], sla['zone']
+
+ agents[master['id']] = master
+ agents[slave['id']] = slave
+ else:
+ if 'single_room' in accommodation:
+ agent_id = '%s_agent_%s' % (unique, ite)
+ agents[agent_id] = dict(id=agent_id,
+ node=node_formula(ite)['host'],
+ zone=node_formula(ite)['zone'],
+ mode='alone')
+
+ if not agents:
+ raise DeploymentException('Not enough compute nodes %(cn)s for '
+ 'requested instance accommodation %(acc)s' %
+ dict(cn=compute_nodes, acc=accommodation))
+
+ # inject availability zone
+ for agent in agents.values():
+ avz = agent['zone']
+ if agent['node']:
+ avz += ':' + agent['node']
+ agent['availability_zone'] = avz
+
+ return agents
+
+
+def _get_stack_values(stack_outputs, vm_name, params):
+ """
+ Collect the output from Heat Stack Deployment
+ """
+ result = {}
+ for param in params:
+ out = stack_outputs.get(vm_name + '_' + param)
+ if out:
+ result[param] = out
+ return result
+
+
+def filter_agents(agents, stack_outputs, override=None):
+ """
+ Filter Deployed Instances - If Required.
+ """
+ deployed_agents = {}
+
+ # first pass, ignore non-deployed
+ for agent in agents.values():
+ stack_values = _get_stack_values(stack_outputs, agent['id'], ['ip'])
+ new_stack_values = _get_stack_values(stack_outputs, agent['id'], ['pip'])
+ mac_values = _get_stack_values(stack_outputs, agent['id'], ['dmac'])
+
+ if override:
+ stack_values.update(override(agent))
+
+ if not stack_values.get('ip'):
+ LOG.info('Ignore non-deployed agent: %s', agent)
+ continue
+
+ if not new_stack_values.get('pip'):
+ LOG.info('Ignore non-deployed agent: %s', agent)
+ continue
+
+ if not mac_values.get('dmac'):
+ LOG.info('Ignore non-deployed agent: %s', agent)
+ continue
+
+ agent.update(stack_values)
+ agent.update(new_stack_values)
+
+ # workaround of Nova bug 1422686
+ if agent.get('mode') == 'slave' and not agent.get('ip'):
+ LOG.info('IP address is missing in agent: %s', agent)
+ continue
+
+ deployed_agents[agent['id']] = agent
+
+ # second pass, check pairs
+ result = {}
+ for agent in deployed_agents.values():
+ print(agent.get('mode'))
+ print(agent.get('ip'))
+ print(agent.get('pip'))
+ print(agent.get('dmac'))
+ if (agent.get('mode') == 'alone' or
+ (agent.get('mode') == 'master' and
+ agent.get('slave_id') in deployed_agents) or
+ (agent.get('mode') == 'slave' and
+ agent.get('master_id') in deployed_agents)):
+ result[agent['id']] = agent
+
+ return result
+
+
+def distribute_agents(agents, get_host_fn):
+ """
+ Distribute TestVNF Instances
+ """
+ result = {}
+
+ hosts = set()
+ buckets = collections.defaultdict(list)
+ for agent in agents.values():
+ agent_id = agent['id']
+ # we assume that server name equals to agent_id
+ host_id = get_host_fn(agent_id)
+
+ if host_id not in hosts:
+ hosts.add(host_id)
+ agent['node'] = host_id
+ buckets[agent['mode']].append(agent)
+ else:
+ LOG.info('Filter out agent %s, host %s is already occupied',
+ agent_id, host_id)
+
+ if buckets['alone']:
+ result = dict((a['id'], a) for a in buckets['alone'])
+ else:
+ for master, slave in zip(buckets['master'], buckets['slave']):
+ master['slave_id'] = slave['id']
+ slave['master_id'] = master['id']
+
+ result[master['id']] = master
+ result[slave['id']] = slave
+
+ return result
+
+
+def normalize_accommodation(accommodation):
+ """
+ Planning the Accomodation of TestVNFs
+ """
+ result = {}
+
+ for stk in accommodation:
+ if isinstance(stk, dict):
+ result.update(stk)
+ else:
+ result[stk] = True
+
+ # override scenario's availability zone accommodation
+ if S.hasValue('SCENARIO_AVAILABILITY_ZONE'):
+ result['zones'] = S.getValue('SCENARIO_AVAILABILITY_ZONE')
+ # override scenario's compute_nodes accommodation
+ if S.hasValue('SCENARIO_COMPUTE_NODES'):
+ result['compute_nodes'] = S.getValue('SCENARIO_COMPUTE_NODES')
+
+ return result
+
+
+class Deployment():
+ """
+ Main Deployment Class
+ """
+ def __init__(self):
+ """
+ Initialize
+ """
+ self.openstack_client = None
+ self.stack_id = None
+ self.privileged_mode = True
+ self.flavor_name = None
+ self.image_name = None
+ self.stack_name = None
+ self.external_net = None
+ self.dns_nameservers = None
+ # The current run "owns" the support stacks, it is tracked
+ # so it can be deleted later.
+ self.support_stacks = []
+ self.trackstack = collections.namedtuple('TrackStack', 'name id')
+
+ def connect_to_openstack(self, openstack_params, flavor_name, image_name,
+ external_net, dns_nameservers):
+ """
+ Connect to Openstack
+ """
+ LOG.debug('Connecting to OpenStack')
+
+ self.openstack_client = openstack.OpenStackClient(openstack_params)
+
+ self.flavor_name = flavor_name
+ self.image_name = image_name
+
+ if S.hasValue('STACK_NAME'):
+ self.stack_name = S.getValue('STACK_NAME')
+ else:
+ self.stack_name = 'testvnf_%s' % utils.random_string()
+
+ self.dns_nameservers = dns_nameservers
+ # intiailizing self.external_net last so that other attributes don't
+ # remain uninitialized in case user forgets to create external network
+ self.external_net = (external_net or
+ neutron.choose_external_net(
+ self.openstack_client.neutron))
+
+ def _get_compute_nodes(self, accommodation):
+ """
+ Get available comput nodes
+ """
+ try:
+ comps = nova.get_available_compute_nodes(self.openstack_client.nova,
+ self.flavor_name)
+ print(comps)
+ return comps
+ except nova.ForbiddenException:
+ # user has no permissions to list compute nodes
+ LOG.info('OpenStack user does not have permission to list compute '
+ 'nodes - treat him as non-admin')
+ self.privileged_mode = False
+ count = accommodation.get('compute_nodes')
+ if not count:
+ raise DeploymentException(
+ 'When run with non-admin user the scenario must specify '
+ 'number of compute nodes to use')
+
+ zones = accommodation.get('zones') or ['nova']
+ return [dict(host=None, zone=zones[n % len(zones)])
+ for n in range(count)]
+
+ #def _deploy_from_hot(self, specification, server_endpoint, base_dir=None):
+ def _deploy_from_hot(self, specification, base_dir=None):
+ """
+ Perform Heat stack deployment
+ """
+ accommodation = normalize_accommodation(
+ specification.get('accommodation') or
+ specification.get('vm_accommodation'))
+
+ agents = generate_agents(self._get_compute_nodes(accommodation),
+ accommodation, self.stack_name)
+
+ # render template by jinja
+ vars_values = {
+ 'agents': agents,
+ 'unique': self.stack_name,
+ }
+ heat_template = utils.read_file(specification['template'],
+ base_dir=base_dir)
+ compiled_template = jinja2.Template(heat_template)
+ rendered_template = compiled_template.render(vars_values)
+ LOG.info('Rendered template: %s', rendered_template)
+
+ # create stack by Heat
+ try:
+ merged_parameters = {
+ 'external_net': self.external_net,
+ 'image': self.image_name,
+ 'flavor': self.flavor_name,
+ 'dns_nameservers': self.dns_nameservers,
+ }
+ except AttributeError as err:
+ LOG.error('Failed to gather required parameters to create '
+ 'heat stack: %s', err)
+ raise
+
+ merged_parameters.update(specification.get('template_parameters', {}))
+ try:
+ self.stack_id = heat.create_stack(
+ self.openstack_client.heat, self.stack_name,
+ rendered_template, merged_parameters, None)
+ except heat.exc.StackFailure as err:
+ self.stack_id = err.args[0]
+ raise
+
+ # get info about deployed objects
+ outputs = heat.get_stack_outputs(self.openstack_client.heat,
+ self.stack_id)
+ override = self._get_override(specification.get('override'))
+
+ agents = filter_agents(agents, outputs, override)
+
+ if (not self.privileged_mode) and accommodation.get('density', 1) == 1:
+ get_host_fn = functools.partial(nova.get_server_host_id,
+ self.openstack_client.nova)
+ agents = distribute_agents(agents, get_host_fn)
+
+ return agents
+
+ def _get_override(self, override_spec):
+ """
+ Collect the overrides
+ """
+ def override_ip(agent, ip_type):
+ """
+ Override the IP
+ """
+ return dict(ip=nova.get_server_ip(
+ self.openstack_client.nova, agent['id'], ip_type))
+
+ if override_spec:
+ if override_spec.get('ip'):
+ return functools.partial(override_ip,
+ ip_type=override_spec.get('ip'))
+
+
+ #def deploy(self, deployment, base_dir=None, server_endpoint=None):
+ def deploy(self, deployment, base_dir=None):
+ """
+ Perform Deployment
+ """
+ agents = {}
+
+ if not deployment:
+ # local mode, create fake agent
+ agents.update(dict(local=dict(id='local', mode='alone',
+ node='localhost')))
+
+ if deployment.get('template'):
+ if self.openstack_client:
+ # deploy topology specified by HOT
+ agents.update(self._deploy_from_hot(
+ #deployment, server_endpoint, base_dir=base_dir))
+ deployment, base_dir=base_dir))
+ else:
+ raise DeploymentException(
+ 'OpenStack client is not initialized. '
+ 'Template-based deployment is ignored.')
+
+ if not agents:
+ print("No VM Deployed - Deploy")
+ raise Exception('No agents deployed.')
+
+ if deployment.get('agents'):
+ # agents are specified statically
+ agents.update(dict((a['id'], a) for a in deployment.get('agents')))
+
+ return agents
+
+def read_scenario(scenario_name):
+ """
+ Collect all Information about the scenario
+ """
+ scenario_file_name = scenario_name
+ LOG.debug('Scenario %s is resolved to %s', scenario_name,
+ scenario_file_name)
+
+ scenario = utils.read_yaml_file(scenario_file_name)
+
+ schema = utils.read_yaml_file(S.getValue('SCHEMA'))
+ utils.validate_yaml(scenario, schema)
+
+ scenario['title'] = scenario.get('title') or scenario_file_name
+ scenario['file_name'] = scenario_file_name
+
+ return scenario
+
+def _extend_agents(agents_map):
+ """
+ Add More info to deployed Instances
+ """
+ extended_agents = {}
+ for agent in agents_map.values():
+ extended = copy.deepcopy(agent)
+ if agent.get('slave_id'):
+ extended['slave'] = copy.deepcopy(agents_map[agent['slave_id']])
+ if agent.get('master_id'):
+ extended['master'] = copy.deepcopy(agents_map[agent['master_id']])
+ extended_agents[agent['id']] = extended
+ return extended_agents
+
+def play_scenario(scenario):
+ """
+ Deploy a scenario
+ """
+ deployment = None
+ output = dict(scenarios={}, agents={})
+ output['scenarios'][scenario['title']] = scenario
+
+ try:
+ deployment = Deployment()
+
+ openstack_params = utils.pack_openstack_params()
+ try:
+ deployment.connect_to_openstack(
+ openstack_params, S.getValue('FLAVOR_NAME'),
+ S.getValue('IMAGE_NAME'), S.getValue('EXTERNAL_NET'),
+ S.getValue('DNS_NAMESERVERS'))
+ except BaseException as excep:
+ LOG.warning('Failed to connect to OpenStack: %s. Please '
+ 'verify parameters: %s', excep, openstack_params)
+
+ base_dir = os.path.dirname(scenario['file_name'])
+ scenario_deployment = scenario.get('deployment', {})
+ agents = deployment.deploy(scenario_deployment, base_dir=base_dir)
+
+ if not agents:
+ print("No VM Deployed - Play-Scenario")
+ raise Exception('No agents deployed.')
+
+ agents = _extend_agents(agents)
+ output['agents'] = agents
+ LOG.debug('Deployed agents: %s', agents)
+ print(agents)
+
+ if not agents:
+ raise Exception('No agents deployed.')
+
+ except BaseException as excep:
+ if isinstance(excep, KeyboardInterrupt):
+ LOG.info('Caught SIGINT. Terminating')
+ # record = dict(id=utils.make_record_id(), status='interrupted')
+ else:
+ error_msg = 'Error while executing scenario: %s' % excep
+ LOG.exception(error_msg)
+ return output
+
+def act():
+ """
+ Kickstart the Scenario Deployment
+ """
+ for scenario_name in S.getValue('SCENARIOS'):
+ LOG.info('Play scenario: %s', scenario_name)
+ print('Play scenario: {}'.format(scenario_name))
+ scenario = read_scenario(scenario_name)
+ play_output = play_scenario(scenario)
+ print(play_output)
+ return play_output
+ return None
+
+def update_vsperf_configuration(agents):
+ """
+ Create Configuration file for VSPERF.
+ """
+ tgen = S.getValue('TRAFFICGEN')
+ east_chassis_ip = agents[0]['public_ip']
+ # east_data_ip = agents[0]['private_ip']
+ if len(agents) == 2:
+ west_chassis_ip = agents[1]['public_ip']
+ # west_data_ip = agents[1]['private_ip']
+ else:
+ west_chassis_ip = east_chassis_ip
+ # west_data_ip = east_chassis_ip
+ if "TestCenter" in tgen:
+ S.setValue('TRAFFICGEN_STC_EAST_CHASSIS_ADDR', east_chassis_ip)
+ S.setValue('TRAFFICGEN_STC_WEST_CHASSIS_ADDR', west_chassis_ip)
+ if "Ix" in tgen:
+ S.setValue("TRAFFICGEN_EAST_IXIA_HOST", east_chassis_ip)
+ S.setValue("TRAFFICGEN_WEST_IXIA_HOST", west_chassis_ip)
+
+def deploy_testvnf():
+ """
+ Starting function.
+ """
+ output = act()
+ list_of_agents = []
+ if output:
+ for count in range(len(output['agents'])):
+ # ag_dict = collections.defaultdict()
+ name = str(list(output['agents'].keys())[count])
+ private_ip = output['agents'][name]['ip']
+ public_ip = output['agents'][name]['pip']
+ node = output['agents'][name]['node']
+ list_of_agents.append({'name': name,
+ 'private_ip': private_ip,
+ 'public_ip': public_ip,
+ 'compute_node': node})
+ if list_of_agents:
+ update_vsperf_configuration(list_of_agents)
+ return True
+ return False
+
+if __name__ == "__main__":
+ deploy_testvnf()
diff --git a/tools/os_deploy_tgen/templates/hotfiles.md b/tools/os_deploy_tgen/templates/hotfiles.md
new file mode 100644
index 00000000..6e21157e
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/hotfiles.md
@@ -0,0 +1,13 @@
+# How to use these HOT Files.
+
+These hot files are referenced in the yaml files.
+Please ensure you are using correct HOT file.
+
+## L2 - No Routers are setup - Same Subnet.
+
+l2fip.hot - Floating IP is configured. Use this if the Openstack environment supports floating IP.
+l2up - Use this if you want username and password configured for the TestVNFs.
+l2.hot - Use this if the 2 interfaces has fixed IPs from 2 different networks. This applies when TestVNF has connectivity to provider network.
+
+## L3 - Routers are setup - Different Subnets
+l3.hot - Setup TestVNFs on two different subnet and connect them with a router.
diff --git a/tools/os_deploy_tgen/templates/l2.hot b/tools/os_deploy_tgen/templates/l2.hot
new file mode 100644
index 00000000..226e8433
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2.hot
@@ -0,0 +1,89 @@
+heat_template_version: 2013-05-23
+
+description:
+ This Heat template creates a new Neutron network, a router to the external
+ network and plugs instances into this new network. All instances are located
+ in the same L2 domain.
+
+parameters:
+ image:
+ type: string
+ description: Name of image to use for servers
+ flavor:
+ type: string
+ description: Flavor to use for servers
+ external_net:
+ type: string
+ description: ID or name of external network
+# server_endpoint:
+# type: string
+# description: Server endpoint address
+ dns_nameservers:
+ type: comma_delimited_list
+ description: DNS nameservers for the subnet
+
+resources:
+ private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net
+
+ private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: private_net }
+ cidr: 172.172.172.0/24
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router:
+ type: OS::Neutron::Router
+ properties:
+ external_gateway_info:
+ network: { get_param: external_net }
+
+ router_interface:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: private_subnet }
+
+{% for agent in agents.values() %}
+
+ {{ agent.id }}:
+ type: OS::Nova::Server
+ properties:
+ name: {{ agent.id }}
+ image: { get_param: image }
+ flavor: { get_param: flavor }
+ availability_zone: "{{ agent.availability_zone }}"
+ networks:
+ - port: { get_resource: {{ agent.id }}_port }
+ - port: { get_resource: {{ agent.id }}_mgmt_port }
+
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: private_net }
+ fixed_ips:
+ - subnet_id: { get_resource: private_subnet }
+
+ {{ agent.id }}_mgmt_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_param: external_net }
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+ {{ agent.id }}_instance_name:
+ value: { get_attr: [ {{ agent.id }}, instance_name ] }
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}_port, fixed_ips, 0, ip_address ] }
+# value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+ {{ agent.id }}_pip:
+ value: { get_attr: [ {{ agent.id }}_mgmt_port, fixed_ips, 0, ip_address ] }
+ {{ agent.id }}_dmac:
+ value: { get_attr: [ {{ agent.id }}_port, mac_address ] }
+
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2_1c_1i.yaml b/tools/os_deploy_tgen/templates/l2_1c_1i.yaml
new file mode 100644
index 00000000..ec931107
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_1c_1i.yaml
@@ -0,0 +1,8 @@
+title: OpenStack L2 Performance
+
+description:
+ In this scenario tdep launches single instances on a tenant network.
+
+deployment:
+ template: l2.hot
+ accommodation: [single_room, compute_nodes: 1]
diff --git a/tools/os_deploy_tgen/templates/l2_1c_2i.yaml b/tools/os_deploy_tgen/templates/l2_1c_2i.yaml
new file mode 100644
index 00000000..4241a80c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_1c_2i.yaml
@@ -0,0 +1,10 @@
+title: OpenStack L2 Performance
+
+description:
+ In this scenario tdep launches 1 pair of instances in the same tenant
+ network. Both the instances are hosted on same compute node.
+ The traffic goes within the tenant network (L2 domain).
+
+deployment:
+ template: l2up.hot
+ accommodation: [pair, single_room, best_effort, compute_nodes: 1]
diff --git a/tools/os_deploy_tgen/templates/l2_2c_2i.yaml b/tools/os_deploy_tgen/templates/l2_2c_2i.yaml
new file mode 100644
index 00000000..b1f54f0a
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_2c_2i.yaml
@@ -0,0 +1,10 @@
+title: OpenStack L2 Performance
+
+description:
+ In this scenario tdep launches 1 pair of instances in the same tenant
+ network. Each instance is hosted on a separate compute node. The traffic goes
+ within the tenant network (L2 domain).
+
+deployment:
+ template: l2fip.hot
+ accommodation: [pair, single_room, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/l2_old.hot b/tools/os_deploy_tgen/templates/l2_old.hot
new file mode 100644
index 00000000..d2553d76
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2_old.hot
@@ -0,0 +1,93 @@
+heat_template_version: 2013-05-23
+
+description:
+ This Heat template creates a new Neutron network, a router to the external
+ network and plugs instances into this new network. All instances are located
+ in the same L2 domain.
+
+parameters:
+ image:
+ type: string
+ description: Name of image to use for servers
+ flavor:
+ type: string
+ description: Flavor to use for servers
+ external_net:
+ type: string
+ description: ID or name of external network
+# server_endpoint:
+# type: string
+# description: Server endpoint address
+ dns_nameservers:
+ type: comma_delimited_list
+ description: DNS nameservers for the subnet
+
+resources:
+ private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net
+
+ private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: private_net }
+ cidr: 10.0.0.0/16
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router:
+ type: OS::Neutron::Router
+ properties:
+ external_gateway_info:
+ network: { get_param: external_net }
+
+ router_interface:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: private_subnet }
+
+ server_security_group:
+ type: OS::Neutron::SecurityGroup
+ properties:
+ rules: [
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: tcp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: udp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+ {{ agent.id }}:
+ type: OS::Nova::Server
+ properties:
+ name: {{ agent.id }}
+ image: { get_param: image }
+ flavor: { get_param: flavor }
+ availability_zone: "{{ agent.availability_zone }}"
+ networks:
+ - port: { get_resource: {{ agent.id }}_port }
+
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: private_net }
+ fixed_ips:
+ - subnet_id: { get_resource: private_subnet }
+ security_groups: [{ get_resource: server_security_group }]
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+ {{ agent.id }}_instance_name:
+ value: { get_attr: [ {{ agent.id }}, instance_name ] }
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2fip.hot b/tools/os_deploy_tgen/templates/l2fip.hot
new file mode 100644
index 00000000..4d4b52f7
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2fip.hot
@@ -0,0 +1,122 @@
+heat_template_version: 2013-05-23
+
+description:
+ This Heat template creates a new Neutron network, a router to the external
+ network and plugs instances into this new network. All instances are located
+ in the same L2 domain.
+
+parameters:
+ image:
+ type: string
+ description: Name of image to use for servers
+ flavor:
+ type: string
+ description: Flavor to use for servers
+ external_net:
+ type: string
+ description: ID or name of external network
+# server_endpoint:
+# type: string
+# description: Server endpoint address
+ dns_nameservers:
+ type: comma_delimited_list
+ description: DNS nameservers for the subnet
+
+resources:
+ user_config:
+ type: OS::Heat::CloudConfig
+ properties:
+ cloud_config:
+ spirent:
+ driver: "sockets"
+
+ private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net
+ port_security_enabled: false
+
+ private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: private_net }
+ cidr: 172.172.172.0/24
+ dns_nameservers: { get_param: dns_nameservers }
+
+ private_datanet:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_datanet
+ port_security_enabled: false
+
+ private_datasubnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: private_datanet }
+ cidr: 172.172.168.0/24
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router:
+ type: OS::Neutron::Router
+ properties:
+ external_gateway_info:
+ network: { get_param: external_net }
+
+ router_interface:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: private_subnet }
+
+{% for agent in agents.values() %}
+
+ {{ agent.id }}:
+ type: OS::Nova::Server
+ properties:
+ name: {{ agent.id }}
+ image: { get_param: image }
+ flavor: { get_param: flavor }
+ availability_zone: "{{ agent.availability_zone }}"
+ networks:
+ - port: { get_resource: {{ agent.id }}_port }
+ - port: { get_resource: {{ agent.id }}_dataport }
+
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: private_net }
+ port_security_enabled: false
+ fixed_ips:
+ - subnet_id: { get_resource: private_subnet }
+
+ {{ agent.id }}_dataport:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: private_datanet }
+ port_security_enabled: false
+ fixed_ips:
+ - subnet_id: { get_resource: private_datasubnet }
+
+ {{ agent.id }}_fip_port:
+ type: OS::Neutron::FloatingIP
+ depends_on:
+ - router_interface
+ properties:
+ floating_network: { get_param: external_net }
+ port_id: { get_resource: {{ agent.id }}_port }
+
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+ {{ agent.id }}_instance_name:
+ value: { get_attr: [ {{ agent.id }}, instance_name ] }
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}_dataport, fixed_ips, 0, ip_address ] }
+ {{ agent.id }}_pip:
+ value: { get_attr: [ {{ agent.id }}_fip_port, floating_ip_address ] }
+ {{ agent.id }}_dmac:
+ value: { get_attr: [ {{ agent.id }}_dataport, mac_address ] }
+
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l2up.hot b/tools/os_deploy_tgen/templates/l2up.hot
new file mode 100644
index 00000000..58f25831
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l2up.hot
@@ -0,0 +1,126 @@
+heat_template_version: 2013-05-23
+
+description:
+ This Heat template creates a new Neutron network, a router to the external
+ network and plugs instances into this new network. All instances are located
+ in the same L2 domain.
+
+parameters:
+ image:
+ type: string
+ description: Name of image to use for servers
+ flavor:
+ type: string
+ description: Flavor to use for servers
+ external_net:
+ type: string
+ description: ID or name of external network
+# server_endpoint:
+# type: string
+# description: Server endpoint address
+ dns_nameservers:
+ type: comma_delimited_list
+ description: DNS nameservers for the subnet
+
+resources:
+ private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net
+
+ private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: private_net }
+ cidr: 172.172.172.0/24
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router:
+ type: OS::Neutron::Router
+ properties:
+ external_gateway_info:
+ network: { get_param: external_net }
+
+ router_interface:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: private_subnet }
+
+ user_config:
+ type: OS::Heat::CloudConfig
+ properties:
+ cloud_config:
+ users:
+ - default
+ - name: test
+ groups: "users,root"
+ lock-passwd: false
+ passwd: 'test'
+ shell: "/bin/bash"
+ sudo: "ALL=(ALL) NOPASSWD:ALL"
+ ssh_pwauth: true
+ chpasswd:
+ list: |
+ test:test
+ expire: False
+
+ server_security_group:
+ type: OS::Neutron::SecurityGroup
+ properties:
+ rules: [
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: tcp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: udp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+ {{ agent.id }}:
+ type: OS::Nova::Server
+ properties:
+ name: {{ agent.id }}
+ image: { get_param: image }
+ flavor: { get_param: flavor }
+ availability_zone: "{{ agent.availability_zone }}"
+ networks:
+ - port: { get_resource: {{ agent.id }}_port }
+ - port: { get_resource: {{ agent.id }}_mgmt_port }
+ user_data: {get_resource: user_config}
+ user_data_format: RAW
+
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: private_net }
+ fixed_ips:
+ - subnet_id: { get_resource: private_subnet }
+ security_groups: [{ get_resource: server_security_group }]
+
+ {{ agent.id }}_mgmt_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_param: external_net }
+ security_groups: [{ get_resource: server_security_group }]
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+ {{ agent.id }}_instance_name:
+ value: { get_attr: [ {{ agent.id }}, instance_name ] }
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}_port, fixed_ips, 0, ip_address ] }
+# value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [private_net, name] }, 0 ] }
+ {{ agent.id }}_pip:
+ value: { get_attr: [ {{ agent.id }}_mgmt_port, fixed_ips, 0, ip_address ] }
+ {{ agent.id }}_dmac:
+ value: { get_attr: [ {{ agent.id }}_port, mac_address ] }
+
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l3.hot b/tools/os_deploy_tgen/templates/l3.hot
new file mode 100644
index 00000000..4a5ea02c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3.hot
@@ -0,0 +1,125 @@
+heat_template_version: 2013-05-23
+
+description:
+ This Heat template creates a pair of networks plugged into the same router.
+ Master instances and slave instances are connected into different networks.
+
+parameters:
+ image:
+ type: string
+ description: Name of image to use for servers
+ flavor:
+ type: string
+ description: Flavor to use for servers
+ external_net:
+ type: string
+ description: ID or name of external network for which floating IP addresses will be allocated
+# server_endpoint:
+# type: string
+# description: Server endpoint address
+ dns_nameservers:
+ type: comma_delimited_list
+ description: DNS nameservers for the subnets
+
+resources:
+ east_private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net_east
+
+ east_private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: east_private_net }
+ cidr: 10.1.0.0/16
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router:
+ type: OS::Neutron::Router
+ properties:
+ external_gateway_info:
+ network: { get_param: external_net }
+
+ router_interface:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: east_private_subnet }
+
+ west_private_net:
+ type: OS::Neutron::Net
+ properties:
+ name: {{ unique }}_net_west
+
+ west_private_subnet:
+ type: OS::Neutron::Subnet
+ properties:
+ network_id: { get_resource: west_private_net }
+ cidr: 10.2.0.0/16
+ dns_nameservers: { get_param: dns_nameservers }
+
+ router_interface_2:
+ type: OS::Neutron::RouterInterface
+ properties:
+ router_id: { get_resource: router }
+ subnet_id: { get_resource: west_private_subnet }
+
+ server_security_group:
+ type: OS::Neutron::SecurityGroup
+ properties:
+ rules: [
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: tcp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: udp,
+ port_range_min: 1,
+ port_range_max: 65535},
+ {remote_ip_prefix: 0.0.0.0/0,
+ protocol: icmp}]
+
+{% for agent in agents.values() %}
+
+ {{ agent.id }}:
+ type: OS::Nova::Server
+ properties:
+ name: {{ agent.id }}
+ image: { get_param: image }
+ flavor: { get_param: flavor }
+ availability_zone: "{{ agent.availability_zone }}"
+ networks:
+ - port: { get_resource: {{ agent.id }}_port }
+
+{% if agent.mode == 'master' %}
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: east_private_net }
+ fixed_ips:
+ - subnet_id: { get_resource: east_private_subnet }
+ security_groups: [{ get_resource: server_security_group }]
+{% else %}
+ {{ agent.id }}_port:
+ type: OS::Neutron::Port
+ properties:
+ network_id: { get_resource: west_private_net }
+ fixed_ips:
+ - subnet_id: { get_resource: west_private_subnet }
+ security_groups: [{ get_resource: server_security_group }]
+{% endif %}
+
+{% endfor %}
+
+outputs:
+{% for agent in agents.values() %}
+ {{ agent.id }}_instance_name:
+ value: { get_attr: [ {{ agent.id }}, instance_name ] }
+{% if agent.mode == 'master' %}
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [east_private_net, name] }, 0 ] }
+{% else %}
+ {{ agent.id }}_ip:
+ value: { get_attr: [ {{ agent.id }}, networks, { get_attr: [west_private_net, name] }, 0 ] }
+{% endif %}
+{% endfor %}
diff --git a/tools/os_deploy_tgen/templates/l3_1c_2i.yaml b/tools/os_deploy_tgen/templates/l3_1c_2i.yaml
new file mode 100644
index 00000000..0908843c
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3_1c_2i.yaml
@@ -0,0 +1,11 @@
+title: OpenStack L3 East-West Performance
+
+description:
+ In this scenario tdep launches 1 pair of instances, both instances on same
+ compute node. Instances are connected to one of 2 tenant networks, which
+ plugged into single router. The traffic goes from one network to the other
+ (L3 east-west).
+
+deployment:
+ template: l3.hot
+ accommodation: [pair, single_room, best_effort, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/l3_2c_2i.yaml b/tools/os_deploy_tgen/templates/l3_2c_2i.yaml
new file mode 100644
index 00000000..67aee170
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/l3_2c_2i.yaml
@@ -0,0 +1,11 @@
+title: OpenStack L3 East-West Performance
+
+description:
+ In this scenario tdep launches 1 pair of instances, each instance on its own
+ compute node. Instances are connected to one of 2 tenant networks, which
+ plugged into single router. The traffic goes from one network to the other
+ (L3 east-west).
+
+deployment:
+ template: l3.hot
+ accommodation: [pair, single_room, compute_nodes: 2]
diff --git a/tools/os_deploy_tgen/templates/scenario.yaml b/tools/os_deploy_tgen/templates/scenario.yaml
new file mode 100644
index 00000000..c66ec734
--- /dev/null
+++ b/tools/os_deploy_tgen/templates/scenario.yaml
@@ -0,0 +1,44 @@
+name: tdep scenario schema
+type: map
+allowempty: True
+mapping:
+ title:
+ type: str
+ description:
+ type: str
+ deployment:
+ type: map
+ mapping:
+ support_templates:
+ type: seq
+ sequence:
+ - type: map
+ mapping:
+ name:
+ type: str
+ template:
+ type: str
+ env_file:
+ type: str
+ template:
+ type: str
+ env_file:
+ type: str
+ agents:
+ type: any
+ accommodation:
+ type: seq
+ matching: any
+ sequence:
+ - type: str
+ enum: [pair, alone, double_room, single_room, mixed_room, cross_az, best_effort]
+ - type: map
+ mapping:
+ density:
+ type: number
+ compute_nodes:
+ type: number
+ zones:
+ type: seq
+ sequence:
+ - type: str
diff --git a/tools/os_deploy_tgen/utilities/__init__.py b/tools/os_deploy_tgen/utilities/__init__.py
new file mode 100644
index 00000000..56f22a9e
--- /dev/null
+++ b/tools/os_deploy_tgen/utilities/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2020 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utilities package
+"""
diff --git a/tools/os_deploy_tgen/utilities/utils.py b/tools/os_deploy_tgen/utilities/utils.py
new file mode 100644
index 00000000..5208fd2a
--- /dev/null
+++ b/tools/os_deploy_tgen/utilities/utils.py
@@ -0,0 +1,183 @@
+# Copyright 2020 Spirent Communications, Mirantis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Utilities for deploying Trafficgenerator on Openstack.
+This Code is based on Openstack Shaker.
+"""
+
+#import errno
+#import functools
+import logging
+import os
+import random
+import re
+import uuid
+#import collections
+import yaml
+from pykwalify import core as pykwalify_core
+from pykwalify import errors as pykwalify_errors
+
+from conf import settings as S
+
+LOG = logging.getLogger(__name__)
+
+def read_file(file_name, base_dir=''):
+ """
+ Read Files
+ """
+ full_path = os.path.normpath(os.path.join(base_dir, file_name))
+
+ if not os.path.exists(full_path):
+ full_path = os.path.normpath(os.path.join('tools',
+ 'os_deploy_tgen',
+ file_name))
+ if not os.path.exists(full_path):
+ full_path = os.path.normpath(os.path.join('tools',
+ 'os_deploy_tgen',
+ 'templates',
+ file_name))
+ if not os.path.exists(full_path):
+ msg = ('File %s not found by absolute nor by relative path' %
+ file_name)
+ LOG.error(msg)
+ raise IOError(msg)
+
+ fid = None
+ try:
+ fid = open(full_path)
+ return fid.read()
+ except IOError as exc:
+ LOG.error('Error reading file: %s', exc)
+ raise
+ finally:
+ if fid:
+ fid.close()
+
+
+def write_file(data, file_name, base_dir=''):
+ """
+ Write to file
+ """
+ full_path = os.path.normpath(os.path.join(base_dir, file_name))
+ fid = None
+ try:
+ fid = open(full_path, 'w')
+ return fid.write(data)
+ except IOError as err:
+ LOG.error('Error writing file: %s', err)
+ raise
+ finally:
+ if fid:
+ fid.close()
+
+
+def read_yaml_file(file_name):
+ """
+ Read Yaml File
+ """
+ raw = read_file(file_name)
+ return read_yaml(raw)
+
+
+def read_yaml(raw):
+ """
+ Read YAML
+ """
+ try:
+ parsed = yaml.safe_load(raw)
+ return parsed
+ except Exception as error:
+ LOG.error('Failed to parse input %(yaml)s in YAML format: %(err)s',
+ dict(yaml=raw, err=error))
+ raise
+
+
+def split_address(address):
+ """
+ Split addresses
+ """
+ try:
+ host, port = address.split(':')
+ except ValueError:
+ LOG.error('Invalid address: %s, "host:port" expected', address)
+ raise
+ return host, port
+
+
+def random_string(length=6):
+ """
+ Generate Random String
+ """
+ return ''.join(random.sample('adefikmoprstuz', length))
+
+
+def make_record_id():
+ """
+ Create record-ID
+ """
+ return str(uuid.uuid4())
+
+def strict(strc):
+ """
+ Strict Check
+ """
+ return re.sub(r'[^\w\d]+', '_', re.sub(r'\(.+\)', '', strc)).lower()
+
+
+def validate_yaml(data, schema):
+ """
+ Validate Yaml
+ """
+ cor = pykwalify_core.Core(source_data=data, schema_data=schema)
+ try:
+ cor.validate(raise_exception=True)
+ except pykwalify_errors.SchemaError as err:
+ raise Exception('File does not conform to schema') from err
+
+
+def pack_openstack_params():
+ """
+ Packe Openstack Parameters
+ """
+ if not S.hasValue('OS_AUTH_URL'):
+ raise Exception(
+ 'OpenStack authentication endpoint is missing')
+
+ params = dict(auth=dict(username=S.getValue('OS_USERNAME'),
+ password=S.getValue('OS_PASSWORD'),
+ auth_url=S.getValue('OS_AUTH_URL')),
+ os_region_name=S.getValue('OS_REGION_NAME'),
+ os_cacert=S.getValue('OS_CA_CERT'),
+ os_insecure=S.getValue('OS_INSECURE'))
+
+ if S.hasValue('OS_PROJECT_NAME'):
+ value = S.getValue('OS_PROJECT_NAME')
+ params['auth']['project_name'] = value
+ if S.hasValue('OS_PROJECT_DOMAIN_NAME'):
+ value = S.getValue('OS_PROJECT_DOMAIN_NAME')
+ params['auth']['project_domain_name'] = value
+ if S.hasValue('OS_USER_DOMAIN_NAME'):
+ value = S.getValue('OS_USER_DOMAIN_NAME')
+ params['auth']['user_domain_name'] = value
+ if S.hasValue('OS_INTERFACE'):
+ value = S.getValue('OS_INTERFACE')
+ params['os_interface'] = value
+ if S.hasValue('OS_API_VERSION'):
+ value = S.getValue('OS_API_VERSION')
+ params['identity_api_version'] = value
+ if S.hasValue('OS_PROFILE'):
+ value = S.getValue('OS_PROFILE')
+ params['os_profile'] = value
+ return params
--
cgit