diff options
Diffstat (limited to 'yardstick/benchmark')
-rw-r--r-- | yardstick/benchmark/contexts/heat.py | 67 | ||||
-rw-r--r-- | yardstick/benchmark/contexts/model.py | 84 | ||||
-rw-r--r-- | yardstick/benchmark/contexts/node.py | 6 | ||||
-rw-r--r-- | yardstick/benchmark/core/plugin.py | 10 | ||||
-rw-r--r-- | yardstick/benchmark/core/task.py | 43 | ||||
-rw-r--r-- | yardstick/benchmark/runners/duration.py | 5 | ||||
-rw-r--r-- | yardstick/benchmark/scenarios/compute/qemu_migrate.py | 2 | ||||
-rw-r--r-- | yardstick/benchmark/scenarios/compute/qemu_migrate_benchmark.bash | 6 | ||||
-rw-r--r-- | yardstick/benchmark/scenarios/lib/create_keypair.py | 2 | ||||
-rw-r--r-- | yardstick/benchmark/scenarios/networking/ping.py | 2 | ||||
-rw-r--r-- | yardstick/benchmark/scenarios/networking/vnf_generic.py | 144 |
11 files changed, 250 insertions, 121 deletions
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index 575467f8f..9a7b3817f 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -83,9 +83,14 @@ class HeatContext(Context): external_network = os.environ.get("EXTERNAL_NETWORK", "net04_ext") have_external_network = any(net.get("external_network") for net in networks.values()) - if sorted_networks and not have_external_network: - # no external net defined, assign it to first network using os.environ - sorted_networks[0][1]["external_network"] = external_network + if not have_external_network: + # try looking for mgmt network first + try: + networks['mgmt']["external_network"] = external_network + except KeyError: + if sorted_networks: + # otherwise assign it to first network using os.environ + sorted_networks[0][1]["external_network"] = external_network return sorted_networks @@ -295,7 +300,7 @@ class HeatContext(Context): def deploy(self): """deploys template into a stack using cloud""" - print("Deploying context '%s'" % self.name) + LOG.info("Deploying context '%s' START", self.name) heat_template = HeatTemplate(self.name, self.template_file, self.heat_parameters) @@ -325,23 +330,33 @@ class HeatContext(Context): server.public_ip = \ self.stack.outputs[server.floating_ip["stack_name"]] - print("Context '%s' deployed" % self.name) + LOG.info("Deploying context '%s' DONE", self.name) def add_server_port(self, server): - # TODO(hafe) can only handle one internal network for now - port = next(iter(server.ports.values())) - server.private_ip = self.stack.outputs[port["stack_name"]] + # use private ip from first port in first network + try: + private_port = next(iter(server.ports.values()))[0] + except IndexError: + LOG.exception("Unable to find first private port in %s", server.ports) + raise + server.private_ip = self.stack.outputs[private_port["stack_name"]] server.interfaces = {} - for network_name, port in server.ports.items(): - server.interfaces[network_name] = self.make_interface_dict( - network_name, port['stack_name'], self.stack.outputs) - - def make_interface_dict(self, network_name, stack_name, outputs): + for network_name, ports in server.ports.items(): + for port in ports: + # port['port'] is either port name from mapping or default network_name + server.interfaces[port['port']] = self.make_interface_dict(network_name, + port['port'], + port['stack_name'], + self.stack.outputs) + + def make_interface_dict(self, network_name, port, stack_name, outputs): private_ip = outputs[stack_name] mac_address = outputs[h_join(stack_name, "mac_address")] + # these are attributes of the network, not the port output_subnet_cidr = outputs[h_join(self.name, network_name, 'subnet', 'cidr')] + # these are attributes of the network, not the port output_subnet_gateway = outputs[h_join(self.name, network_name, 'subnet', 'gateway_ip')] @@ -355,6 +370,7 @@ class HeatContext(Context): "mac_address": mac_address, "device_id": outputs[h_join(stack_name, "device_id")], "network_id": outputs[h_join(stack_name, "network_id")], + # this should be == vld_id for NSB tests "network_name": network_name, # to match vnf_generic "local_mac": mac_address, @@ -364,10 +380,10 @@ class HeatContext(Context): def undeploy(self): """undeploys stack from cloud""" if self.stack: - print("Undeploying context '%s'" % self.name) + LOG.info("Undeploying context '%s' START", self.name) self.stack.delete() self.stack = None - print("Context '%s' undeployed" % self.name) + LOG.info("Undeploying context '%s' DONE", self.name) if os.path.exists(self.key_filename): try: @@ -397,10 +413,6 @@ class HeatContext(Context): attr_name: either a name for a server created by yardstick or a dict with attribute name mapping when using external heat templates """ - key_filename = pkg_resources.resource_filename( - 'yardstick.resources', - h_join('files/yardstick_key', get_short_key_uuid(self.key_uuid))) - if isinstance(attr_name, collections.Mapping): node_name, cname = self.split_name(attr_name['name']) if cname is None or cname != self.name: @@ -418,14 +430,20 @@ class HeatContext(Context): if server is None: return None + pkey = pkg_resources.resource_string( + 'yardstick.resources', + h_join('files/yardstick_key', get_short_key_uuid(self.key_uuid))).decode('utf-8') + result = { "user": server.context.user, - "key_filename": key_filename, + "pkey": pkey, "private_ip": server.private_ip, "interfaces": server.interfaces, "routing_table": self.generate_routing_table(server), # empty IPv6 routing table "nd_route_tbl": [], + # we want to save the contex name so we can generate pod.yaml + "name": server.name, } # Target server may only have private_ip if server.public_ip: @@ -438,9 +456,11 @@ class HeatContext(Context): network = self.networks.get(attr_name, None) else: - # Don't generalize too much Just support vld_id - vld_id = attr_name.get('vld_id', {}) - network_iter = (n for n in self.networks.values() if n.vld_id == vld_id) + # Only take the first key, value + key, value = next(iter(attr_name.items()), (None, None)) + if key is None: + return None + network_iter = (n for n in self.networks.values() if getattr(n, key) == value) network = next(network_iter, None) if network is None: @@ -448,7 +468,6 @@ class HeatContext(Context): result = { "name": network.name, - "vld_id": network.vld_id, "segmentation_id": network.segmentation_id, "network_type": network.network_type, "physical_network": network.physical_network, diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py index 0b8197ce9..facfab892 100644 --- a/yardstick/benchmark/contexts/model.py +++ b/yardstick/benchmark/contexts/model.py @@ -11,9 +11,15 @@ """ from __future__ import absolute_import + +import six +import logging from six.moves import range +LOG = logging.getLogger(__name__) + + class Object(object): """Base class for classes in the logical model Contains common attributes and methods @@ -208,6 +214,7 @@ class Server(Object): # pragma: no cover self.instances = attrs["instances"] # dict with key network name, each item is a dict with port name and ip + self.network_ports = attrs.get("network_ports", {}) self.ports = {} self.floating_ip = None @@ -253,37 +260,54 @@ class Server(Object): # pragma: no cover """adds to the template one server and corresponding resources""" port_name_list = [] for network in networks: - port_name = server_name + "-" + network.name + "-port" - self.ports[network.name] = {"stack_name": port_name} - # we can't use secgroups if port_security_enabled is False - if network.port_security_enabled is False: - sec_group_id = None + # if explicit mapping skip unused networks + if self.network_ports: + try: + ports = self.network_ports[network.name] + except KeyError: + # no port for this network + continue + else: + if isinstance(ports, six.string_types): + if ports.startswith('-'): + LOG.warning("possible YAML error, port name starts with - '%s", ports) + ports = [ports] + # otherwise add a port for every network with port name as network name else: - # if port_security_enabled is None we still need to add to secgroup - sec_group_id = self.secgroup_name - # don't refactor to pass in network object, that causes JSON - # circular ref encode errors - template.add_port(port_name, network.stack_name, network.subnet_stack_name, - network.vnic_type, sec_group_id=sec_group_id, - provider=network.provider, - allowed_address_pairs=network.allowed_address_pairs) - port_name_list.append(port_name) - - if self.floating_ip: - external_network = self.floating_ip["external_network"] - if network.has_route_to(external_network): - self.floating_ip["stack_name"] = server_name + "-fip" - template.add_floating_ip(self.floating_ip["stack_name"], - external_network, - port_name, - network.router.stack_if_name, - sec_group_id) - self.floating_ip_assoc["stack_name"] = \ - server_name + "-fip-assoc" - template.add_floating_ip_association( - self.floating_ip_assoc["stack_name"], - self.floating_ip["stack_name"], - port_name) + ports = [network.name] + for port in ports: + port_name = "{0}-{1}-port".format(server_name, port) + self.ports.setdefault(network.name, []).append( + {"stack_name": port_name, "port": port}) + # we can't use secgroups if port_security_enabled is False + if network.port_security_enabled is False: + sec_group_id = None + else: + # if port_security_enabled is None we still need to add to secgroup + sec_group_id = self.secgroup_name + # don't refactor to pass in network object, that causes JSON + # circular ref encode errors + template.add_port(port_name, network.stack_name, network.subnet_stack_name, + network.vnic_type, sec_group_id=sec_group_id, + provider=network.provider, + allowed_address_pairs=network.allowed_address_pairs) + port_name_list.append(port_name) + + if self.floating_ip: + external_network = self.floating_ip["external_network"] + if network.has_route_to(external_network): + self.floating_ip["stack_name"] = server_name + "-fip" + template.add_floating_ip(self.floating_ip["stack_name"], + external_network, + port_name, + network.router.stack_if_name, + sec_group_id) + self.floating_ip_assoc["stack_name"] = \ + server_name + "-fip-assoc" + template.add_floating_ip_association( + self.floating_ip_assoc["stack_name"], + self.floating_ip["stack_name"], + port_name) if self.flavor: if isinstance(self.flavor, dict): self.flavor["name"] = \ diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py index 250032efc..ffc82c8ed 100644 --- a/yardstick/benchmark/contexts/node.py +++ b/yardstick/benchmark/contexts/node.py @@ -74,11 +74,11 @@ class NodeContext(Context): self.nodes.extend(cfg["nodes"]) self.controllers.extend([node for node in cfg["nodes"] - if node["role"] == "Controller"]) + if node.get("role") == "Controller"]) self.computes.extend([node for node in cfg["nodes"] - if node["role"] == "Compute"]) + if node.get("role") == "Compute"]) self.baremetals.extend([node for node in cfg["nodes"] - if node["role"] == "Baremetal"]) + if node.get("role") == "Baremetal"]) LOG.debug("Nodes: %r", self.nodes) LOG.debug("Controllers: %r", self.controllers) LOG.debug("Computes: %r", self.computes) diff --git a/yardstick/benchmark/core/plugin.py b/yardstick/benchmark/core/plugin.py index 24f1b6b25..56ecd29d9 100644 --- a/yardstick/benchmark/core/plugin.py +++ b/yardstick/benchmark/core/plugin.py @@ -38,19 +38,19 @@ class Plugin(object): plugins, deployment = parser.parse_plugin() plugin_name = plugins.get("name") - print("Installing plugin: %s" % plugin_name) + LOG.info("Installing plugin: %s", plugin_name) - LOG.info("Executing _install_setup()") + LOG.debug("Executing _install_setup()") self._install_setup(plugin_name, deployment) - LOG.info("Executing _run()") + LOG.debug("Executing _run()") self._run(plugin_name) total_end_time = time.time() - LOG.info("total finished in %d secs", + LOG.info("Total finished in %d secs", total_end_time - total_start_time) - print("Done, exiting") + LOG.info("Plugin %s Done, exiting", plugin_name) def remove(self, args): """Remove a plugin.""" diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py index 4d4255eb7..0b6e3230b 100644 --- a/yardstick/benchmark/core/task.py +++ b/yardstick/benchmark/core/task.py @@ -97,8 +97,8 @@ class Task(object): # pragma: no cover task_args = [args.task_args] task_args_fnames = [args.task_args_file] - LOG.info("\ntask_files:%s, \ntask_args:%s, \ntask_args_fnames:%s", - task_files, task_args, task_args_fnames) + LOG.debug("task_files:%s, task_args:%s, task_args_fnames:%s", + task_files, task_args, task_args_fnames) if args.parse_only: sys.exit(0) @@ -139,7 +139,7 @@ class Task(object): # pragma: no cover context.undeploy() self.contexts = [] one_task_end_time = time.time() - LOG.info("task %s finished in %d secs", task_files[i], + LOG.info("Task %s finished in %d secs", task_files[i], one_task_end_time - one_task_start_time) result = self._get_format_result(testcases) @@ -148,14 +148,13 @@ class Task(object): # pragma: no cover self._generate_reporting(result) total_end_time = time.time() - LOG.info("total finished in %d secs", + LOG.info("Total finished in %d secs", total_end_time - total_start_time) scenario = scenarios[0] - print("To generate report execute => yardstick report generate ", - scenario['task_id'], scenario['tc']) - - print("Done, exiting") + LOG.info("To generate report, execute command " + "'yardstick report generate %(task_id)s %(tc)s'", scenario) + LOG.info("Task ALL DONE, exiting") return result def _generate_reporting(self, result): @@ -163,7 +162,7 @@ class Task(object): # pragma: no cover with open(constants.REPORTING_FILE, 'w') as f: f.write(env.from_string(report_template).render(result)) - LOG.info('yardstick reporting generate in %s', constants.REPORTING_FILE) + LOG.info("Report can be found in '%s'", constants.REPORTING_FILE) def _set_log(self): log_format = '%(asctime)s %(name)s %(filename)s:%(lineno)d %(levelname)s %(message)s' @@ -265,7 +264,7 @@ class Task(object): # pragma: no cover raise RuntimeError self.outputs.update(runner.get_output()) result.extend(runner.get_result()) - print("Runner ended, output in", output_file) + LOG.info("Runner ended, output in %s", output_file) else: # run serially for scenario in scenarios: @@ -277,7 +276,7 @@ class Task(object): # pragma: no cover raise RuntimeError self.outputs.update(runner.get_output()) result.extend(runner.get_result()) - print("Runner ended, output in", output_file) + LOG.info("Runner ended, output in %s", output_file) # Abort background runners for runner in background_runners: @@ -302,7 +301,7 @@ class Task(object): # pragma: no cover base_runner.Runner.terminate_all() if self.contexts: - print("Undeploying all contexts") + LOG.info("Undeploying all contexts") for context in self.contexts[::-1]: context.undeploy() @@ -370,9 +369,10 @@ class Task(object): # pragma: no cover context_cfg["nodes"] = parse_nodes_with_context(scenario_cfg) context_cfg["networks"] = get_networks_from_nodes( context_cfg["nodes"]) + runner = base_runner.Runner.get(runner_cfg) - print("Starting runner of type '%s'" % runner_cfg["type"]) + LOG.info("Starting runner of type '%s'", runner_cfg["type"]) runner.run(scenario_cfg, context_cfg) return runner @@ -384,8 +384,6 @@ class Task(object): # pragma: no cover target_attr: either a name for a server created by yardstick or a dict with attribute name mapping when using external heat templates """ - host = None - target = None for context in self.contexts: if context.__context_type__ != "Heat": continue @@ -484,7 +482,7 @@ class TaskParser(object): # pragma: no cover def parse_task(self, task_id, task_args=None, task_args_file=None): """parses the task file and return an context and scenario instances""" - print("Parsing task config:", self.path) + LOG.info("Parsing task config: %s", self.path) try: kw = {} @@ -501,10 +499,9 @@ class TaskParser(object): # pragma: no cover input_task = f.read() rendered_task = TaskTemplate.render(input_task, **kw) except Exception as e: - print("Failed to render template:\n%(task)s\n%(err)s\n" - % {"task": input_task, "err": e}) + LOG.exception('Failed to render template:\n%s\n', input_task) raise e - print("Input task is:\n%s\n" % rendered_task) + LOG.debug("Input task is:\n%s\n", rendered_task) cfg = yaml_load(rendered_task) except IOError as ioerror: @@ -635,11 +632,11 @@ def get_networks_from_nodes(nodes): continue interfaces = node.get('interfaces', {}) for interface in interfaces.values(): - vld_id = interface.get('vld_id') - # mgmt network doesn't have vld_id - if not vld_id: + # vld_id is network_name + network_name = interface.get('network_name') + if not network_name: continue - network = Context.get_network({"vld_id": vld_id}) + network = Context.get_network(network_name) if network: networks[network['name']] = network return networks diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py index 69d744562..c2c6a8f19 100644 --- a/yardstick/benchmark/runners/duration.py +++ b/yardstick/benchmark/runners/duration.py @@ -40,7 +40,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg, interval = runner_cfg.get("interval", 1) duration = runner_cfg.get("duration", 60) - LOG.info("worker START, duration %d sec, class %s", duration, cls) + LOG.info("Worker START, duration is %ds", duration) + LOG.debug("class is %s", cls) runner_cfg['runner_id'] = os.getpid() @@ -95,7 +96,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, if (errors and sla_action is None) or \ (time.time() - start > duration or aborted.is_set()): - LOG.info("worker END") + LOG.info("Worker END") break benchmark.teardown() diff --git a/yardstick/benchmark/scenarios/compute/qemu_migrate.py b/yardstick/benchmark/scenarios/compute/qemu_migrate.py index 6c0446bb7..6cfedc17a 100644 --- a/yardstick/benchmark/scenarios/compute/qemu_migrate.py +++ b/yardstick/benchmark/scenarios/compute/qemu_migrate.py @@ -106,7 +106,7 @@ class QemuMigrate(base.Scenario): cmd_args = " %s %s %s %s %s %s" %\ (smp, qmp_sock_src, qmp_sock_dst, incoming_ip, migrate_to_port, max_down_time) - cmd = "bash migrate_benchmark.sh %s" % (cmd_args) + cmd = "bash qemu_migrate_benchmark.sh %s" % (cmd_args) LOG.debug("Executing command: %s", cmd) status, stdout, stderr = self.host.execute(cmd) if status: diff --git a/yardstick/benchmark/scenarios/compute/qemu_migrate_benchmark.bash b/yardstick/benchmark/scenarios/compute/qemu_migrate_benchmark.bash index 552098103..d9a440c89 100644 --- a/yardstick/benchmark/scenarios/compute/qemu_migrate_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/qemu_migrate_benchmark.bash @@ -14,6 +14,7 @@ set -e # Commandline arguments src=$2 +dst=$3 dst_ip=$4 migrate_to_port=$5 max_down_time=$6 @@ -22,7 +23,6 @@ OUTPUT_FILE=/tmp/output-qemu.log do_migrate() { -# local src=`echo $OPTIONS | cut -d ':' -f 2 | cut -d ',' -f 1` echo "info status" | nc -U $src # with no speed limit echo "migrate_set_speed 0" |nc -U $src @@ -45,7 +45,9 @@ output_qemu() # print detail information echo "info migrate" | nc -U $src echo "quit" | nc -U $src + echo "quit" | nc -u $dst sleep 5 + echo "Migration executed successfully" } > $OUTPUT_FILE @@ -64,5 +66,7 @@ echo -e "{ \ main() { do_migrate + output_qemu + output_json } main diff --git a/yardstick/benchmark/scenarios/lib/create_keypair.py b/yardstick/benchmark/scenarios/lib/create_keypair.py index 2185bfa5d..f5b1fff7a 100644 --- a/yardstick/benchmark/scenarios/lib/create_keypair.py +++ b/yardstick/benchmark/scenarios/lib/create_keypair.py @@ -47,7 +47,7 @@ class CreateKeypair(base.Scenario): rsa_key = paramiko.RSAKey.generate(bits=2048, progress_func=None) rsa_key.write_private_key_file(self.key_filename) - print("Writing %s ..." % self.key_filename) + LOG.info("Writing key_file %s ...", self.key_filename) with open(self.key_filename + ".pub", "w") as pubkey_file: pubkey_file.write( "%s %s\n" % (rsa_key.get_name(), rsa_key.get_base64())) diff --git a/yardstick/benchmark/scenarios/networking/ping.py b/yardstick/benchmark/scenarios/networking/ping.py index 6a7927de4..3bade73e2 100644 --- a/yardstick/benchmark/scenarios/networking/ping.py +++ b/yardstick/benchmark/scenarios/networking/ping.py @@ -67,7 +67,7 @@ class Ping(base.Scenario): else: target_vm = self.scenario_cfg['target'] - LOG.debug("ping '%s' '%s'", options, dest) + LOG.debug("ping %s %s", options, dest) with open(self.target_script, "r") as stdin_file: exit_status, stdout, stderr = self.connection.execute( "/bin/sh -s {0} {1}".format(dest, options), diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py index f7b2915a2..450f83f6a 100644 --- a/yardstick/benchmark/scenarios/networking/vnf_generic.py +++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py @@ -25,10 +25,11 @@ import re from itertools import chain import six -from operator import itemgetter +import yaml from collections import defaultdict from yardstick.benchmark.scenarios import base +from yardstick.common.constants import LOG_DIR from yardstick.common.utils import import_modules_from_package, itersubclasses from yardstick.common.yaml_loader import yaml_load from yardstick.network_services.collector.subscriber import Collector @@ -63,10 +64,11 @@ class IncorrectSetup(Exception): class SshManager(object): - def __init__(self, node): + def __init__(self, node, timeout=120): super(SshManager, self).__init__() self.node = node self.conn = None + self.timeout = timeout def __enter__(self): """ @@ -75,7 +77,7 @@ class SshManager(object): """ try: self.conn = ssh.SSH.from_node(self.node) - self.conn.wait() + self.conn.wait(timeout=self.timeout) except SSHError as error: LOG.info("connect failed to %s, due to %s", self.node["ip"], error) # self.conn defaults to None @@ -134,6 +136,7 @@ class NetworkServiceTestCase(base.Scenario): self.vnfs = [] self.collector = None self.traffic_profile = None + self.node_netdevs = {} def _get_ip_flow_range(self, ip_start_range): @@ -168,15 +171,17 @@ class NetworkServiceTestCase(base.Scenario): def _get_traffic_flow(self): flow = {} try: + # TODO: should be .0 or .1 so we can use list + # but this also roughly matches uplink_0, downlink_0 fflow = self.scenario_cfg["options"]["flow"] for index, src in enumerate(fflow.get("src_ip", [])): - flow["src_ip{}".format(index)] = self._get_ip_flow_range(src) + flow["src_ip_{}".format(index)] = self._get_ip_flow_range(src) for index, dst in enumerate(fflow.get("dst_ip", [])): - flow["dst_ip{}".format(index)] = self._get_ip_flow_range(dst) + flow["dst_ip_{}".format(index)] = self._get_ip_flow_range(dst) - for index, publicip in enumerate(fflow.get("publicip", [])): - flow["public_ip{}".format(index)] = publicip + for index, publicip in enumerate(fflow.get("public_ip", [])): + flow["public_ip_{}".format(index)] = publicip flow["count"] = fflow["count"] except KeyError: @@ -201,8 +206,8 @@ class NetworkServiceTestCase(base.Scenario): traffic_map_data = { 'flow': self._get_traffic_flow(), 'imix': self._get_traffic_imix(), - 'private': {}, - 'public': {}, + TrafficProfile.UPLINK: {}, + TrafficProfile.DOWNLINK: {}, } traffic_vnfd = vnfdgen.generate_vnfd(traffic_mapping, traffic_map_data) @@ -216,7 +221,26 @@ class NetworkServiceTestCase(base.Scenario): @staticmethod def get_vld_networks(networks): - return {n['vld_id']: n for n in networks.values()} + # network name is vld_id + vld_map = {} + for name, n in networks.items(): + try: + vld_map[n['vld_id']] = n + except KeyError: + vld_map[name] = n + return vld_map + + @staticmethod + def find_node_if(nodes, name, if_name, vld_id): + try: + # check for xe0, xe1 + intf = nodes[name]["interfaces"][if_name] + except KeyError: + # if not xe0, then maybe vld_id, uplink_0, downlink_0 + # pop it and re-insert with the correct name from topology + intf = nodes[name]["interfaces"].pop(vld_id) + nodes[name]["interfaces"][if_name] = intf + return intf def _resolve_topology(self): for vld in self.topology["vld"]: @@ -234,8 +258,8 @@ class NetworkServiceTestCase(base.Scenario): try: nodes = self.context_cfg["nodes"] - node0_if = nodes[node0_name]["interfaces"][node0_if_name] - node1_if = nodes[node1_name]["interfaces"][node1_if_name] + node0_if = self.find_node_if(nodes, node0_name, node0_if_name, vld["id"]) + node1_if = self.find_node_if(nodes, node1_name, node1_if_name, vld["id"]) # names so we can do reverse lookups node0_if["ifname"] = node0_if_name @@ -244,7 +268,6 @@ class NetworkServiceTestCase(base.Scenario): node0_if["node_name"] = node0_name node1_if["node_name"] = node1_name - vld_networks = self.get_vld_networks(self.context_cfg["networks"]) node0_if["vld_id"] = vld["id"] node1_if["vld_id"] = vld["id"] @@ -257,6 +280,7 @@ class NetworkServiceTestCase(base.Scenario): node1_if["peer_ifname"] = node0_if_name # just load the network + vld_networks = self.get_vld_networks(self.context_cfg["networks"]) node0_if["network"] = vld_networks.get(vld["id"], {}) node1_if["network"] = vld_networks.get(vld["id"], {}) @@ -285,8 +309,8 @@ class NetworkServiceTestCase(base.Scenario): node1_if_name = node1_data["vnfd-connection-point-ref"] nodes = self.context_cfg["nodes"] - node0_if = nodes[node0_name]["interfaces"][node0_if_name] - node1_if = nodes[node1_name]["interfaces"][node1_if_name] + node0_if = self.find_node_if(nodes, node0_name, node0_if_name, vld["id"]) + node1_if = self.find_node_if(nodes, node1_name, node1_if_name, vld["id"]) # add peer interface dict, but remove circular link # TODO: don't waste memory @@ -306,17 +330,16 @@ class NetworkServiceTestCase(base.Scenario): vnfd = self._find_vnfd_from_vnf_idx(vnf_idx) self.context_cfg["nodes"][vnf_name].update(vnfd) - @staticmethod - def _sort_dpdk_port_num(netdevs): - # dpdk_port_num is PCI BUS ID ordering, lowest first - s = sorted(netdevs.values(), key=itemgetter('pci_bus_id')) - for dpdk_port_num, netdev in enumerate(s): - netdev['dpdk_port_num'] = dpdk_port_num + def _probe_netdevs(self, node, node_dict, timeout=120): + try: + return self.node_netdevs[node] + except KeyError: + pass - def _probe_netdevs(self, node, node_dict): - cmd = "PATH=$PATH:/sbin:/usr/sbin ip addr show" netdevs = {} - with SshManager(node_dict) as conn: + cmd = "PATH=$PATH:/sbin:/usr/sbin ip addr show" + + with SshManager(node_dict, timeout=timeout) as conn: if conn: exit_status = conn.execute(cmd)[0] if exit_status != 0: @@ -327,6 +350,8 @@ class NetworkServiceTestCase(base.Scenario): raise IncorrectSetup( "Cannot find netdev info in sysfs" % node) netdevs = node_dict['netdevs'] = self.parse_netdev_info(stdout) + + self.node_netdevs[node] = netdevs return netdevs @classmethod @@ -342,6 +367,36 @@ class NetworkServiceTestCase(base.Scenario): 'ifindex': netdev['ifindex'], }) + def _generate_pod_yaml(self): + context_yaml = os.path.join(LOG_DIR, "pod-{}.yaml".format(self.scenario_cfg['task_id'])) + # convert OrderedDict to a list + # pod.yaml nodes is a list + nodes = [] + for node in self.context_cfg["nodes"].values(): + # name field is required + # remove context suffix + node['name'] = node['name'].split('.')[0] + nodes.append(node) + nodes = self._convert_pkeys_to_string(nodes) + pod_dict = { + "nodes": nodes, + "networks": self.context_cfg["networks"] + } + with open(context_yaml, "w") as context_out: + yaml.safe_dump(pod_dict, context_out, default_flow_style=False, + explicit_start=True) + + @staticmethod + def _convert_pkeys_to_string(nodes): + # make copy because we are mutating + nodes = nodes[:] + for i, node in enumerate(nodes): + try: + nodes[i] = dict(node, pkey=ssh.convert_key_to_str(node["pkey"])) + except KeyError: + pass + return nodes + TOPOLOGY_REQUIRED_KEYS = frozenset({ "vpci", "local_ip", "netmask", "local_mac", "driver"}) @@ -351,6 +406,10 @@ class NetworkServiceTestCase(base.Scenario): :return: None. Side effect: context_cfg is updated """ + num_nodes = len(self.context_cfg["nodes"]) + # OpenStack instance creation time is probably proportional to the number + # of instances + timeout = 120 * num_nodes for node, node_dict in self.context_cfg["nodes"].items(): for network in node_dict["interfaces"].values(): @@ -361,7 +420,7 @@ class NetworkServiceTestCase(base.Scenario): # only ssh probe if there are missing values # ssh probe won't work on Ixia, so we had better define all our values try: - netdevs = self._probe_netdevs(node, node_dict) + netdevs = self._probe_netdevs(node, node_dict, timeout=timeout) except (SSHError, SSHTimeout): raise IncorrectConfig( "Unable to probe missing interface fields '%s', on node %s " @@ -378,6 +437,8 @@ class NetworkServiceTestCase(base.Scenario): "Require interface fields '%s' not found, topology file " "corrupted" % ', '.join(missing)) + # we have to generate pod.yaml here so we have vpci and driver + self._generate_pod_yaml() # 3. Use topology file to find connections & resolve dest address self._resolve_topology() self._update_context_with_topology() @@ -439,10 +500,26 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ (expected_name, classes_found)) @staticmethod - def update_interfaces_from_node(vnfd, node): - for intf in vnfd["vdu"][0]["external-interface"]: - node_intf = node['interfaces'][intf['name']] - intf['virtual-interface'].update(node_intf) + def create_interfaces_from_node(vnfd, node): + ext_intfs = vnfd["vdu"][0]["external-interface"] = [] + # have to sort so xe0 goes first + for intf_name, intf in sorted(node['interfaces'].items()): + # only interfaces with vld_id are added. + # Thus there are two layers of filters, only intefaces with vld_id + # show up in interfaces, and only interfaces with traffic profiles + # are used by the generators + if intf.get('vld_id'): + # force dpkd_port_num to int so we can do reverse lookup + try: + intf['dpdk_port_num'] = int(intf['dpdk_port_num']) + except KeyError: + pass + ext_intf = { + "name": intf_name, + "virtual-interface": intf, + "vnfd-connection-point-ref": intf_name, + } + ext_intfs.append(ext_intf) def load_vnf_models(self, scenario_cfg=None, context_cfg=None): """ Create VNF objects based on YAML descriptors @@ -472,7 +549,14 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ vnfd = vnfdgen.generate_vnfd(vnf_model, node) # TODO: here add extra context_cfg["nodes"] regardless of template vnfd = vnfd["vnfd:vnfd-catalog"]["vnfd"][0] - self.update_interfaces_from_node(vnfd, node) + # force inject pkey if it exists + # we want to standardize Heat using pkey as a string so we don't rely + # on the filesystem + try: + vnfd['mgmt-interface']['pkey'] = node['pkey'] + except KeyError: + pass + self.create_interfaces_from_node(vnfd, node) vnf_impl = self.get_vnf_impl(vnfd['id']) vnf_instance = vnf_impl(node_name, vnfd) vnfs.append(vnf_instance) |