From 7e83d0876ddb84a45e130eeba28bc40ef53c074b Mon Sep 17 00:00:00 2001 From: Yaron Yogev Date: Thu, 27 Jul 2017 09:02:54 +0300 Subject: Calipso initial release for OPNFV Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev --- app/discover/fetchers/cli/__init__.py | 9 + app/discover/fetchers/cli/cli_access.py | 206 +++++++++++++++++++++ app/discover/fetchers/cli/cli_fetch_host_pnics.py | 122 ++++++++++++ .../fetchers/cli/cli_fetch_host_pnics_vpp.py | 44 +++++ .../fetchers/cli/cli_fetch_host_vservice.py | 80 ++++++++ .../fetchers/cli/cli_fetch_host_vservices.py | 27 +++ .../fetchers/cli/cli_fetch_instance_vnics.py | 22 +++ .../fetchers/cli/cli_fetch_instance_vnics_base.py | 68 +++++++ .../fetchers/cli/cli_fetch_instance_vnics_vpp.py | 18 ++ app/discover/fetchers/cli/cli_fetch_oteps_lxb.py | 86 +++++++++ app/discover/fetchers/cli/cli_fetch_vconnectors.py | 40 ++++ .../fetchers/cli/cli_fetch_vconnectors_lxb.py | 35 ++++ .../fetchers/cli/cli_fetch_vconnectors_ovs.py | 56 ++++++ .../fetchers/cli/cli_fetch_vconnectors_vpp.py | 64 +++++++ app/discover/fetchers/cli/cli_fetch_vpp_vedges.py | 58 ++++++ .../fetchers/cli/cli_fetch_vservice_vnics.py | 140 ++++++++++++++ 16 files changed, 1075 insertions(+) create mode 100644 app/discover/fetchers/cli/__init__.py create mode 100644 app/discover/fetchers/cli/cli_access.py create mode 100644 app/discover/fetchers/cli/cli_fetch_host_pnics.py create mode 100644 app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py create mode 100644 app/discover/fetchers/cli/cli_fetch_host_vservice.py create mode 100644 app/discover/fetchers/cli/cli_fetch_host_vservices.py create mode 100644 app/discover/fetchers/cli/cli_fetch_instance_vnics.py create mode 100644 app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py create mode 100644 app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py create mode 100644 app/discover/fetchers/cli/cli_fetch_oteps_lxb.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vconnectors.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vpp_vedges.py create mode 100644 app/discover/fetchers/cli/cli_fetch_vservice_vnics.py (limited to 'app/discover/fetchers/cli') diff --git a/app/discover/fetchers/cli/__init__.py b/app/discover/fetchers/cli/__init__.py new file mode 100644 index 0000000..b0637e9 --- /dev/null +++ b/app/discover/fetchers/cli/__init__.py @@ -0,0 +1,9 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py new file mode 100644 index 0000000..1db84ea --- /dev/null +++ b/app/discover/fetchers/cli/cli_access.py @@ -0,0 +1,206 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re +import time + +from discover.fetcher import Fetcher +from utils.binary_converter import BinaryConverter +from utils.logging.console_logger import ConsoleLogger +from utils.ssh_conn import SshConn + + +class CliAccess(BinaryConverter, Fetcher): + connections = {} + ssh_cmd = "ssh -o StrictHostKeyChecking=no " + call_count_per_con = {} + max_call_count_per_con = 100 + cache_lifetime = 60 # no. of seconds to cache results + cached_commands = {} + + def __init__(self): + super().__init__() + self.log = ConsoleLogger() + + @staticmethod + def is_gateway_host(ssh_to_host): + ssh_conn = SshConn(ssh_to_host) + return ssh_conn.is_gateway_host(ssh_to_host) + + def run_on_gateway(self, cmd, ssh_to_host="", enable_cache=True, + use_sudo=True): + self.run(cmd, ssh_to_host=ssh_to_host, enable_cache=enable_cache, + on_gateway=True, use_sudo=use_sudo) + + def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False, + ssh=None, use_sudo=True): + ssh_conn = ssh if ssh else SshConn(ssh_to_host) + if use_sudo and not cmd.strip().startswith("sudo "): + cmd = "sudo " + cmd + if not on_gateway and ssh_to_host \ + and not ssh_conn.is_gateway_host(ssh_to_host): + cmd = self.ssh_cmd + ssh_to_host + " " + cmd + curr_time = time.time() + cmd_path = ssh_to_host + ',' + cmd + if enable_cache and cmd_path in self.cached_commands: + # try to re-use output from last call + cached = self.cached_commands[cmd_path] + if cached["timestamp"] + self.cache_lifetime < curr_time: + # result expired + self.cached_commands.pop(cmd_path, None) + else: + # result is good to use - skip the SSH call + self.log.info('CliAccess: ****** using cached result, ' + + 'host: ' + ssh_to_host + ', cmd: %s ******', cmd) + return cached["result"] + + self.log.info('CliAccess: host: %s, cmd: %s', ssh_to_host, cmd) + ret = ssh_conn.exec(cmd) + self.cached_commands[cmd_path] = {"timestamp": curr_time, "result": ret} + return ret + + def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True): + out = self.run(cmd, ssh_to_host, enable_cache) + if not out: + return [] + # first try to split lines by whitespace + ret = out.splitlines() + # if split by whitespace did not work, try splitting by "\\n" + if len(ret) == 1: + ret = [l for l in out.split("\\n") if l != ""] + return ret + + # parse command output columns separated by whitespace + # since headers can contain whitespace themselves, + # it is the caller's responsibility to provide the headers + def parse_cmd_result_with_whitespace(self, lines, headers, remove_first): + if remove_first: + # remove headers line + del lines[:1] + results = [self.parse_line_with_ws(line, headers) + for line in lines] + return results + + # parse command output with "|" column separators and "-" row separators + def parse_cmd_result_with_separators(self, lines): + headers = self.parse_headers_line_with_separators(lines[1]) + # remove line with headers and formatting lines above it and below it + del lines[:3] + # remove formatting line in the end + lines.pop() + results = [self.parse_content_line_with_separators(line, headers) + for line in lines] + return results + + # parse a line with columns separated by whitespace + def parse_line_with_ws(self, line, headers): + s = line if isinstance(line, str) else self.binary2str(line) + parts = [word.strip() for word in s.split() if word.strip()] + ret = {} + for i, p in enumerate(parts): + header = headers[i] + ret[header] = p + return ret + + # parse a line with "|" column separators + def parse_line_with_separators(self, line): + s = self.binary2str(line) + parts = [word.strip() for word in s.split("|") if word.strip()] + # remove the ID field + del parts[:1] + return parts + + def parse_headers_line_with_separators(self, line): + return self.parse_line_with_separators(line) + + def parse_content_line_with_separators(self, line, headers): + content_parts = self.parse_line_with_separators(line) + content = {} + for i in range(0, len(content_parts)): + content[headers[i]] = content_parts[i] + return content + + def merge_ws_spillover_lines(self, lines): + # with WS-separated output, extra output sometimes spills to next line + # detect that and add to the end of the previous line for our procesing + pending_line = None + fixed_lines = [] + # remove headers line + for l in lines: + if l[0] == '\t': + # this is a spill-over line + if pending_line: + # add this line to the end of the previous line + pending_line = pending_line.strip() + "," + l.strip() + else: + # add the previous pending line to the fixed lines list + if pending_line: + fixed_lines.append(pending_line) + # make current line the pending line + pending_line = l + if pending_line: + fixed_lines.append(pending_line) + return fixed_lines + + """ + given output lines from CLI command like 'ip -d link show', + find lines belonging to section describing a specific interface + parameters: + - lines: list of strings, output of command + - header_regexp: regexp marking the start of the section + - end_regexp: regexp marking the end of the section + """ + def get_section_lines(self, lines, header_regexp, end_regexp): + if not lines: + return [] + header_re = re.compile(header_regexp) + start_pos = None + # find start_pos of section + line_count = len(lines) + for line_num in range(0, line_count-1): + matches = header_re.match(lines[line_num]) + if matches: + start_pos = line_num + break + if not start_pos: + return [] + # find end of section + end_pos = line_count + end_re = re.compile(end_regexp) + for line_num in range(start_pos+1, end_pos-1): + matches = end_re.match(lines[line_num]) + if matches: + end_pos = line_num + break + return lines[start_pos:end_pos] + + def get_object_data(self, o, lines, regexps): + """ + find object data in output lines from CLI command + parameters: + - o: object (dict), to which we'll add attributes with the data found + - lines: list of strings + - regexps: dict, keys are attribute names, values are regexp to match + for finding the value of the attribute + """ + for line in lines: + self.find_matching_regexps(o, line, regexps) + for regexp_tuple in regexps: + name = regexp_tuple['name'] + if 'name' not in o and 'default' in regexp_tuple: + o[name] = regexp_tuple['default'] + + def find_matching_regexps(self, o, line, regexps): + for regexp_tuple in regexps: + name = regexp_tuple['name'] + regex = regexp_tuple['re'] + regex = re.compile(regex) + matches = regex.search(line) + if matches: + o[name] = matches.group(1) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py new file mode 100644 index 0000000..3516e25 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -0,0 +1,122 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchHostPnics(CliAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.ethtool_attr = re.compile('^\s+([^:]+):\s(.*)$') + self.regexps = [ + {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'}, + {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'}, + {'name': 'IP Address', 're': '^\s*inet addr:?(\S+)\s.*$'}, + {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'}, + {'name': 'IPv6 Address', 're': '^\s*inet6 addr:\s*(\S+)(\s.*)?$'}, + {'name': 'IPv6 Address', 're': '^\s*inet6 \s*(\S+)(\s.*)?$'} + ] + + def get(self, id): + host_id = id[:id.rindex("-")] + cmd = 'ls -l /sys/class/net | grep ^l | grep -v "/virtual/"' + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("CliFetchHostPnics: host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + + ", host: " + str(host)) + return [] + host_types = host["host_type"] + if "Network" not in host_types and "Compute" not in host_types: + return [] + interface_lines = self.run_fetch_lines(cmd, host_id) + interfaces = [] + for line in interface_lines: + interface_name = line[line.rindex('/')+1:] + interface_name = interface_name.strip() + # run ifconfig with specific interface name, + # since running it with no name yields a list without inactive pNICs + interface = self.find_interface_details(host_id, interface_name) + if interface: + interfaces.append(interface) + return interfaces + + def find_interface_details(self, host_id, interface_name): + lines = self.run_fetch_lines("ifconfig " + interface_name, host_id) + interface = None + status_up = None + for line in [l for l in lines if l != '']: + tokens = None + if interface is None: + tokens = line.split() + name = tokens[0].strip('- :') + name = name.strip() + if name == interface_name: + line_remainder = line.strip('-')[len(interface_name)+2:] + line_remainder = line_remainder.strip(' :') + id = interface_name + interface = { + "host": host_id, + "name": id, + "local_name": interface_name, + "lines": [] + } + self.handle_line(interface, line_remainder) + if ' ""] + results = [] + # Note: there are 2 ids here of instances with local names, which are + # not connected to the data we have thus far for the instance + # therefore, we will decide whether the instance is the correct one + # based on comparison of the uuid in the dumpxml output + for id in virsh_ids: + results.extend(self.get_vnics_from_dumpxml(id, instance)) + return results + + def get_vnics_from_dumpxml(self, id, instance): + xml_string = self.run("virsh dumpxml " + id, instance["host"]) + if not xml_string.strip(): + return [] + response = xmltodict.parse(xml_string) + if instance["uuid"] != response["domain"]["uuid"]: + # this is the wrong instance - skip it + return [] + try: + vnics = response["domain"]["devices"]["interface"] + except KeyError: + return [] + if isinstance(vnics, dict): + vnics = [vnics] + for v in vnics: + self.set_vnic_properties(v, instance) + return vnics + + def set_vnic_properties(self, v, instance): + v["name"] = self.get_vnic_name(v, instance) + v["id"] = v["name"] + v["vnic_type"] = "instance_vnic" + v["host"] = instance["host"] + v["instance_id"] = instance["id"] + v["instance_db_id"] = instance["_id"] + v["mac_address"] = v["mac"]["@address"] + instance["mac_address"] = v["mac_address"] + self.inv.set(instance) diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py new file mode 100644 index 0000000..58facd2 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py @@ -0,0 +1,18 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_instance_vnics_base import CliFetchInstanceVnicsBase + + +class CliFetchInstanceVnicsVpp(CliFetchInstanceVnicsBase): + def __init__(self): + super().__init__() + + def get_vnic_name(self, v, instance): + return instance["name"] + "-" + v["@type"] + "-" + v["mac"]["@address"] diff --git a/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py new file mode 100644 index 0000000..1e65a14 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py @@ -0,0 +1,86 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_access import CliAccess +from discover.fetchers.db.db_access import DbAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchOtepsLxb(CliAccess, DbAccess): + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, parent_id): + vconnector = self.inv.get_by_id(self.get_env(), parent_id) + if not vconnector: + return [] + configurations = vconnector['configurations'] + tunneling_ip = configurations['tunneling_ip'] + tunnel_types_used = configurations['tunnel_types'] + if not tunnel_types_used: + return [] + tunnel_type = tunnel_types_used[0] + if not tunnel_type: + return [] + # check only interfaces with name matching tunnel type + ret = [i for i in vconnector['interfaces'].values() + if i['name'].startswith(tunnel_type + '-')] + for otep in ret: + otep['ip_address'] = tunneling_ip + otep['host'] = vconnector['host'] + self.get_otep_ports(otep) + otep['id'] = otep['host'] + '-otep-' + otep['name'] + otep['name'] = otep['id'] + otep['vconnector'] = vconnector['name'] + otep['overlay_type'] = tunnel_type + self.get_udp_port(otep) + return ret + + """ + fetch OTEP data from CLI command 'ip -d link show' + """ + def get_otep_ports(self, otep): + cmd = 'ip -d link show' + lines = self.run_fetch_lines(cmd, otep['host']) + header_format = '[0-9]+: ' + otep['name'] + ':' + interface_lines = self.get_section_lines(lines, header_format, '\S') + otep['data'] = '\n'.join(interface_lines) + regexps = [ + {'name': 'state', 're': ',UP,', 'default': 'DOWN'}, + {'name': 'mac_address', 're': '.*\slink/ether\s(\S+)\s'}, + {'name': 'mtu', 're': '.*\smtu\s(\S+)\s'}, + ] + self.get_object_data(otep, interface_lines, regexps) + cmd = 'bridge fdb show' + dst_line_format = ' dev ' + otep['name'] + ' dst ' + lines = self.run_fetch_lines(cmd, otep['host']) + lines = [l for l in lines if dst_line_format in l] + if lines: + l = lines[0] + otep['bridge dst'] = l[l.index(' dst ')+5:] + return otep + + def get_udp_port(self, otep): + table_name = "neutron.ml2_" + otep['overlay_type'] + "_endpoints" + results = None + try: + results = self.get_objects_list_for_id( + """ + SELECT udp_port + FROM {} + WHERE host = %s + """.format(table_name), + "vedge", otep['host']) + except Exception as e: + self.log.error('failed to fetch UDP port for OTEP: ' + str(e)) + otep['udp_port'] = 0 + for result in results: + otep['udp_port'] = result['udp_port'] diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors.py b/app/discover/fetchers/cli/cli_fetch_vconnectors.py new file mode 100644 index 0000000..78b767a --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors.py @@ -0,0 +1,40 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from abc import abstractmethod, ABCMeta + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr +from utils.singleton import Singleton + + +class ABCSingleton(ABCMeta, Singleton): + pass + + +class CliFetchVconnectors(CliAccess, metaclass=ABCSingleton): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + @abstractmethod + def get_vconnectors(self, host): + raise NotImplementedError("Subclass must override get_vconnectors()") + + def get(self, id): + host_id = id[:id.rindex('-')] + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("CliFetchVconnectors: host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + \ + ", host: " + str(host)) + return [] + return self.get_vconnectors(host) diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py new file mode 100644 index 0000000..648dc63 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py @@ -0,0 +1,35 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json + +from discover.fetchers.cli.cli_fetch_vconnectors_ovs import CliFetchVconnectorsOvs +from discover.fetchers.db.db_access import DbAccess + + +class CliFetchVconnectorsLxb(CliFetchVconnectorsOvs, DbAccess): + + def __init__(self): + super().__init__() + + def get(self, id): + ret = super().get(id) + for doc in ret: + query = """ + SELECT configurations + FROM {}.agents + WHERE agent_type="Linux bridge agent" AND host = %s + """.format(self.neutron_db) + host = doc['host'] + matches = self.get_objects_list_for_id(query, '', host) + if not matches: + raise ValueError('No Linux bridge agent in DB for host: {}'.format(host)) + agent = matches[0] + doc['configurations'] = json.loads(agent['configurations']) + return ret diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py new file mode 100644 index 0000000..ff37569 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py @@ -0,0 +1,56 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors + + +class CliFetchVconnectorsOvs(CliFetchVconnectors): + def __init__(self): + super().__init__() + + def get_vconnectors(self, host): + host_id = host['id'] + lines = self.run_fetch_lines("brctl show", host_id) + headers = ["bridge_name", "bridge_id", "stp_enabled", "interfaces"] + headers_count = len(headers) + # since we hard-coded the headers list, remove the headers line + del lines[:1] + + # intefaces can spill to next line - need to detect that and add + # them to the end of the previous line for our procesing + fixed_lines = self.merge_ws_spillover_lines(lines) + + results = self.parse_cmd_result_with_whitespace(fixed_lines, headers, False) + ret = [] + for doc in results: + doc["name"] = doc.pop("bridge_name") + doc["id"] = doc["name"] + "-" + doc.pop("bridge_id") + doc["host"] = host_id + doc["connector_type"] = "bridge" + if "interfaces" in doc: + interfaces = {} + interface_names = doc["interfaces"].split(",") + for interface_name in interface_names: + # find MAC address for this interface from ports list + port_id_prefix = interface_name[3:] + port = self.inv.find_items({ + "environment": self.get_env(), + "type": "port", + "binding:host_id": host_id, + "id": {"$regex": r"^" + re.escape(port_id_prefix)} + }, get_single=True) + mac_address = '' if not port else port['mac_address'] + interface = {'name': interface_name, 'mac_address': mac_address} + interfaces[interface_name] = interface + doc["interfaces"] = interfaces + doc['interfaces_names'] = list(interfaces.keys()) + ret.append(doc) + return ret diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py new file mode 100644 index 0000000..479e1db --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py @@ -0,0 +1,64 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors + + +class CliFetchVconnectorsVpp(CliFetchVconnectors): + def __init__(self): + super().__init__() + + def get_vconnectors(self, host): + lines = self.run_fetch_lines("vppctl show mode", host['id']) + vconnectors = {} + for l in lines: + if not l.startswith('l2 bridge'): + continue + line_parts = l.split(' ') + name = line_parts[2] + bd_id = line_parts[4] + if bd_id in vconnectors: + vconnector = vconnectors[bd_id] + else: + vconnector = { + 'host': host['id'], + 'id': host['id'] + '-vconnector-' + bd_id, + 'bd_id': bd_id, + 'name': "bridge-domain-" + bd_id, + 'interfaces': {}, + 'interfaces_names': [] + } + vconnectors[bd_id] = vconnector + interface = self.get_interface_details(host, name) + if interface: + vconnector['interfaces'][name] = interface + vconnector['interfaces_names'].append(name) + return list(vconnectors.values()) + + def get_interface_details(self, host, name): + # find vconnector interfaces + cmd = "vppctl show hardware-int " + name + interface_lines = self.run_fetch_lines(cmd, host['id']) + # remove header line + interface_lines.pop(0) + interface = None + for l in interface_lines: + if not l.strip(): + continue # ignore empty lines + if not l.startswith(' '): + details = l.split() + interface = { + "name": details[0], + "hardware": details[3], + "state": details[2], + "id": details[1], + } + elif l.startswith(' Ethernet address '): + interface['mac_address'] = l[l.rindex(' ') + 1:] + return interface diff --git a/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py new file mode 100644 index 0000000..f9c622d --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py @@ -0,0 +1,58 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +# Copyright 2016 cisco Corporation +#oslo related message handling + +from oslo_serialization import jsonutils +from oslo_utils import uuidutils +import yaml + +from neutronclient.tests.functional import base + + +class TestCLIFormatter(base.ClientTestBase): + +## old stuff ..not related to vpp..disregard + def setUp(self): + super(TestCLIFormatter, self).setUp() + self.net_name = 'net-%s' % uuidutils.generate_uuid() + self.addCleanup(self.neutron, 'net-delete %s' % self.net_name) + + def _create_net(self, fmt, col_attrs): + params = ['-c %s' % attr for attr in col_attrs] + params.append('-f %s' % fmt) + params.append(self.net_name) + param_string = ' '.join(params) + return self.neutron('net-create', params=param_string) + + def test_net_create_with_json_formatter(self): + result = self._create_net('json', ['name', 'admin_state_up']) + self.assertDictEqual({'name': self.net_name, + 'admin_state_up': True}, + jsonutils.loads(result)) + + def test_net_create_with_yaml_formatter(self): + result = self._create_net('yaml', ['name', 'admin_state_up']) + self.assertDictEqual({'name': self.net_name, + 'admin_state_up': True}, + yaml.load(result)) + + def test_net_create_with_value_formatter(self): + # NOTE(amotoki): In 'value' formatter, there is no guarantee + # in the order of attribute, so we use one attribute in this test. + result = self._create_net('value', ['name']) + self.assertEqual(self.net_name, result.strip()) + + def test_net_create_with_shell_formatter(self): + result = self._create_net('shell', ['name', 'admin_state_up']) + result_lines = set(result.strip().split('\n')) + self.assertSetEqual(set(['name="%s"' % self.net_name, + 'admin_state_up="True"']), +result_lines) diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py new file mode 100644 index 0000000..44ac8d6 --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py @@ -0,0 +1,140 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchVserviceVnics(CliAccess): + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + self.if_header = re.compile('^[-]?(\S+)\s+(.*)$') + self.regexps = [ + {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'}, + {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'}, + {'name': 'netmask', 're': '^.*\sMask:\s?([0-9.]+)(\s.*)?$'}, + {'name': 'netmask', 're': '^.*\snetmask\s([0-9.]+)(\s.*)?$'}, + {'name': 'IP Address', 're': '^\s*inet addr:(\S+)\s.*$'}, + {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'}, + {'name': 'IPv6 Address', + 're': '^\s*inet6 addr: ?\s*([0-9a-f:/]+)(\s.*)?$'}, + {'name': 'IPv6 Address', + 're': '^\s*inet6 \s*([0-9a-f:/]+)(\s.*)?$'} + ] + + def get(self, host_id): + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error("host not found: " + host_id) + return [] + if "host_type" not in host: + self.log.error("host does not have host_type: " + host_id + + ", host: " + str(host)) + return [] + if "Network" not in host["host_type"]: + return [] + lines = self.run_fetch_lines("ip netns", host_id) + ret = [] + for l in [l for l in lines + if l.startswith("qdhcp") or l.startswith("qrouter")]: + service = l.strip() + service = service if ' ' not in service \ + else service[:service.index(' ')] + ret.extend(self.handle_service(host_id, service)) + return ret + + def handle_service(self, host, service, enable_cache=True): + cmd = "ip netns exec " + service + " ifconfig" + lines = self.run_fetch_lines(cmd, host, enable_cache) + interfaces = [] + current = None + for line in lines: + matches = self.if_header.match(line) + if matches: + if current: + self.set_interface_data(current) + name = matches.group(1).strip(":") + # ignore 'lo' interface + if name == 'lo': + current = None + else: + line_remainder = matches.group(2) + vservice_id = host + "-" + service + current = { + "id": host + "-" + name, + "type": "vnic", + "vnic_type": "vservice_vnic", + "host": host, + "name": name, + "master_parent_type": "vservice", + "master_parent_id": vservice_id, + "parent_type": "vnics_folder", + "parent_id": vservice_id + "-vnics", + "parent_text": "vNICs", + "lines": [] + } + interfaces.append(current) + self.handle_line(current, line_remainder) + else: + if current: + self.handle_line(current, line) + if current: + self.set_interface_data(current) + return interfaces + + def handle_line(self, interface, line): + self.find_matching_regexps(interface, line, self.regexps) + interface["lines"].append(line.strip()) + + def set_interface_data(self, interface): + if not interface or 'IP Address' not in interface or 'netmask' not in interface: + return + + interface["data"] = "\n".join(interface.pop("lines", None)) + interface["cidr"] = self.get_cidr_for_vnic(interface) + network = self.inv.get_by_field(self.get_env(), "network", "cidrs", + interface["cidr"], get_single=True) + if not network: + return + interface["network"] = network["id"] + # set network for the vservice, to check network on clique creation + vservice = self.inv.get_by_id(self.get_env(), + interface["master_parent_id"]) + network_id = network["id"] + if "network" not in vservice: + vservice["network"] = list() + if network_id not in vservice["network"]: + vservice["network"].append(network_id) + self.inv.set(vservice) + + # find CIDR string by IP address and netmask + def get_cidr_for_vnic(self, vnic): + if "IP Address" not in vnic: + vnic["IP Address"] = "No IP Address" + return "No IP Address" + ipaddr = vnic["IP Address"].split('.') + netmask = vnic["netmask"].split('.') + + # calculate network start + net_start = [] + for pos in range(0, 4): + net_start.append(str(int(ipaddr[pos]) & int(netmask[pos]))) + + cidr_string = '.'.join(net_start) + '/' + cidr_string = cidr_string + self.get_net_size(netmask) + return cidr_string + + def get_net_size(self, netmask): + binary_str = '' + for octet in netmask: + binary_str += bin(int(octet))[2:].zfill(8) + return str(len(binary_str.rstrip('0'))) -- cgit 1.2.3-korg