From d32f75145676bacefde0d08a14680a5984623451 Mon Sep 17 00:00:00 2001 From: Koren Lev Date: Fri, 29 Sep 2017 01:38:18 +0300 Subject: release 1.0 calipso for opnfv apex Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2 Signed-off-by: Koren Lev --- app/discover/fetchers/cli/cli_access.py | 62 ++++++++-- .../fetchers/cli/cli_fetch_bond_host_pnics.py | 134 +++++++++++++++++++++ app/discover/fetchers/cli/cli_fetch_host_pnics.py | 26 ++-- .../fetchers/cli/cli_fetch_host_vservice.py | 22 ++-- .../fetchers/cli/cli_fetch_host_vservices.py | 2 +- .../fetchers/cli/cli_fetch_instance_vnics_base.py | 2 +- .../fetchers/cli/cli_fetch_vservice_vnics.py | 8 +- app/discover/fetchers/db/db_access.py | 43 +++++-- .../fetchers/db/db_fetch_host_network_agents.py | 3 +- app/discover/fetchers/db/db_fetch_oteps.py | 4 +- app/discover/fetchers/db/db_fetch_vedges_ovs.py | 63 ++++++---- 11 files changed, 291 insertions(+), 78 deletions(-) create mode 100644 app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py (limited to 'app/discover/fetchers') diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py index 275a3e8..c77b22a 100644 --- a/app/discover/fetchers/cli/cli_access.py +++ b/app/discover/fetchers/cli/cli_access.py @@ -12,6 +12,7 @@ import time from discover.fetcher import Fetcher from utils.binary_converter import BinaryConverter +from utils.cli_dist_translator import CliDistTranslator from utils.logging.console_logger import ConsoleLogger from utils.ssh_conn import SshConn @@ -41,11 +42,16 @@ class CliAccess(BinaryConverter, Fetcher): def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False, ssh=None, use_sudo=True): ssh_conn = ssh if ssh else SshConn(ssh_to_host) - if use_sudo and not cmd.strip().startswith("sudo "): - cmd = "sudo " + cmd - if not on_gateway and ssh_to_host \ - and not ssh_conn.is_gateway_host(ssh_to_host): - cmd = self.ssh_cmd + ssh_to_host + " " + cmd + commands = self.adapt_cmd_to_env(ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host) + out = '' + for c in commands: + out += self.run_single_command(c, ssh_conn, ssh_to_host, + enable_cache=enable_cache) + return out + + def run_single_command(self, cmd, ssh_conn, ssh_to_host="", + enable_cache=True): curr_time = time.time() cmd_path = ssh_to_host + ',' + cmd if enable_cache and cmd_path in self.cached_commands: @@ -73,9 +79,44 @@ class CliAccess(BinaryConverter, Fetcher): ret = out.splitlines() # if split by whitespace did not work, try splitting by "\\n" if len(ret) == 1: - ret = [l for l in out.split("\\n") if l != ""] + ret = [line for line in out.split("\\n") if line != ""] return ret + MULTI_COMMAND_SEPARATOR = ';;;' + + @staticmethod + def handle_split_cmd(cmd: str): + if CliAccess.MULTI_COMMAND_SEPARATOR in cmd: + return cmd.split(CliAccess.MULTI_COMMAND_SEPARATOR) + return [cmd] + + def adapt_cmd_to_env(self, ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host): + cmd = self.adapt_cmd_to_dist(cmd) + commands = self.handle_split_cmd(cmd) + return [self.adapt_cmd_to_environment(c, use_sudo, on_gateway, + ssh_to_host, ssh_conn) + for c in commands] + + def adapt_cmd_to_environment(self, cmd, use_sudo, on_gateway, ssh_to_host, + ssh_conn): + if self.configuration.environment["distribution"] == "Mercury": + use_sudo = False + if use_sudo and not cmd.strip().startswith("sudo "): + cmd = "sudo " + cmd + if not on_gateway and ssh_to_host \ + and not ssh_conn.is_gateway_host(ssh_to_host): + cmd = self.ssh_cmd + ssh_to_host + " " + cmd + return cmd + + def adapt_cmd_to_dist(self, cmd): + env_conf = self.configuration.get_env_config() + dist = env_conf.get('distribution') + dist_version = env_conf.get('distribution_version') + translator = CliDistTranslator(dist, dist_version=dist_version) + cmd = translator.translate(cmd) + return cmd + # parse command output columns separated by whitespace # since headers can contain whitespace themselves, # it is the caller's responsibility to provide the headers @@ -126,7 +167,8 @@ class CliAccess(BinaryConverter, Fetcher): content[headers[i]] = content_parts[i] return content - def merge_ws_spillover_lines(self, lines): + @staticmethod + def merge_ws_spillover_lines(lines): # with WS-separated output, extra output sometimes spills to next line # detect that and add to the end of the previous line for our procesing pending_line = None @@ -156,7 +198,8 @@ class CliAccess(BinaryConverter, Fetcher): - header_regexp: regexp marking the start of the section - end_regexp: regexp marking the end of the section """ - def get_section_lines(self, lines, header_regexp, end_regexp): + @staticmethod + def get_section_lines(lines, header_regexp, end_regexp): if not lines: return [] header_re = re.compile(header_regexp) @@ -196,7 +239,8 @@ class CliAccess(BinaryConverter, Fetcher): if 'name' not in o and 'default' in regexp_tuple: o[name] = regexp_tuple['default'] - def find_matching_regexps(self, o, line, regexps): + @staticmethod + def find_matching_regexps(o, line, regexps): for regexp_tuple in regexps: name = regexp_tuple['name'] regex = regexp_tuple['re'] diff --git a/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py new file mode 100644 index 0000000..77f149f --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py @@ -0,0 +1,134 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from collections import deque + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchBondHostPnics(CliAccess): + BOND_DIR = '/proc/net/bonding/' + SLAVE_INTERFACE_HEADER = 'Slave Interface: ' + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, parent_id: str): + self.log.info('CliFetchBondHostPnics: checking under {}' + .format(parent_id)) + host_id = parent_id[:parent_id.rindex('-')] + cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR) + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error('CliFetchBondHostPnics: host not found: ' + host_id) + return [] + host_types = host['host_type'] + if 'Network' not in host_types and 'Compute' not in host_types: + return [] + lines = self.run_fetch_lines(cmd, host_id) + if lines and 'No such file or directory' in lines[0]: + return [] # no bonds so directory does not exist + bonds = [] + for line in lines: + bond = self.get_bond_details(host_id, line) + if bond: + bonds.append(bond) + return bonds + + def get_bond_details(self, host_id: str, interface_name: str) -> dict: + lines = self.run_fetch_lines('cat {}{}' + .format(self.BOND_DIR, interface_name), + host_id) + status, mac_address = \ + self.get_bond_status_and_mac_address(host_id, interface_name) + interface_id = '{}-{}'.format(interface_name, mac_address) + interface = { + 'host': host_id, + 'name': interface_name, + 'id': interface_id, + 'local_name': interface_name, + 'mac_address': mac_address, + 'Link detected': 'yes' if status == 'up' else 'no', + 'EtherChannel': True, + 'EtherChannel Master': '', + 'members': {} + } + # keep stack of info objects to support multi-level info + info_objects = deque([interface]) + for line in [line for line in lines if line != '']: + if line.startswith(self.SLAVE_INTERFACE_HEADER): + name = line[line.index(':')+1:].strip() + slave = { + 'name': name, + 'EtherChannel Master': interface_id + } + # remove any pending info objects, keep only interface + info_objects = deque([interface]) + info_objects.append(slave) + interface['members'][name] = slave + elif line.rstrip(':').lower().endswith('info'): + # move to lower level info object + info_name = line.rstrip(':') + upper_info_obj = info_objects[-1] + info_obj = {} + upper_info_obj[info_name] = info_obj + info_objects.append(info_obj) + else: + self.get_attribute_from_line(info_objects[-1], line) + for slave in list(interface['members'].values()): + self.set_slave_host_pnic_bond_attributes(host_id, slave, + interface_id) + return interface + + def get_bond_status_and_mac_address(self, host_id: str, name: str): + output = self.run_fetch_lines('ip link show {}'.format(name), host_id) + status_line = output[0] + status = status_line[status_line.index(' state ') + len(' state '):] + status = status[:status.index(' ')] + matches = [line.strip() for line in output if 'link/ether' in line] + if not matches: + self.log.error('Failed to find line with MAC address ' + 'for bond {} (host: {})' + .format(name, host_id)) + tokens = matches[0].split() + if len(tokens) < 2: + self.log.error('Failed to find MAC address in line: {}' + .format(matches[0])) + mac_address = tokens[1] + return status.lower(), mac_address + + def get_attribute_from_line(self, obj: dict, line: str): + if ':' not in line: + self.log.error('object {}: failed to find ":" in line: {}' + .format(obj['name'], line)) + return + attr = line[:line.index(':')] + value = line[len(attr)+1:] + obj[attr.strip()] = value.strip() + + def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id): + pnic = self.inv.find_one({ + 'environment': self.get_env(), + 'host': host, + 'type': 'host_pnic', + 'name': slave['name'] + }) + if not pnic: + self.log.error('unable to find slave pNIC {} under bond {}' + .format(slave_id, interface_id)) + return + mac_address = pnic['mac_address'] + slave_id = '{}-{}'.format(slave.get('name', ''), mac_address) + slave['mac_address'] = mac_address + slave['id'] = slave_id + pnic['EtherChannel'] = True + pnic['EtherChannel Master'] = interface_id + self.inv.set(pnic) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py index 5df4d3b..4af3ebc 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -67,21 +67,17 @@ class CliFetchHostPnics(CliAccess): tokens = None if interface is None: tokens = line.split() - name = tokens[0].strip('- :') - name = name.strip() - if name == interface_name: - line_remainder = line.strip('-')[len(interface_name)+2:] - line_remainder = line_remainder.strip(' :') - id = interface_name - interface = { - "host": host_id, - "name": id, - "local_name": interface_name, - "lines": [] - } - self.handle_line(interface, line_remainder) - if '= 25) + DbAccess.query_count_per_con += 1 + cursor = DbAccess.conn.cursor(dictionary=True) + try: + res = method(self, *args, cursor=cursor, **kwargs) + DbAccess.conn.commit() + return res + except: + DbAccess.conn.rollback() + raise + finally: + cursor.close() + return wrap + + class DbAccess(Fetcher): conn = None query_count_per_con = 0 @@ -47,10 +66,9 @@ class DbAccess(Fetcher): return DbAccess.query_count_per_con = 0 - @staticmethod - def get_neutron_db_name(): + @with_cursor + def get_neutron_db_name(self, cursor=None): # check if DB schema 'neutron' exists - cursor = DbAccess.conn.cursor(dictionary=True) cursor.execute('SHOW DATABASES') matches = [row.get('Database', '') for row in cursor if 'neutron' in row.get('Database', '')] @@ -68,6 +86,8 @@ class DbAccess(Fetcher): self.log.info("DbAccess: ****** forcing reconnect, " + "query count: %s ******", DbAccess.query_count_per_con) + DbAccess.conn.commit() + DbAccess.conn.close() DbAccess.conn = None self.conf = self.config.get("mysql") cnf = self.conf @@ -76,16 +96,15 @@ class DbAccess(Fetcher): cnf["user"], cnf["pwd"], cnf["schema"]) - def get_objects_list_for_id(self, query, object_type, id): - self.connect_to_db(DbAccess.query_count_per_con >= 25) - DbAccess.query_count_per_con += 1 + @with_cursor + def get_objects_list_for_id(self, query, object_type, object_id, + cursor=None): self.log.debug("query count: %s, running query:\n%s\n", str(DbAccess.query_count_per_con), query) - cursor = DbAccess.conn.cursor(dictionary=True) try: - if id: - cursor.execute(query, [str(id)]) + if object_id: + cursor.execute(query, [str(object_id)]) else: cursor.execute(query) except (AttributeError, mysql.connector.errors.OperationalError) as e: @@ -93,13 +112,13 @@ class DbAccess(Fetcher): self.connect_to_db(True) # try again to run the query cursor = DbAccess.conn.cursor(dictionary=True) - if id: - cursor.execute(query, [str(id)]) + if object_id: + cursor.execute(query, [str(object_id)]) else: cursor.execute(query) rows = [] - for row in cursor: + for row in cursor.fetchall(): rows.append(row) return rows diff --git a/app/discover/fetchers/db/db_fetch_host_network_agents.py b/app/discover/fetchers/db/db_fetch_host_network_agents.py index c323573..7d415f2 100644 --- a/app/discover/fetchers/db/db_fetch_host_network_agents.py +++ b/app/discover/fetchers/db/db_fetch_host_network_agents.py @@ -27,9 +27,8 @@ class DbFetchHostNetworkAgents(DbAccess): host_id = id[:-1 * len("-network_agents")] results = self.get_objects_list_for_id(query, "network_agent", host_id) mechanism_drivers = self.env_config['mechanism_drivers'] - id_prefix = mechanism_drivers[0] if mechanism_drivers else 'network_agent' for o in results: o["configurations"] = json.loads(o["configurations"]) o["name"] = o["binary"] - o['id'] = id_prefix + '-' + o['id'] + o['id'] = o['name'] + '-' + o['id'] return results diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py index 3e3f4e1..f7eb8bd 100644 --- a/app/discover/fetchers/db/db_fetch_oteps.py +++ b/app/discover/fetchers/db/db_fetch_oteps.py @@ -35,7 +35,9 @@ class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton): table_name = "{}.ml2_{}_endpoints".format(self.neutron_db, tunnel_type) env_config = self.config.get_env_config() distribution = env_config["distribution"] - if distribution == "Canonical-icehouse": + distribution_version = env_config["distribution_version"] + dist_ver = "{}-{}".format(distribution, distribution_version) + if dist_ver == "Canonical-icehouse": # for Icehouse, we only get IP address from the DB, so take the # host IP address and from the host data in Mongo host = self.inv.get_by_id(self.get_env(), host_id) diff --git a/app/discover/fetchers/db/db_fetch_vedges_ovs.py b/app/discover/fetchers/db/db_fetch_vedges_ovs.py index 838ccb9..f516d10 100644 --- a/app/discover/fetchers/db/db_fetch_vedges_ovs.py +++ b/app/discover/fetchers/db/db_fetch_vedges_ovs.py @@ -24,8 +24,8 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$") self.port_line_header_prefix = " " * 8 + "Port " - def get(self, id): - host_id = id[:id.rindex('-')] + def get(self, parent_id): + host_id = parent_id[:parent_id.rindex('-')] results = self.get_objects_list_for_id( """ SELECT * @@ -66,11 +66,11 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): if not port_matches: continue port = {} - id = port_matches.group(1) + port_id = port_matches.group(1) name = port_matches.group(2) is_internal = port_matches.group(3) == " (internal)" port["internal"] = is_internal - port["id"] = id + port["id"] = port_id port["name"] = name ports[name] = port return ports @@ -106,7 +106,7 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): if "tunneling_ip" not in doc["configurations"]: return {} if not doc["configurations"]["tunneling_ip"]: - self.get_bridge_pnic(doc) + self.get_pnics(doc) return {} # read the 'br-tun' interface ports @@ -148,31 +148,48 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): tunnel_ports[port["name"]] = port return tunnel_ports - def get_bridge_pnic(self, doc): - conf = doc["configurations"] - if "bridge_mappings" not in conf or not conf["bridge_mappings"]: - return - for v in conf["bridge_mappings"].values(): br = v - ifaces_list_lines = self.run_fetch_lines("ovs-vsctl list-ifaces " + br, - doc["host"]) - br_pnic_postfix = br + "--br-" - interface = "" + def get_pnics(self, vedge) -> dict: + bridges = vedge["configurations"].get("bridge_mappings", {}) + pnics = {} + for bridge in bridges.values(): + self.get_bridge_pnic(pnics, vedge, bridge) + return pnics + + MIRANTIS_DIST = "Mirantis" + + def get_bridge_pnic(self, pnics: dict, vedge: dict, bridge: dict): + cmd = "ovs-vsctl list-ifaces {}".format(bridge) + ifaces_list_lines = self.run_fetch_lines(cmd, vedge["host"]) + env_config = self.configuration.get_env_config() + distribution = env_config.get("distribution") + dist_version = env_config.get("distribution_version") + use_br_postfix = distribution == self.MIRANTIS_DIST and \ + dist_version in ["6.0", "7.0", "8.0"] for l in ifaces_list_lines: - if l.startswith(br_pnic_postfix): - interface = l[len(br_pnic_postfix):] - break - if not interface: - return - doc["pnic"] = interface + if use_br_postfix: + br_pnic_postfix = "{}--br-".format(bridge) + interface = l[len(br_pnic_postfix):] \ + if l.startswith(br_pnic_postfix) \ + else "" + else: + interface = l + if interface: + pnic = self.find_pnic_for_interface(vedge, interface) + if pnic: + pnics[pnic["name"]] = pnic + + def find_pnic_for_interface(self, vedge, interface): # add port ID to pNIC pnic = self.inv.find_items({ "environment": self.get_env(), "type": "host_pnic", - "host": doc["host"], + "host": vedge["host"], "name": interface }, get_single=True) if not pnic: return - port = doc["ports"][interface] - pnic["port_id"] = port["id"] + vedge["pnic"] = interface + port = vedge["ports"].get(interface, {}) + pnic["port_id"] = port.get("id", "") self.inv.set(pnic) + return pnic -- cgit 1.2.3-korg