diff options
author | 2017-09-29 01:38:18 +0300 | |
---|---|---|
committer | 2017-09-29 01:38:18 +0300 | |
commit | d32f75145676bacefde0d08a14680a5984623451 (patch) | |
tree | 4b5eaf1107e6973b1eac636309a99c83074acbfc /app/discover/fetchers/cli | |
parent | 0c5426cd309d720db1e30641e43d311ee0b751b0 (diff) |
release 1.0 calipso for opnfv apex
Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2
Signed-off-by: Koren Lev <korenlev@gmail.com>
Diffstat (limited to 'app/discover/fetchers/cli')
7 files changed, 216 insertions, 40 deletions
diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py index 275a3e8..c77b22a 100644 --- a/app/discover/fetchers/cli/cli_access.py +++ b/app/discover/fetchers/cli/cli_access.py @@ -12,6 +12,7 @@ import time from discover.fetcher import Fetcher from utils.binary_converter import BinaryConverter +from utils.cli_dist_translator import CliDistTranslator from utils.logging.console_logger import ConsoleLogger from utils.ssh_conn import SshConn @@ -41,11 +42,16 @@ class CliAccess(BinaryConverter, Fetcher): def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False, ssh=None, use_sudo=True): ssh_conn = ssh if ssh else SshConn(ssh_to_host) - if use_sudo and not cmd.strip().startswith("sudo "): - cmd = "sudo " + cmd - if not on_gateway and ssh_to_host \ - and not ssh_conn.is_gateway_host(ssh_to_host): - cmd = self.ssh_cmd + ssh_to_host + " " + cmd + commands = self.adapt_cmd_to_env(ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host) + out = '' + for c in commands: + out += self.run_single_command(c, ssh_conn, ssh_to_host, + enable_cache=enable_cache) + return out + + def run_single_command(self, cmd, ssh_conn, ssh_to_host="", + enable_cache=True): curr_time = time.time() cmd_path = ssh_to_host + ',' + cmd if enable_cache and cmd_path in self.cached_commands: @@ -73,9 +79,44 @@ class CliAccess(BinaryConverter, Fetcher): ret = out.splitlines() # if split by whitespace did not work, try splitting by "\\n" if len(ret) == 1: - ret = [l for l in out.split("\\n") if l != ""] + ret = [line for line in out.split("\\n") if line != ""] return ret + MULTI_COMMAND_SEPARATOR = ';;;' + + @staticmethod + def handle_split_cmd(cmd: str): + if CliAccess.MULTI_COMMAND_SEPARATOR in cmd: + return cmd.split(CliAccess.MULTI_COMMAND_SEPARATOR) + return [cmd] + + def adapt_cmd_to_env(self, ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host): + cmd = self.adapt_cmd_to_dist(cmd) + commands = self.handle_split_cmd(cmd) + return [self.adapt_cmd_to_environment(c, use_sudo, on_gateway, + ssh_to_host, ssh_conn) + for c in commands] + + def adapt_cmd_to_environment(self, cmd, use_sudo, on_gateway, ssh_to_host, + ssh_conn): + if self.configuration.environment["distribution"] == "Mercury": + use_sudo = False + if use_sudo and not cmd.strip().startswith("sudo "): + cmd = "sudo " + cmd + if not on_gateway and ssh_to_host \ + and not ssh_conn.is_gateway_host(ssh_to_host): + cmd = self.ssh_cmd + ssh_to_host + " " + cmd + return cmd + + def adapt_cmd_to_dist(self, cmd): + env_conf = self.configuration.get_env_config() + dist = env_conf.get('distribution') + dist_version = env_conf.get('distribution_version') + translator = CliDistTranslator(dist, dist_version=dist_version) + cmd = translator.translate(cmd) + return cmd + # parse command output columns separated by whitespace # since headers can contain whitespace themselves, # it is the caller's responsibility to provide the headers @@ -126,7 +167,8 @@ class CliAccess(BinaryConverter, Fetcher): content[headers[i]] = content_parts[i] return content - def merge_ws_spillover_lines(self, lines): + @staticmethod + def merge_ws_spillover_lines(lines): # with WS-separated output, extra output sometimes spills to next line # detect that and add to the end of the previous line for our procesing pending_line = None @@ -156,7 +198,8 @@ class CliAccess(BinaryConverter, Fetcher): - header_regexp: regexp marking the start of the section - end_regexp: regexp marking the end of the section """ - def get_section_lines(self, lines, header_regexp, end_regexp): + @staticmethod + def get_section_lines(lines, header_regexp, end_regexp): if not lines: return [] header_re = re.compile(header_regexp) @@ -196,7 +239,8 @@ class CliAccess(BinaryConverter, Fetcher): if 'name' not in o and 'default' in regexp_tuple: o[name] = regexp_tuple['default'] - def find_matching_regexps(self, o, line, regexps): + @staticmethod + def find_matching_regexps(o, line, regexps): for regexp_tuple in regexps: name = regexp_tuple['name'] regex = regexp_tuple['re'] diff --git a/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py new file mode 100644 index 0000000..77f149f --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py @@ -0,0 +1,134 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from collections import deque + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchBondHostPnics(CliAccess): + BOND_DIR = '/proc/net/bonding/' + SLAVE_INTERFACE_HEADER = 'Slave Interface: ' + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, parent_id: str): + self.log.info('CliFetchBondHostPnics: checking under {}' + .format(parent_id)) + host_id = parent_id[:parent_id.rindex('-')] + cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR) + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error('CliFetchBondHostPnics: host not found: ' + host_id) + return [] + host_types = host['host_type'] + if 'Network' not in host_types and 'Compute' not in host_types: + return [] + lines = self.run_fetch_lines(cmd, host_id) + if lines and 'No such file or directory' in lines[0]: + return [] # no bonds so directory does not exist + bonds = [] + for line in lines: + bond = self.get_bond_details(host_id, line) + if bond: + bonds.append(bond) + return bonds + + def get_bond_details(self, host_id: str, interface_name: str) -> dict: + lines = self.run_fetch_lines('cat {}{}' + .format(self.BOND_DIR, interface_name), + host_id) + status, mac_address = \ + self.get_bond_status_and_mac_address(host_id, interface_name) + interface_id = '{}-{}'.format(interface_name, mac_address) + interface = { + 'host': host_id, + 'name': interface_name, + 'id': interface_id, + 'local_name': interface_name, + 'mac_address': mac_address, + 'Link detected': 'yes' if status == 'up' else 'no', + 'EtherChannel': True, + 'EtherChannel Master': '', + 'members': {} + } + # keep stack of info objects to support multi-level info + info_objects = deque([interface]) + for line in [line for line in lines if line != '']: + if line.startswith(self.SLAVE_INTERFACE_HEADER): + name = line[line.index(':')+1:].strip() + slave = { + 'name': name, + 'EtherChannel Master': interface_id + } + # remove any pending info objects, keep only interface + info_objects = deque([interface]) + info_objects.append(slave) + interface['members'][name] = slave + elif line.rstrip(':').lower().endswith('info'): + # move to lower level info object + info_name = line.rstrip(':') + upper_info_obj = info_objects[-1] + info_obj = {} + upper_info_obj[info_name] = info_obj + info_objects.append(info_obj) + else: + self.get_attribute_from_line(info_objects[-1], line) + for slave in list(interface['members'].values()): + self.set_slave_host_pnic_bond_attributes(host_id, slave, + interface_id) + return interface + + def get_bond_status_and_mac_address(self, host_id: str, name: str): + output = self.run_fetch_lines('ip link show {}'.format(name), host_id) + status_line = output[0] + status = status_line[status_line.index(' state ') + len(' state '):] + status = status[:status.index(' ')] + matches = [line.strip() for line in output if 'link/ether' in line] + if not matches: + self.log.error('Failed to find line with MAC address ' + 'for bond {} (host: {})' + .format(name, host_id)) + tokens = matches[0].split() + if len(tokens) < 2: + self.log.error('Failed to find MAC address in line: {}' + .format(matches[0])) + mac_address = tokens[1] + return status.lower(), mac_address + + def get_attribute_from_line(self, obj: dict, line: str): + if ':' not in line: + self.log.error('object {}: failed to find ":" in line: {}' + .format(obj['name'], line)) + return + attr = line[:line.index(':')] + value = line[len(attr)+1:] + obj[attr.strip()] = value.strip() + + def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id): + pnic = self.inv.find_one({ + 'environment': self.get_env(), + 'host': host, + 'type': 'host_pnic', + 'name': slave['name'] + }) + if not pnic: + self.log.error('unable to find slave pNIC {} under bond {}' + .format(slave_id, interface_id)) + return + mac_address = pnic['mac_address'] + slave_id = '{}-{}'.format(slave.get('name', ''), mac_address) + slave['mac_address'] = mac_address + slave['id'] = slave_id + pnic['EtherChannel'] = True + pnic['EtherChannel Master'] = interface_id + self.inv.set(pnic) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py index 5df4d3b..4af3ebc 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -67,21 +67,17 @@ class CliFetchHostPnics(CliAccess): tokens = None if interface is None: tokens = line.split() - name = tokens[0].strip('- :') - name = name.strip() - if name == interface_name: - line_remainder = line.strip('-')[len(interface_name)+2:] - line_remainder = line_remainder.strip(' :') - id = interface_name - interface = { - "host": host_id, - "name": id, - "local_name": interface_name, - "lines": [] - } - self.handle_line(interface, line_remainder) - if '<UP,' in line: - status_up = True + line_remainder = line.strip('-')[len(interface_name)+2:] + line_remainder = line_remainder.strip(' :') + interface = { + "host": host_id, + "name": interface_name, + "local_name": interface_name, + "lines": [] + } + self.handle_line(interface, line_remainder) + if '<UP,' in line: + status_up = True if status_up is None: if tokens is None: tokens = line.split() diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservice.py b/app/discover/fetchers/cli/cli_fetch_host_vservice.py index 9f8173f..ae7c656 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_vservice.py +++ b/app/discover/fetchers/cli/cli_fetch_host_vservice.py @@ -31,35 +31,37 @@ class CliFetchHostVservice(CliAccess, DbAccess): def set_details(self, host_id, r): # keep the index without prefix id_full = r["local_service_id"].strip() - prefix = id_full[1:id_full.index('-')] - id_clean = id_full[id_full.index('-') + 1:] - r["service_type"] = prefix - name = self.get_router_name(r, id_clean) if prefix == "router" \ + prefix = id_full[:id_full.index('-')] + id_clean = id_full[len(prefix)+1:] + r["service_type"] = prefix[1:] + name = self.get_router_name(r, id_clean) \ + if r["service_type"] == "router" \ else self.get_network_name(id_clean) r["name"] = prefix + "-" + name r["host"] = host_id - r["id"] = host_id + "-" + id_full + r["id"] = "{}-{}".format(host_id, id_full) self.set_agent_type(r) - def get_network_name(self, id): + def get_network_name(self, network_id): query = """ SELECT name FROM {}.networks WHERE id = %s """.format(self.neutron_db) - results = self.get_objects_list_for_id(query, "router", id) + results = self.get_objects_list_for_id(query, "router", network_id) if not list(results): - return id + return network_id for db_row in results: return db_row["name"] - def get_router_name(self, r, id): + def get_router_name(self, r, router_id): query = """ SELECT * FROM {}.routers WHERE id = %s """.format(self.neutron_db) - results = self.get_objects_list_for_id(query, "router", id.strip()) + results = self.get_objects_list_for_id(query, "router", + router_id.strip()) for db_row in results: r.update(db_row) return r["name"] diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservices.py b/app/discover/fetchers/cli/cli_fetch_host_vservices.py index 9b62dcb..b9496bc 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_vservices.py +++ b/app/discover/fetchers/cli/cli_fetch_host_vservices.py @@ -19,7 +19,7 @@ class CliFetchHostVservices(CliFetchHostVservice): if "Network" not in host["host_type"]: return [] services_ids = [l[:l.index(' ')] if ' ' in l else l - for l in self.run_fetch_lines("ip netns", host_id)] + for l in self.run_fetch_lines("ip netns list", host_id)] results = [{"local_service_id": s} for s in services_ids if self.type_re.match(s)] for r in results: self.set_details(host_id, r) diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py index 4de1840..bb1e7fc 100644 --- a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py +++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py @@ -58,7 +58,7 @@ class CliFetchInstanceVnicsBase(CliAccess): def set_vnic_properties(self, v, instance): v["name"] = self.get_vnic_name(v, instance) - v["id"] = v["name"] + v["id"] = "{}-{}".format(instance["host"], v["name"]) v["vnic_type"] = "instance_vnic" v["host"] = instance["host"] v["instance_id"] = instance["id"] diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py index d10d99e..239ecd7 100644 --- a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py +++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py @@ -42,7 +42,7 @@ class CliFetchVserviceVnics(CliAccess): return [] if "Network" not in host["host_type"]: return [] - lines = self.run_fetch_lines("ip netns", host_id) + lines = self.run_fetch_lines("ip netns list", host_id) ret = [] for l in [l for l in lines if l.startswith("qdhcp") or l.startswith("qrouter")]: @@ -68,7 +68,7 @@ class CliFetchVserviceVnics(CliAccess): current = None else: line_remainder = matches.group(2) - vservice_id = host + "-" + service + master_parent_id = "{}-{}".format(host, service) current = { "id": host + "-" + name, "type": "vnic", @@ -76,9 +76,9 @@ class CliFetchVserviceVnics(CliAccess): "host": host, "name": name, "master_parent_type": "vservice", - "master_parent_id": vservice_id, + "master_parent_id": master_parent_id, "parent_type": "vnics_folder", - "parent_id": vservice_id + "-vnics", + "parent_id": "{}-vnics".format(master_parent_id), "parent_text": "vNICs", "lines": [] } |