From d32f75145676bacefde0d08a14680a5984623451 Mon Sep 17 00:00:00 2001
From: Koren Lev <korenlev@gmail.com>
Date: Fri, 29 Sep 2017 01:38:18 +0300
Subject: release 1.0 calipso for opnfv apex

Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2
Signed-off-by: Koren Lev <korenlev@gmail.com>
---
 app/discover/fetchers/cli/cli_access.py            |  62 ++++++++--
 .../fetchers/cli/cli_fetch_bond_host_pnics.py      | 134 +++++++++++++++++++++
 app/discover/fetchers/cli/cli_fetch_host_pnics.py  |  26 ++--
 .../fetchers/cli/cli_fetch_host_vservice.py        |  22 ++--
 .../fetchers/cli/cli_fetch_host_vservices.py       |   2 +-
 .../fetchers/cli/cli_fetch_instance_vnics_base.py  |   2 +-
 .../fetchers/cli/cli_fetch_vservice_vnics.py       |   8 +-
 7 files changed, 216 insertions(+), 40 deletions(-)
 create mode 100644 app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py

(limited to 'app/discover/fetchers/cli')

diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py
index 275a3e8..c77b22a 100644
--- a/app/discover/fetchers/cli/cli_access.py
+++ b/app/discover/fetchers/cli/cli_access.py
@@ -12,6 +12,7 @@ import time
 
 from discover.fetcher import Fetcher
 from utils.binary_converter import BinaryConverter
+from utils.cli_dist_translator import CliDistTranslator
 from utils.logging.console_logger import ConsoleLogger
 from utils.ssh_conn import SshConn
 
@@ -41,11 +42,16 @@ class CliAccess(BinaryConverter, Fetcher):
     def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False,
             ssh=None, use_sudo=True):
         ssh_conn = ssh if ssh else SshConn(ssh_to_host)
-        if use_sudo and not cmd.strip().startswith("sudo "):
-            cmd = "sudo " + cmd
-        if not on_gateway and ssh_to_host \
-                and not ssh_conn.is_gateway_host(ssh_to_host):
-            cmd = self.ssh_cmd + ssh_to_host + " " + cmd
+        commands = self.adapt_cmd_to_env(ssh_conn, cmd, use_sudo, on_gateway,
+                                         ssh_to_host)
+        out = ''
+        for c in commands:
+            out += self.run_single_command(c, ssh_conn, ssh_to_host,
+                                           enable_cache=enable_cache)
+        return out
+
+    def run_single_command(self, cmd, ssh_conn, ssh_to_host="",
+                           enable_cache=True):
         curr_time = time.time()
         cmd_path = ssh_to_host + ',' + cmd
         if enable_cache and cmd_path in self.cached_commands:
@@ -73,9 +79,44 @@ class CliAccess(BinaryConverter, Fetcher):
         ret = out.splitlines()
         # if split by whitespace did not work, try splitting by "\\n"
         if len(ret) == 1:
-            ret = [l for l in out.split("\\n") if l != ""]
+            ret = [line for line in out.split("\\n") if line != ""]
         return ret
 
+    MULTI_COMMAND_SEPARATOR = ';;;'
+
+    @staticmethod
+    def handle_split_cmd(cmd: str):
+        if CliAccess.MULTI_COMMAND_SEPARATOR in cmd:
+            return cmd.split(CliAccess.MULTI_COMMAND_SEPARATOR)
+        return [cmd]
+
+    def adapt_cmd_to_env(self, ssh_conn, cmd, use_sudo, on_gateway,
+                         ssh_to_host):
+        cmd = self.adapt_cmd_to_dist(cmd)
+        commands = self.handle_split_cmd(cmd)
+        return [self.adapt_cmd_to_environment(c, use_sudo, on_gateway,
+                                              ssh_to_host, ssh_conn)
+                for c in commands]
+
+    def adapt_cmd_to_environment(self, cmd, use_sudo, on_gateway, ssh_to_host,
+                                 ssh_conn):
+        if self.configuration.environment["distribution"] == "Mercury":
+            use_sudo = False
+        if use_sudo and not cmd.strip().startswith("sudo "):
+            cmd = "sudo " + cmd
+        if not on_gateway and ssh_to_host \
+                and not ssh_conn.is_gateway_host(ssh_to_host):
+            cmd = self.ssh_cmd + ssh_to_host + " " + cmd
+        return cmd
+
+    def adapt_cmd_to_dist(self, cmd):
+        env_conf = self.configuration.get_env_config()
+        dist = env_conf.get('distribution')
+        dist_version = env_conf.get('distribution_version')
+        translator = CliDistTranslator(dist, dist_version=dist_version)
+        cmd = translator.translate(cmd)
+        return cmd
+
     # parse command output columns separated by whitespace
     # since headers can contain whitespace themselves,
     # it is the caller's responsibility to provide the headers
@@ -126,7 +167,8 @@ class CliAccess(BinaryConverter, Fetcher):
             content[headers[i]] = content_parts[i]
         return content
 
-    def merge_ws_spillover_lines(self, lines):
+    @staticmethod
+    def merge_ws_spillover_lines(lines):
         # with WS-separated output, extra output sometimes spills to next line
         # detect that and add to the end of the previous line for our procesing
         pending_line = None
@@ -156,7 +198,8 @@ class CliAccess(BinaryConverter, Fetcher):
     - header_regexp: regexp marking the start of the section
     - end_regexp: regexp marking the end of the section
     """
-    def get_section_lines(self, lines, header_regexp, end_regexp):
+    @staticmethod
+    def get_section_lines(lines, header_regexp, end_regexp):
         if not lines:
             return []
         header_re = re.compile(header_regexp)
@@ -196,7 +239,8 @@ class CliAccess(BinaryConverter, Fetcher):
             if 'name' not in o and 'default' in regexp_tuple:
                 o[name] = regexp_tuple['default']
 
-    def find_matching_regexps(self, o, line, regexps):
+    @staticmethod
+    def find_matching_regexps(o, line, regexps):
         for regexp_tuple in regexps:
             name = regexp_tuple['name']
             regex = regexp_tuple['re']
diff --git a/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py
new file mode 100644
index 0000000..77f149f
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py
@@ -0,0 +1,134 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from collections import deque
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchBondHostPnics(CliAccess):
+    BOND_DIR = '/proc/net/bonding/'
+    SLAVE_INTERFACE_HEADER = 'Slave Interface: '
+
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, parent_id: str):
+        self.log.info('CliFetchBondHostPnics: checking under {}'
+                      .format(parent_id))
+        host_id = parent_id[:parent_id.rindex('-')]
+        cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR)
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host:
+            self.log.error('CliFetchBondHostPnics: host not found: ' + host_id)
+            return []
+        host_types = host['host_type']
+        if 'Network' not in host_types and 'Compute' not in host_types:
+            return []
+        lines = self.run_fetch_lines(cmd, host_id)
+        if lines and 'No such file or directory' in lines[0]:
+            return []  # no bonds so directory does not exist
+        bonds = []
+        for line in lines:
+            bond = self.get_bond_details(host_id, line)
+            if bond:
+                bonds.append(bond)
+        return bonds
+
+    def get_bond_details(self, host_id: str, interface_name: str) -> dict:
+        lines = self.run_fetch_lines('cat {}{}'
+                                     .format(self.BOND_DIR, interface_name),
+                                     host_id)
+        status, mac_address = \
+            self.get_bond_status_and_mac_address(host_id, interface_name)
+        interface_id = '{}-{}'.format(interface_name, mac_address)
+        interface = {
+            'host': host_id,
+            'name': interface_name,
+            'id': interface_id,
+            'local_name': interface_name,
+            'mac_address': mac_address,
+            'Link detected': 'yes' if status == 'up' else 'no',
+            'EtherChannel': True,
+            'EtherChannel Master': '',
+            'members': {}
+        }
+        # keep stack of info objects to support multi-level info
+        info_objects = deque([interface])
+        for line in [line for line in lines if line != '']:
+            if line.startswith(self.SLAVE_INTERFACE_HEADER):
+                name = line[line.index(':')+1:].strip()
+                slave = {
+                    'name': name,
+                    'EtherChannel Master': interface_id
+                }
+                # remove any pending info objects, keep only interface
+                info_objects = deque([interface])
+                info_objects.append(slave)
+                interface['members'][name] = slave
+            elif line.rstrip(':').lower().endswith('info'):
+                # move to lower level info object
+                info_name = line.rstrip(':')
+                upper_info_obj = info_objects[-1]
+                info_obj = {}
+                upper_info_obj[info_name] = info_obj
+                info_objects.append(info_obj)
+            else:
+                self.get_attribute_from_line(info_objects[-1], line)
+        for slave in list(interface['members'].values()):
+            self.set_slave_host_pnic_bond_attributes(host_id, slave,
+                                                     interface_id)
+        return interface
+
+    def get_bond_status_and_mac_address(self, host_id: str, name: str):
+        output = self.run_fetch_lines('ip link show {}'.format(name), host_id)
+        status_line = output[0]
+        status = status_line[status_line.index(' state ') + len(' state '):]
+        status = status[:status.index(' ')]
+        matches = [line.strip() for line in output if 'link/ether' in line]
+        if not matches:
+            self.log.error('Failed to find line with MAC address '
+                           'for bond {} (host: {})'
+                           .format(name, host_id))
+        tokens = matches[0].split()
+        if len(tokens) < 2:
+            self.log.error('Failed to find MAC address in line: {}'
+                           .format(matches[0]))
+        mac_address = tokens[1]
+        return status.lower(), mac_address
+
+    def get_attribute_from_line(self, obj: dict, line: str):
+        if ':' not in line:
+            self.log.error('object {}: failed to find ":" in line: {}'
+                           .format(obj['name'], line))
+            return
+        attr = line[:line.index(':')]
+        value = line[len(attr)+1:]
+        obj[attr.strip()] = value.strip()
+
+    def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id):
+        pnic = self.inv.find_one({
+            'environment': self.get_env(),
+            'host': host,
+            'type': 'host_pnic',
+            'name': slave['name']
+        })
+        if not pnic:
+            self.log.error('unable to find slave pNIC {} under bond {}'
+                           .format(slave_id, interface_id))
+            return
+        mac_address = pnic['mac_address']
+        slave_id = '{}-{}'.format(slave.get('name', ''), mac_address)
+        slave['mac_address'] = mac_address
+        slave['id'] = slave_id
+        pnic['EtherChannel'] = True
+        pnic['EtherChannel Master'] = interface_id
+        self.inv.set(pnic)
diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
index 5df4d3b..4af3ebc 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
@@ -67,21 +67,17 @@ class CliFetchHostPnics(CliAccess):
             tokens = None
             if interface is None:
                 tokens = line.split()
-                name = tokens[0].strip('- :')
-                name = name.strip()
-                if name == interface_name:
-                    line_remainder = line.strip('-')[len(interface_name)+2:]
-                    line_remainder = line_remainder.strip(' :')
-                    id = interface_name
-                    interface = {
-                        "host": host_id,
-                        "name": id,
-                        "local_name": interface_name,
-                        "lines": []
-                    }
-                    self.handle_line(interface, line_remainder)
-                    if '<UP,' in line:
-                        status_up = True
+                line_remainder = line.strip('-')[len(interface_name)+2:]
+                line_remainder = line_remainder.strip(' :')
+                interface = {
+                    "host": host_id,
+                    "name": interface_name,
+                    "local_name": interface_name,
+                    "lines": []
+                }
+                self.handle_line(interface, line_remainder)
+                if '<UP,' in line:
+                    status_up = True
             if status_up is None:
                 if tokens is None:
                     tokens = line.split()
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservice.py b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
index 9f8173f..ae7c656 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_vservice.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
@@ -31,35 +31,37 @@ class CliFetchHostVservice(CliAccess, DbAccess):
     def set_details(self, host_id, r):
         # keep the index without prefix
         id_full = r["local_service_id"].strip()
-        prefix = id_full[1:id_full.index('-')]
-        id_clean = id_full[id_full.index('-') + 1:]
-        r["service_type"] = prefix
-        name = self.get_router_name(r, id_clean) if prefix == "router" \
+        prefix = id_full[:id_full.index('-')]
+        id_clean = id_full[len(prefix)+1:]
+        r["service_type"] = prefix[1:]
+        name = self.get_router_name(r, id_clean) \
+            if r["service_type"] == "router" \
             else self.get_network_name(id_clean)
         r["name"] = prefix + "-" + name
         r["host"] = host_id
-        r["id"] = host_id + "-" + id_full
+        r["id"] = "{}-{}".format(host_id, id_full)
         self.set_agent_type(r)
 
-    def get_network_name(self, id):
+    def get_network_name(self, network_id):
         query = """
                 SELECT name
                 FROM {}.networks
                 WHERE id = %s
                 """.format(self.neutron_db)
-        results = self.get_objects_list_for_id(query, "router", id)
+        results = self.get_objects_list_for_id(query, "router", network_id)
         if not list(results):
-            return id
+            return network_id
         for db_row in results:
             return db_row["name"]
 
-    def get_router_name(self, r, id):
+    def get_router_name(self, r, router_id):
         query = """
                 SELECT *
                 FROM {}.routers
                 WHERE id = %s
                 """.format(self.neutron_db)
-        results = self.get_objects_list_for_id(query, "router", id.strip())
+        results = self.get_objects_list_for_id(query, "router",
+                                               router_id.strip())
         for db_row in results:
             r.update(db_row)
         return r["name"]
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservices.py b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
index 9b62dcb..b9496bc 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_vservices.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
@@ -19,7 +19,7 @@ class CliFetchHostVservices(CliFetchHostVservice):
         if "Network" not in host["host_type"]:
             return []
         services_ids = [l[:l.index(' ')] if ' ' in l else l
-                        for l in self.run_fetch_lines("ip netns", host_id)]
+                        for l in self.run_fetch_lines("ip netns list", host_id)]
         results = [{"local_service_id": s} for s in services_ids if self.type_re.match(s)]
         for r in results:
             self.set_details(host_id, r)
diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
index 4de1840..bb1e7fc 100644
--- a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
+++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
@@ -58,7 +58,7 @@ class CliFetchInstanceVnicsBase(CliAccess):
 
     def set_vnic_properties(self, v, instance):
         v["name"] = self.get_vnic_name(v, instance)
-        v["id"] = v["name"]
+        v["id"] = "{}-{}".format(instance["host"], v["name"])
         v["vnic_type"] = "instance_vnic"
         v["host"] = instance["host"]
         v["instance_id"] = instance["id"]
diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
index d10d99e..239ecd7 100644
--- a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
@@ -42,7 +42,7 @@ class CliFetchVserviceVnics(CliAccess):
             return []
         if "Network" not in host["host_type"]:
             return []
-        lines = self.run_fetch_lines("ip netns", host_id)
+        lines = self.run_fetch_lines("ip netns list", host_id)
         ret = []
         for l in [l for l in lines
                   if l.startswith("qdhcp") or l.startswith("qrouter")]:
@@ -68,7 +68,7 @@ class CliFetchVserviceVnics(CliAccess):
                     current = None
                 else:
                     line_remainder = matches.group(2)
-                    vservice_id = host + "-" + service
+                    master_parent_id = "{}-{}".format(host, service)
                     current = {
                         "id": host + "-" + name,
                         "type": "vnic",
@@ -76,9 +76,9 @@ class CliFetchVserviceVnics(CliAccess):
                         "host": host,
                         "name": name,
                         "master_parent_type": "vservice",
-                        "master_parent_id": vservice_id,
+                        "master_parent_id": master_parent_id,
                         "parent_type": "vnics_folder",
-                        "parent_id": vservice_id + "-vnics",
+                        "parent_id": "{}-vnics".format(master_parent_id),
                         "parent_text": "vNICs",
                         "lines": []
                     }
-- 
cgit 1.2.3-korg