release 1.0 calipso for opnfv apex

Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2
Signed-off-by: Koren Lev <korenlev@gmail.com>
(cherry picked from commit d32f75145676bacefde0d08a14680a5984623451)

27 files changed, 375 insertions, 144 deletions

diff --git a/app/discover/event_manager.py b/app/discover/event_manager.py
index 6a56912..e2f8282 100644
--- a/app/discover/event_manager.py
+++ b/app/discover/event_manager.py
@@ -40,12 +40,18 @@ class EventManager(Manager):
     }
 
     LISTENERS = {
-        'Mirantis-6.0': DefaultListener,
-        'Mirantis-7.0': DefaultListener,
-        'Mirantis-8.0': DefaultListener,
-        'RDO-Mitaka': DefaultListener,
-        'RDO-Liberty': DefaultListener,
-        'Apex-Euphrates': DefaultListener,
+        'Mirantis': {
+            '6.0': DefaultListener,
+            '7.0': DefaultListener,
+            '8.0': DefaultListener,
+        },
+        'RDO': {
+            'Mitaka': DefaultListener,
+            'Liberty': DefaultListener,
+        },
+        'Apex': {
+            'Euphrates': DefaultListener,
+        },
     }
 
     def __init__(self):
@@ -105,7 +111,8 @@ class EventManager(Manager):
 
     def get_listener(self, env: str):
         env_config = self.inv.get_env_config(env)
-        return self.LISTENERS.get(env_config.get('distribution'))
+        return (self.LISTENERS.get(env_config.get('distribution'), {})
+                              .get(env_config.get('distribution_version')))
 
     def listen_to_events(self, listener: ListenerBase, env_name: str, process_vars: dict):
         listener.listen({
diff --git a/app/discover/events/event_interface_add.py b/app/discover/events/event_interface_add.py
index 698559c..e54bedb 100644
--- a/app/discover/events/event_interface_add.py
+++ b/app/discover/events/event_interface_add.py
@@ -83,11 +83,12 @@ class EventInterfaceAdd(EventBase):
 
     def handle(self, env, values):
         interface = values['payload']['router_interface']
+        project_id = values['_context_project_id']
         project = values['_context_project_name']
         host_id = values["publisher_id"].replace("network.", "", 1)
         port_id = interface['port_id']
         subnet_id = interface['subnet_id']
-        router_id = encode_router_id(host_id, interface['id'])
+        router_id = encode_router_id(interface['id'])
 
         network_document = self.inv.get_by_field(env, "network", "subnet_ids",
                                                  subnet_id, get_single=True)
@@ -98,10 +99,10 @@ class EventInterfaceAdd(EventBase):
         network_id = network_document['id']
 
         # add router-interface port document.
-        if len(ApiAccess.regions) == 0:
+        if not ApiAccess.regions:
             fetcher = ApiFetchRegions()
             fetcher.set_env(env)
-            fetcher.get(None)
+            fetcher.get(project_id)
         port_doc = EventSubnetAdd().add_port_document(env, port_id,
                                                       network_name=network_name)
 
diff --git a/app/discover/events/event_interface_delete.py b/app/discover/events/event_interface_delete.py
index b1df978..f4ec400 100644
--- a/app/discover/events/event_interface_delete.py
+++ b/app/discover/events/event_interface_delete.py
@@ -18,8 +18,7 @@ class EventInterfaceDelete(EventDeleteBase):
     def handle(self, env, values):
         interface = values['payload']['router_interface']
         port_id = interface['port_id']
-        host_id = values["publisher_id"].replace("network.", "", 1)
-        router_id = encode_router_id(host_id, interface['id'])
+        router_id = encode_router_id(interface['id'])
 
         # update router document
         port_doc = self.inv.get_by_id(env, port_id)
diff --git a/app/discover/events/event_router_add.py b/app/discover/events/event_router_add.py
index 3c1c9e2..1fb2244 100644
--- a/app/discover/events/event_router_add.py
+++ b/app/discover/events/event_router_add.py
@@ -96,7 +96,7 @@ class EventRouterAdd(EventBase):
         router = values['payload']['router']
         host_id = values["publisher_id"].replace("network.", "", 1)
         project_id = values['_context_project_id']
-        router_id = encode_router_id(host_id, router['id'])
+        router_id = encode_router_id(router['id'])
         host = self.inv.get_by_id(env, host_id)
 
         fetcher = CliFetchHostVservice()
diff --git a/app/discover/events/event_router_delete.py b/app/discover/events/event_router_delete.py
index 65072d6..d0bd645 100644
--- a/app/discover/events/event_router_delete.py
+++ b/app/discover/events/event_router_delete.py
@@ -21,7 +21,6 @@ class EventRouterDelete(EventDeleteBase):
             self.log.error("Publisher_id is not in event values. Aborting router delete")
             return EventResult(result=False, retry=False)
 
-        host_id = values['publisher_id'].replace('network.', '', 1)
         if 'router_id' in payload:
             router_id = payload['router_id']
         elif 'id' in payload:
@@ -33,5 +32,5 @@ class EventRouterDelete(EventDeleteBase):
             self.log.error("Router id is not in payload. Aborting router delete")
             return EventResult(result=False, retry=False)
 
-        router_full_id = encode_router_id(host_id, router_id)
+        router_full_id = encode_router_id(router_id)
         return self.delete_handler(env, router_full_id, "vservice")
diff --git a/app/discover/events/event_router_update.py b/app/discover/events/event_router_update.py
index cfbbf58..b63b224 100644
--- a/app/discover/events/event_router_update.py
+++ b/app/discover/events/event_router_update.py
@@ -26,7 +26,7 @@ class EventRouterUpdate(EventBase):
         host_id = values["publisher_id"].replace("network.", "", 1)
         router_id = payload['id'] if 'id' in payload else router['id']
 
-        router_full_id = encode_router_id(host_id, router_id)
+        router_full_id = encode_router_id(router_id)
         router_doc = self.inv.get_by_id(env, router_full_id)
         if not router_doc:
             self.log.info("Router document not found, aborting router updating")
diff --git a/app/discover/events/event_subnet_add.py b/app/discover/events/event_subnet_add.py
index fcae5fd..4126e0c 100644
--- a/app/discover/events/event_subnet_add.py
+++ b/app/discover/events/event_subnet_add.py
@@ -131,10 +131,10 @@ class EventSubnetAdd(EventBase):
         # Check DHCP enable, if true, scan network.
         if subnet['enable_dhcp'] is True:
             # update network
-            if len(ApiAccess.regions) == 0:
+            if not ApiAccess.regions:
                 fetcher = ApiFetchRegions()
                 fetcher.set_env(env)
-                fetcher.get(None)
+                fetcher.get(project_id)
 
             self.log.info("add new subnet.")
             host_id = notification["publisher_id"].replace("network.", "", 1)
diff --git a/app/discover/events/event_subnet_update.py b/app/discover/events/event_subnet_update.py
index 3529f0d..59b0afb 100644
--- a/app/discover/events/event_subnet_update.py
+++ b/app/discover/events/event_subnet_update.py
@@ -23,6 +23,7 @@ class EventSubnetUpdate(EventBase):
     def handle(self, env, notification):
         # check for network document.
         subnet = notification['payload']['subnet']
+        project_id = notification['_context_project_id']
         project = notification['_context_project_name']
         host_id = notification['publisher_id'].replace('network.', '', 1)
         subnet_id = subnet['id']
@@ -47,10 +48,10 @@ class EventSubnetUpdate(EventBase):
                                                network_document['name'])
 
                 # make sure that self.regions is not empty.
-                if len(ApiAccess.regions) == 0:
+                if not ApiAccess.regions:
                     fetcher = ApiFetchRegions()
                     fetcher.set_env(env)
-                    fetcher.get(None)
+                    fetcher.get(project_id)
 
                 self.log.info("add port binding to DHCP server.")
                 port_id = DbFetchPort(). \
diff --git a/app/discover/events/listeners/default_listener.py b/app/discover/events/listeners/default_listener.py
index 54453a7..273f3e3 100755
--- a/app/discover/events/listeners/default_listener.py
+++ b/app/discover/events/listeners/default_listener.py
@@ -30,17 +30,19 @@ from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager
 from utils.constants import OperationalStatus, EnvironmentFeatures
 from utils.inventory_mgr import InventoryMgr
 from utils.logging.full_logger import FullLogger
+from utils.logging.logger import Logger
 from utils.mongo_access import MongoAccess
-from utils.string_utils import stringify_datetime
 from utils.util import SignalHandler, setup_args
 
 
 class DefaultListener(ListenerBase, ConsumerMixin):
 
     SOURCE_SYSTEM = "OpenStack"
-
     COMMON_METADATA_FILE = "events.json"
 
+    LOG_FILENAME = "default_listener.log"
+    LOG_LEVEL = Logger.INFO
+
     DEFAULTS = {
         "env": "Mirantis-Liberty",
         "mongo_config": "",
@@ -92,7 +94,7 @@ class DefaultListener(ListenerBase, ConsumerMixin):
             return False, None
 
     def process_task(self, body, message):
-        received_timestamp = stringify_datetime(datetime.datetime.now())
+        received_timestamp = datetime.datetime.now()
         processable, event_data = self._extract_event_data(body)
         # If env listener can't process the message
         # or it's not intended for env listener to handle,
@@ -100,7 +102,7 @@ class DefaultListener(ListenerBase, ConsumerMixin):
         if processable and event_data["event_type"] in self.handler.handlers:
             event_result = self.handle_event(event_data["event_type"],
                                              event_data)
-            finished_timestamp = stringify_datetime(datetime.datetime.now())
+            finished_timestamp = datetime.datetime.now()
             self.save_message(message_body=event_data,
                               result=event_result,
                               started=received_timestamp,
@@ -143,8 +145,8 @@ class DefaultListener(ListenerBase, ConsumerMixin):
     # 'Retry' flag specifies if the error is recoverable or not
     # 'Retry' flag is checked only is 'result' is False
     def handle_event(self, event_type: str, notification: dict) -> EventResult:
-        print("Got notification.\nEvent_type: {}\nNotification:\n{}".
-              format(event_type, notification))
+        self.log.error("Got notification.\nEvent_type: {}\nNotification:\n{}".
+                       format(event_type, notification))
         try:
             result = self.handler.handle(event_name=event_type,
                                          notification=notification)
@@ -154,7 +156,7 @@ class DefaultListener(ListenerBase, ConsumerMixin):
             return EventResult(result=False, retry=False)
 
     def save_message(self, message_body: dict, result: EventResult,
-                     started: str, finished: str):
+                     started: datetime, finished: datetime):
         try:
             message = Message(
                 msg_id=message_body.get('message_id'),
diff --git a/app/discover/events/listeners/listener_base.py b/app/discover/events/listeners/listener_base.py
index 7052dc9..4ff4e57 100644
--- a/app/discover/events/listeners/listener_base.py
+++ b/app/discover/events/listeners/listener_base.py
@@ -7,11 +7,25 @@
 # which accompanies this distribution, and is available at                    #
 # http://www.apache.org/licenses/LICENSE-2.0                                  #
 ###############################################################################
+import os
 from abc import ABC, abstractmethod
 
+from utils.logging.console_logger import ConsoleLogger
+from utils.logging.file_logger import FileLogger
+from utils.logging.logger import Logger
+
 
 class ListenerBase(ABC):
 
+    LOG_FILENAME = "listener_base.log"
+    LOG_LEVEL = Logger.WARNING
+
+    def __init__(self):
+        super().__init__()
+        self.log_file = os.path.join(FileLogger.LOG_DIRECTORY,
+                                     self.LOG_FILENAME)
+        self.log = ConsoleLogger(level=Logger.INFO)
+
     @staticmethod
     @abstractmethod
     def listen(self):
diff --git a/app/discover/fetcher_new.py b/app/discover/fetcher_new.py
deleted file mode 100644
index f545554..0000000
--- a/app/discover/fetcher_new.py
+++ /dev/null
@@ -1,30 +0,0 @@
-###############################################################################
-# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
-# and others                                                                  #
-#                                                                             #
-# All rights reserved. This program and the accompanying materials            #
-# are made available under the terms of the Apache License, Version 2.0       #
-# which accompanies this distribution, and is available at                    #
-# http://www.apache.org/licenses/LICENSE-2.0                                  #
-###############################################################################
-from discover.fetcher import Fetcher

-##old stuff

-class FetchHostObjectTypes(Fetcher):

-  

-  

-  def get(self, parent):

-    ret = {

-      "type": "host_object_type",

-      "id": "",

-      "parent": parent,

-      "rows": [

-        {"id": "instances_root", "text": "Instances", "descendants": 1},

-        {"id": "networks_root", "text": "Networks", "descendants": 1},

-        {"id": "pnics_root", "text": "pNICs", "descendants": 1},

-        {"id": "vservices_root", "text": "vServices", "descendants": 1}

-      ]

-    }

-    return ret

-

-    ## old/moved

-

diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py
index 275a3e8..c77b22a 100644
--- a/app/discover/fetchers/cli/cli_access.py
+++ b/app/discover/fetchers/cli/cli_access.py
@@ -12,6 +12,7 @@ import time
 
 from discover.fetcher import Fetcher
 from utils.binary_converter import BinaryConverter
+from utils.cli_dist_translator import CliDistTranslator
 from utils.logging.console_logger import ConsoleLogger
 from utils.ssh_conn import SshConn
 
@@ -41,11 +42,16 @@ class CliAccess(BinaryConverter, Fetcher):
     def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False,
             ssh=None, use_sudo=True):
         ssh_conn = ssh if ssh else SshConn(ssh_to_host)
-        if use_sudo and not cmd.strip().startswith("sudo "):
-            cmd = "sudo " + cmd
-        if not on_gateway and ssh_to_host \
-                and not ssh_conn.is_gateway_host(ssh_to_host):
-            cmd = self.ssh_cmd + ssh_to_host + " " + cmd
+        commands = self.adapt_cmd_to_env(ssh_conn, cmd, use_sudo, on_gateway,
+                                         ssh_to_host)
+        out = ''
+        for c in commands:
+            out += self.run_single_command(c, ssh_conn, ssh_to_host,
+                                           enable_cache=enable_cache)
+        return out
+
+    def run_single_command(self, cmd, ssh_conn, ssh_to_host="",
+                           enable_cache=True):
         curr_time = time.time()
         cmd_path = ssh_to_host + ',' + cmd
         if enable_cache and cmd_path in self.cached_commands:
@@ -73,9 +79,44 @@ class CliAccess(BinaryConverter, Fetcher):
         ret = out.splitlines()
         # if split by whitespace did not work, try splitting by "\\n"
         if len(ret) == 1:
-            ret = [l for l in out.split("\\n") if l != ""]
+            ret = [line for line in out.split("\\n") if line != ""]
         return ret
 
+    MULTI_COMMAND_SEPARATOR = ';;;'
+
+    @staticmethod
+    def handle_split_cmd(cmd: str):
+        if CliAccess.MULTI_COMMAND_SEPARATOR in cmd:
+            return cmd.split(CliAccess.MULTI_COMMAND_SEPARATOR)
+        return [cmd]
+
+    def adapt_cmd_to_env(self, ssh_conn, cmd, use_sudo, on_gateway,
+                         ssh_to_host):
+        cmd = self.adapt_cmd_to_dist(cmd)
+        commands = self.handle_split_cmd(cmd)
+        return [self.adapt_cmd_to_environment(c, use_sudo, on_gateway,
+                                              ssh_to_host, ssh_conn)
+                for c in commands]
+
+    def adapt_cmd_to_environment(self, cmd, use_sudo, on_gateway, ssh_to_host,
+                                 ssh_conn):
+        if self.configuration.environment["distribution"] == "Mercury":
+            use_sudo = False
+        if use_sudo and not cmd.strip().startswith("sudo "):
+            cmd = "sudo " + cmd
+        if not on_gateway and ssh_to_host \
+                and not ssh_conn.is_gateway_host(ssh_to_host):
+            cmd = self.ssh_cmd + ssh_to_host + " " + cmd
+        return cmd
+
+    def adapt_cmd_to_dist(self, cmd):
+        env_conf = self.configuration.get_env_config()
+        dist = env_conf.get('distribution')
+        dist_version = env_conf.get('distribution_version')
+        translator = CliDistTranslator(dist, dist_version=dist_version)
+        cmd = translator.translate(cmd)
+        return cmd
+
     # parse command output columns separated by whitespace
     # since headers can contain whitespace themselves,
     # it is the caller's responsibility to provide the headers
@@ -126,7 +167,8 @@ class CliAccess(BinaryConverter, Fetcher):
             content[headers[i]] = content_parts[i]
         return content
 
-    def merge_ws_spillover_lines(self, lines):
+    @staticmethod
+    def merge_ws_spillover_lines(lines):
         # with WS-separated output, extra output sometimes spills to next line
         # detect that and add to the end of the previous line for our procesing
         pending_line = None
@@ -156,7 +198,8 @@ class CliAccess(BinaryConverter, Fetcher):
     - header_regexp: regexp marking the start of the section
     - end_regexp: regexp marking the end of the section
     """
-    def get_section_lines(self, lines, header_regexp, end_regexp):
+    @staticmethod
+    def get_section_lines(lines, header_regexp, end_regexp):
         if not lines:
             return []
         header_re = re.compile(header_regexp)
@@ -196,7 +239,8 @@ class CliAccess(BinaryConverter, Fetcher):
             if 'name' not in o and 'default' in regexp_tuple:
                 o[name] = regexp_tuple['default']
 
-    def find_matching_regexps(self, o, line, regexps):
+    @staticmethod
+    def find_matching_regexps(o, line, regexps):
         for regexp_tuple in regexps:
             name = regexp_tuple['name']
             regex = regexp_tuple['re']
diff --git a/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py
new file mode 100644
index 0000000..77f149f
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py
@@ -0,0 +1,134 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from collections import deque
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchBondHostPnics(CliAccess):
+    BOND_DIR = '/proc/net/bonding/'
+    SLAVE_INTERFACE_HEADER = 'Slave Interface: '
+
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, parent_id: str):
+        self.log.info('CliFetchBondHostPnics: checking under {}'
+                      .format(parent_id))
+        host_id = parent_id[:parent_id.rindex('-')]
+        cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR)
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host:
+            self.log.error('CliFetchBondHostPnics: host not found: ' + host_id)
+            return []
+        host_types = host['host_type']
+        if 'Network' not in host_types and 'Compute' not in host_types:
+            return []
+        lines = self.run_fetch_lines(cmd, host_id)
+        if lines and 'No such file or directory' in lines[0]:
+            return []  # no bonds so directory does not exist
+        bonds = []
+        for line in lines:
+            bond = self.get_bond_details(host_id, line)
+            if bond:
+                bonds.append(bond)
+        return bonds
+
+    def get_bond_details(self, host_id: str, interface_name: str) -> dict:
+        lines = self.run_fetch_lines('cat {}{}'
+                                     .format(self.BOND_DIR, interface_name),
+                                     host_id)
+        status, mac_address = \
+            self.get_bond_status_and_mac_address(host_id, interface_name)
+        interface_id = '{}-{}'.format(interface_name, mac_address)
+        interface = {
+            'host': host_id,
+            'name': interface_name,
+            'id': interface_id,
+            'local_name': interface_name,
+            'mac_address': mac_address,
+            'Link detected': 'yes' if status == 'up' else 'no',
+            'EtherChannel': True,
+            'EtherChannel Master': '',
+            'members': {}
+        }
+        # keep stack of info objects to support multi-level info
+        info_objects = deque([interface])
+        for line in [line for line in lines if line != '']:
+            if line.startswith(self.SLAVE_INTERFACE_HEADER):
+                name = line[line.index(':')+1:].strip()
+                slave = {
+                    'name': name,
+                    'EtherChannel Master': interface_id
+                }
+                # remove any pending info objects, keep only interface
+                info_objects = deque([interface])
+                info_objects.append(slave)
+                interface['members'][name] = slave
+            elif line.rstrip(':').lower().endswith('info'):
+                # move to lower level info object
+                info_name = line.rstrip(':')
+                upper_info_obj = info_objects[-1]
+                info_obj = {}
+                upper_info_obj[info_name] = info_obj
+                info_objects.append(info_obj)
+            else:
+                self.get_attribute_from_line(info_objects[-1], line)
+        for slave in list(interface['members'].values()):
+            self.set_slave_host_pnic_bond_attributes(host_id, slave,
+                                                     interface_id)
+        return interface
+
+    def get_bond_status_and_mac_address(self, host_id: str, name: str):
+        output = self.run_fetch_lines('ip link show {}'.format(name), host_id)
+        status_line = output[0]
+        status = status_line[status_line.index(' state ') + len(' state '):]
+        status = status[:status.index(' ')]
+        matches = [line.strip() for line in output if 'link/ether' in line]
+        if not matches:
+            self.log.error('Failed to find line with MAC address '
+                           'for bond {} (host: {})'
+                           .format(name, host_id))
+        tokens = matches[0].split()
+        if len(tokens) < 2:
+            self.log.error('Failed to find MAC address in line: {}'
+                           .format(matches[0]))
+        mac_address = tokens[1]
+        return status.lower(), mac_address
+
+    def get_attribute_from_line(self, obj: dict, line: str):
+        if ':' not in line:
+            self.log.error('object {}: failed to find ":" in line: {}'
+                           .format(obj['name'], line))
+            return
+        attr = line[:line.index(':')]
+        value = line[len(attr)+1:]
+        obj[attr.strip()] = value.strip()
+
+    def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id):
+        pnic = self.inv.find_one({
+            'environment': self.get_env(),
+            'host': host,
+            'type': 'host_pnic',
+            'name': slave['name']
+        })
+        if not pnic:
+            self.log.error('unable to find slave pNIC {} under bond {}'
+                           .format(slave_id, interface_id))
+            return
+        mac_address = pnic['mac_address']
+        slave_id = '{}-{}'.format(slave.get('name', ''), mac_address)
+        slave['mac_address'] = mac_address
+        slave['id'] = slave_id
+        pnic['EtherChannel'] = True
+        pnic['EtherChannel Master'] = interface_id
+        self.inv.set(pnic)
diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
index 5df4d3b..4af3ebc 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
@@ -67,21 +67,17 @@ class CliFetchHostPnics(CliAccess):
             tokens = None
             if interface is None:
                 tokens = line.split()
-                name = tokens[0].strip('- :')
-                name = name.strip()
-                if name == interface_name:
-                    line_remainder = line.strip('-')[len(interface_name)+2:]
-                    line_remainder = line_remainder.strip(' :')
-                    id = interface_name
-                    interface = {
-                        "host": host_id,
-                        "name": id,
-                        "local_name": interface_name,
-                        "lines": []
-                    }
-                    self.handle_line(interface, line_remainder)
-                    if '<UP,' in line:
-                        status_up = True
+                line_remainder = line.strip('-')[len(interface_name)+2:]
+                line_remainder = line_remainder.strip(' :')
+                interface = {
+                    "host": host_id,
+                    "name": interface_name,
+                    "local_name": interface_name,
+                    "lines": []
+                }
+                self.handle_line(interface, line_remainder)
+                if '<UP,' in line:
+                    status_up = True
             if status_up is None:
                 if tokens is None:
                     tokens = line.split()
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservice.py b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
index 9f8173f..ae7c656 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_vservice.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
@@ -31,35 +31,37 @@ class CliFetchHostVservice(CliAccess, DbAccess):
     def set_details(self, host_id, r):
         # keep the index without prefix
         id_full = r["local_service_id"].strip()
-        prefix = id_full[1:id_full.index('-')]
-        id_clean = id_full[id_full.index('-') + 1:]
-        r["service_type"] = prefix
-        name = self.get_router_name(r, id_clean) if prefix == "router" \
+        prefix = id_full[:id_full.index('-')]
+        id_clean = id_full[len(prefix)+1:]
+        r["service_type"] = prefix[1:]
+        name = self.get_router_name(r, id_clean) \
+            if r["service_type"] == "router" \
             else self.get_network_name(id_clean)
         r["name"] = prefix + "-" + name
         r["host"] = host_id
-        r["id"] = host_id + "-" + id_full
+        r["id"] = "{}-{}".format(host_id, id_full)
         self.set_agent_type(r)
 
-    def get_network_name(self, id):
+    def get_network_name(self, network_id):
         query = """
                 SELECT name
                 FROM {}.networks
                 WHERE id = %s
                 """.format(self.neutron_db)
-        results = self.get_objects_list_for_id(query, "router", id)
+        results = self.get_objects_list_for_id(query, "router", network_id)
         if not list(results):
-            return id
+            return network_id
         for db_row in results:
             return db_row["name"]
 
-    def get_router_name(self, r, id):
+    def get_router_name(self, r, router_id):
         query = """
                 SELECT *
                 FROM {}.routers
                 WHERE id = %s
                 """.format(self.neutron_db)
-        results = self.get_objects_list_for_id(query, "router", id.strip())
+        results = self.get_objects_list_for_id(query, "router",
+                                               router_id.strip())
         for db_row in results:
             r.update(db_row)
         return r["name"]
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservices.py b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
index 9b62dcb..b9496bc 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_vservices.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
@@ -19,7 +19,7 @@ class CliFetchHostVservices(CliFetchHostVservice):
         if "Network" not in host["host_type"]:
             return []
         services_ids = [l[:l.index(' ')] if ' ' in l else l
-                        for l in self.run_fetch_lines("ip netns", host_id)]
+                        for l in self.run_fetch_lines("ip netns list", host_id)]
         results = [{"local_service_id": s} for s in services_ids if self.type_re.match(s)]
         for r in results:
             self.set_details(host_id, r)
diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
index 4de1840..bb1e7fc 100644
--- a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
+++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
@@ -58,7 +58,7 @@ class CliFetchInstanceVnicsBase(CliAccess):
 
     def set_vnic_properties(self, v, instance):
         v["name"] = self.get_vnic_name(v, instance)
-        v["id"] = v["name"]
+        v["id"] = "{}-{}".format(instance["host"], v["name"])
         v["vnic_type"] = "instance_vnic"
         v["host"] = instance["host"]
         v["instance_id"] = instance["id"]
diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
index d10d99e..239ecd7 100644
--- a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
@@ -42,7 +42,7 @@ class CliFetchVserviceVnics(CliAccess):
             return []
         if "Network" not in host["host_type"]:
             return []
-        lines = self.run_fetch_lines("ip netns", host_id)
+        lines = self.run_fetch_lines("ip netns list", host_id)
         ret = []
         for l in [l for l in lines
                   if l.startswith("qdhcp") or l.startswith("qrouter")]:
@@ -68,7 +68,7 @@ class CliFetchVserviceVnics(CliAccess):
                     current = None
                 else:
                     line_remainder = matches.group(2)
-                    vservice_id = host + "-" + service
+                    master_parent_id = "{}-{}".format(host, service)
                     current = {
                         "id": host + "-" + name,
                         "type": "vnic",
@@ -76,9 +76,9 @@ class CliFetchVserviceVnics(CliAccess):
                         "host": host,
                         "name": name,
                         "master_parent_type": "vservice",
-                        "master_parent_id": vservice_id,
+                        "master_parent_id": master_parent_id,
                         "parent_type": "vnics_folder",
-                        "parent_id": vservice_id + "-vnics",
+                        "parent_id": "{}-vnics".format(master_parent_id),
                         "parent_text": "vNICs",
                         "lines": []
                     }
diff --git a/app/discover/fetchers/db/db_access.py b/app/discover/fetchers/db/db_access.py
index 49fdb5e..64d7372 100644
--- a/app/discover/fetchers/db/db_access.py
+++ b/app/discover/fetchers/db/db_access.py
@@ -7,6 +7,7 @@
 # which accompanies this distribution, and is available at                    #
 # http://www.apache.org/licenses/LICENSE-2.0                                  #
 ###############################################################################
+import functools
 import mysql.connector
 
 from discover.configuration import Configuration
@@ -15,6 +16,24 @@ from discover.scan_error import ScanError
 from utils.string_utils import jsonify
 
 
+def with_cursor(method):
+    @functools.wraps(method)
+    def wrap(self, *args, **kwargs):
+        self.connect_to_db(DbAccess.query_count_per_con >= 25)
+        DbAccess.query_count_per_con += 1
+        cursor = DbAccess.conn.cursor(dictionary=True)
+        try:
+            res = method(self, *args, cursor=cursor, **kwargs)
+            DbAccess.conn.commit()
+            return res
+        except:
+            DbAccess.conn.rollback()
+            raise
+        finally:
+            cursor.close()
+    return wrap
+
+
 class DbAccess(Fetcher):
     conn = None
     query_count_per_con = 0
@@ -47,10 +66,9 @@ class DbAccess(Fetcher):
             return
         DbAccess.query_count_per_con = 0
 
-    @staticmethod
-    def get_neutron_db_name():
+    @with_cursor
+    def get_neutron_db_name(self, cursor=None):
         # check if DB schema 'neutron' exists
-        cursor = DbAccess.conn.cursor(dictionary=True)
         cursor.execute('SHOW DATABASES')
         matches = [row.get('Database', '') for row in cursor
                    if 'neutron' in row.get('Database', '')]
@@ -68,6 +86,8 @@ class DbAccess(Fetcher):
             self.log.info("DbAccess: ****** forcing reconnect, " +
                           "query count: %s ******",
                           DbAccess.query_count_per_con)
+            DbAccess.conn.commit()
+            DbAccess.conn.close()
             DbAccess.conn = None
         self.conf = self.config.get("mysql")
         cnf = self.conf
@@ -76,16 +96,15 @@ class DbAccess(Fetcher):
                         cnf["user"], cnf["pwd"],
                         cnf["schema"])
 
-    def get_objects_list_for_id(self, query, object_type, id):
-        self.connect_to_db(DbAccess.query_count_per_con >= 25)
-        DbAccess.query_count_per_con += 1
+    @with_cursor
+    def get_objects_list_for_id(self, query, object_type, object_id,
+                                cursor=None):
         self.log.debug("query count: %s, running query:\n%s\n",
                        str(DbAccess.query_count_per_con), query)
 
-        cursor = DbAccess.conn.cursor(dictionary=True)
         try:
-            if id:
-                cursor.execute(query, [str(id)])
+            if object_id:
+                cursor.execute(query, [str(object_id)])
             else:
                 cursor.execute(query)
         except (AttributeError, mysql.connector.errors.OperationalError) as e:
@@ -93,13 +112,13 @@ class DbAccess(Fetcher):
             self.connect_to_db(True)
             # try again to run the query
             cursor = DbAccess.conn.cursor(dictionary=True)
-            if id:
-                cursor.execute(query, [str(id)])
+            if object_id:
+                cursor.execute(query, [str(object_id)])
             else:
                 cursor.execute(query)
 
         rows = []
-        for row in cursor:
+        for row in cursor.fetchall():
             rows.append(row)
         return rows
 
diff --git a/app/discover/fetchers/db/db_fetch_host_network_agents.py b/app/discover/fetchers/db/db_fetch_host_network_agents.py
index c323573..7d415f2 100644
--- a/app/discover/fetchers/db/db_fetch_host_network_agents.py
+++ b/app/discover/fetchers/db/db_fetch_host_network_agents.py
@@ -27,9 +27,8 @@ class DbFetchHostNetworkAgents(DbAccess):
         host_id = id[:-1 * len("-network_agents")]
         results = self.get_objects_list_for_id(query, "network_agent", host_id)
         mechanism_drivers = self.env_config['mechanism_drivers']
-        id_prefix = mechanism_drivers[0] if mechanism_drivers else 'network_agent'
         for o in results:
             o["configurations"] = json.loads(o["configurations"])
             o["name"] = o["binary"]
-            o['id'] = id_prefix + '-' + o['id']
+            o['id'] = o['name'] + '-' + o['id']
         return results
diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py
index 3e3f4e1..f7eb8bd 100644
--- a/app/discover/fetchers/db/db_fetch_oteps.py
+++ b/app/discover/fetchers/db/db_fetch_oteps.py
@@ -35,7 +35,9 @@ class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton):
         table_name = "{}.ml2_{}_endpoints".format(self.neutron_db, tunnel_type)
         env_config = self.config.get_env_config()
         distribution = env_config["distribution"]
-        if distribution == "Canonical-icehouse":
+        distribution_version = env_config["distribution_version"]
+        dist_ver = "{}-{}".format(distribution, distribution_version)
+        if dist_ver == "Canonical-icehouse":
             # for Icehouse, we only get IP address from the DB, so take the
             # host IP address and from the host data in Mongo
             host = self.inv.get_by_id(self.get_env(), host_id)
diff --git a/app/discover/fetchers/db/db_fetch_vedges_ovs.py b/app/discover/fetchers/db/db_fetch_vedges_ovs.py
index 838ccb9..f516d10 100644
--- a/app/discover/fetchers/db/db_fetch_vedges_ovs.py
+++ b/app/discover/fetchers/db/db_fetch_vedges_ovs.py
@@ -24,8 +24,8 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton):
         self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$")
         self.port_line_header_prefix = " " * 8 + "Port "
 
-    def get(self, id):
-        host_id = id[:id.rindex('-')]
+    def get(self, parent_id):
+        host_id = parent_id[:parent_id.rindex('-')]
         results = self.get_objects_list_for_id(
             """
               SELECT *
@@ -66,11 +66,11 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton):
             if not port_matches:
                 continue
             port = {}
-            id = port_matches.group(1)
+            port_id = port_matches.group(1)
             name = port_matches.group(2)
             is_internal = port_matches.group(3) == " (internal)"
             port["internal"] = is_internal
-            port["id"] = id
+            port["id"] = port_id
             port["name"] = name
             ports[name] = port
         return ports
@@ -106,7 +106,7 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton):
         if "tunneling_ip" not in doc["configurations"]:
             return {}
         if not doc["configurations"]["tunneling_ip"]:
-            self.get_bridge_pnic(doc)
+            self.get_pnics(doc)
             return {}
 
         # read the 'br-tun' interface ports
@@ -148,31 +148,48 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton):
             tunnel_ports[port["name"]] = port
         return tunnel_ports
 
-    def get_bridge_pnic(self, doc):
-        conf = doc["configurations"]
-        if "bridge_mappings" not in conf or not conf["bridge_mappings"]:
-            return
-        for v in conf["bridge_mappings"].values(): br = v
-        ifaces_list_lines = self.run_fetch_lines("ovs-vsctl list-ifaces " + br,
-                                                 doc["host"])
-        br_pnic_postfix = br + "--br-"
-        interface = ""
+    def get_pnics(self, vedge) -> dict:
+        bridges = vedge["configurations"].get("bridge_mappings", {})
+        pnics = {}
+        for bridge in bridges.values():
+            self.get_bridge_pnic(pnics, vedge, bridge)
+        return pnics
+
+    MIRANTIS_DIST = "Mirantis"
+
+    def get_bridge_pnic(self, pnics: dict, vedge: dict, bridge: dict):
+        cmd = "ovs-vsctl list-ifaces {}".format(bridge)
+        ifaces_list_lines = self.run_fetch_lines(cmd, vedge["host"])
+        env_config = self.configuration.get_env_config()
+        distribution = env_config.get("distribution")
+        dist_version = env_config.get("distribution_version")
+        use_br_postfix = distribution == self.MIRANTIS_DIST and \
+            dist_version in ["6.0", "7.0", "8.0"]
         for l in ifaces_list_lines:
-            if l.startswith(br_pnic_postfix):
-                interface = l[len(br_pnic_postfix):]
-                break
-        if not interface:
-            return
-        doc["pnic"] = interface
+            if use_br_postfix:
+                br_pnic_postfix = "{}--br-".format(bridge)
+                interface = l[len(br_pnic_postfix):] \
+                    if l.startswith(br_pnic_postfix) \
+                    else ""
+            else:
+                interface = l
+            if interface:
+                pnic = self.find_pnic_for_interface(vedge, interface)
+                if pnic:
+                    pnics[pnic["name"]] = pnic
+
+    def find_pnic_for_interface(self, vedge, interface):
         # add port ID to pNIC
         pnic = self.inv.find_items({
             "environment": self.get_env(),
             "type": "host_pnic",
-            "host": doc["host"],
+            "host": vedge["host"],
             "name": interface
         }, get_single=True)
         if not pnic:
             return
-        port = doc["ports"][interface]
-        pnic["port_id"] = port["id"]
+        vedge["pnic"] = interface
+        port = vedge["ports"].get(interface, {})
+        pnic["port_id"] = port.get("id", "")
         self.inv.set(pnic)
+        return pnic
diff --git a/app/discover/link_finders/__init__.py b/app/discover/link_finders/__init__.py
index e69de29..1e85a2a 100644
--- a/app/discover/link_finders/__init__.py
+++ b/app/discover/link_finders/__init__.py
@@ -0,0 +1,10 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+
diff --git a/app/discover/link_finders/find_links_for_pnics.py b/app/discover/link_finders/find_links_for_pnics.py
index 1f02426..94eba7b 100644
--- a/app/discover/link_finders/find_links_for_pnics.py
+++ b/app/discover/link_finders/find_links_for_pnics.py
@@ -41,6 +41,18 @@ class FindLinksForPnics(FindLinks):
 
     def add_pnic_network_links(self, pnic):
         host = pnic["host"]
+        if self.configuration.get_env_config()['type_drivers'] == "vlan":
+            # take this pnic only if we can find matching vedge-pnic links
+            matches = self.inv.find({
+                "environment": self.get_env(),
+                "link_type": "vedge-host_pnic",
+                "host": host,
+                "target_id": pnic["id"]},
+                projection={"_id": 1},
+                collection="links",
+                get_single=True)
+            if not matches:
+                return
         # find ports for that host, and fetch just the network ID
         ports = self.inv.find_items({
             "environment": self.get_env(),
diff --git a/app/discover/link_finders/find_links_for_vconnectors.py b/app/discover/link_finders/find_links_for_vconnectors.py
index edb351a..0703cd8 100644
--- a/app/discover/link_finders/find_links_for_vconnectors.py
+++ b/app/discover/link_finders/find_links_for_vconnectors.py
@@ -31,7 +31,8 @@ class FindLinksForVconnectors(FindLinks):
         is_ovs = mechanism_drivers and mechanism_drivers[0] == 'OVS'
         if is_ovs:
             # interface ID for OVS
-            vnic = self.inv.get_by_id(self.get_env(), interface_name)
+            vnic_id = "{}-{}".format(vconnector["host"], interface_name)
+            vnic = self.inv.get_by_id(self.get_env(), vnic_id)
         else:
             # interface ID for VPP - match interface MAC address to vNIC MAC
             interface = vconnector['interfaces'][interface_name]
diff --git a/app/discover/link_finders/find_links_for_vservice_vnics.py b/app/discover/link_finders/find_links_for_vservice_vnics.py
index ca9bc4a..f975c92 100644
--- a/app/discover/link_finders/find_links_for_vservice_vnics.py
+++ b/app/discover/link_finders/find_links_for_vservice_vnics.py
@@ -33,11 +33,6 @@ class FindLinksForVserviceVnics(FindLinks):
         host = self.inv.get_by_id(self.get_env(), v["host"])
         if "Network" not in host["host_type"]:
             return
-        if "network" not in v:
-            return
-        network = self.inv.get_by_id(self.get_env(), v["network"])
-        if network == []:
-            return
         vservice_id = v["parent_id"]
         vservice_id = vservice_id[:vservice_id.rindex('-')]
         vservice = self.inv.get_by_id(self.get_env(), vservice_id)
@@ -46,7 +41,14 @@ class FindLinksForVserviceVnics(FindLinks):
         target = v["_id"]
         target_id = v["id"]
         link_type = "vservice-vnic"
-        link_name = network["name"]
+        extra_attributes = None
+        if "network" in v:
+            network = self.inv.get_by_id(self.get_env(), v["network"])
+            link_name = network["name"]
+            extra_attributes = {'network': v['network']}
+        else:
+            link_name = "{}-{}".format(vservice["object_name"],
+                                       v["object_name"])
         state = "up"  # TBD
         link_weight = 0  # TBD
         self.create_link(self.get_env(),
@@ -54,4 +56,4 @@ class FindLinksForVserviceVnics(FindLinks):
                          target, target_id,
                          link_type, link_name, state, link_weight,
                          host=v["host"],
-                         extra_attributes={'network': v['network']})
+                         extra_attributes=extra_attributes)
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
index d1323bd..1fbcc68 100644
--- a/app/discover/scanner.py
+++ b/app/discover/scanner.py
@@ -240,7 +240,7 @@ class Scanner(Fetcher):
         run_app_path = conf.get('run_app_path', '')
         if not run_app_path:
             run_app_path = conf.get('app_path', '/etc/calipso')
-            return run_app_path
+        return run_app_path
 
     def load_scanners_metadata(self):
         parser = ScanMetadataParser(self.inv)