US3541 merge various fixes to OPNFV branch

timestamp of last commit tt was merged: 26-Jan-2018 16:25.

Change-Id: I7b0bf7885d7d0badb81c794a52c480b905d78459
Signed-off-by: yayogev <yaronyogev@gmail.com>

27 files changed, 286 insertions, 168 deletions

diff --git a/app/discover/clique_finder.py b/app/discover/clique_finder.py
index 4d68eb4..4e04e7e 100644
--- a/app/discover/clique_finder.py
+++ b/app/discover/clique_finder.py
@@ -42,43 +42,74 @@ class CliqueFinder(Fetcher):
         return self.links.find({'target': db_id})
 
     def find_cliques(self):
-        self.log.info("scanning for cliques")
+        self.log.info("Scanning for cliques")
         clique_types = self.get_clique_types().values()
         for clique_type in clique_types:
             self.find_cliques_for_type(clique_type)
-        self.log.info("finished scanning for cliques")
+        self.log.info("Finished scanning for cliques")
 
     # Calculate priority score for clique type per environment and configuration
-    def _get_priority_score(self, clique_type):
+    def get_priority_score(self, clique_type):
         # environment-specific clique type takes precedence
-        if self.env == clique_type['environment']:
-            return 16
-        if (self.env_config['distribution'] == clique_type.get('distribution')
-            and
-            self.env_config['distribution_version'] ==
-                clique_type.get('distribution_version')):
-            return 8
-        if clique_type.get('mechanism_drivers') \
-                in self.env_config['mechanism_drivers']:
-            return 4
-        if self.env_config['type_drivers'] == clique_type.get('type_drivers'):
-            return 2
-        if clique_type.get('environment', '') == 'ANY':
-            # environment=ANY serves as fallback option, but it's not mandatory
-            return 1
-        else:
+        env = clique_type.get('environment')
+        config = self.env_config
+        # ECT - Clique Type with Environment name
+        if env:
+            if self.env == env:
+                return 2**6
+            if env == 'ANY':
+                # environment=ANY serves as fallback option
+                return 2**0
             return 0
+        # NECT - Clique Type without Environment name
+        else:
+            env_type = clique_type.get('environment_type')
+            # TODO: remove backward compatibility ('if not env_type' check)
+            if env_type and env_type != config.get('environment_type'):
+                return 0
+
+            score = 0
+
+            distribution = clique_type.get('distribution')
+            if distribution:
+                if config['distribution'] != distribution:
+                    return 0
+
+                score += 2**5
+
+                dv = clique_type.get('distribution_version')
+                if dv:
+                    if dv != config['distribution_version']:
+                        return 0
+                    score += 2**4
+
+            mechanism_drivers = clique_type.get('mechanism_drivers')
+            if mechanism_drivers:
+                if mechanism_drivers not in config['mechanism_drivers']:
+                    return 0
+                score += 2**3
+
+            type_drivers = clique_type.get('type_drivers')
+            if type_drivers:
+                if type_drivers != config['type_drivers']:
+                    return 0
+                score += 2**2
+
+            # If no configuration is specified, this clique type
+            # is a fallback for its environment type
+            return max(score, 2**1)
 
     # Get clique type with max priority
     # for given focal point type
     def _get_clique_type(self, clique_types):
-        scored_clique_types = [{'score': self._get_priority_score(clique_type),
+        scored_clique_types = [{'score': self.get_priority_score(clique_type),
                                 'clique_type': clique_type}
                                for clique_type in clique_types]
         max_score = max(scored_clique_types, key=lambda t: t['score'])
         if max_score['score'] == 0:
-            self.log.warn('No matching clique types for focal point type: {}'
-                          .format(clique_types[0].get('focal_point_type')))
+            self.log.warn('No matching clique types '
+                          'for focal point type: {fp_type}'
+                          .format(fp_type=clique_types[0].get('focal_point_type')))
             return None
         return max_score.get('clique_type')
 
@@ -143,8 +174,9 @@ class CliqueFinder(Fetcher):
             clique["constraints"][c] = val
         allow_implicit = clique_type.get('use_implicit_links', False)
         for link_type in clique_type["link_types"]:
-            self.check_link_type(clique, link_type, nodes_of_type,
-                                 allow_implicit=allow_implicit)
+            if not self.check_link_type(clique, link_type, nodes_of_type,
+                                        allow_implicit=allow_implicit):
+                break
 
         # after adding the links to the clique, create/update the clique
         if not clique["links"]:
@@ -197,7 +229,7 @@ class CliqueFinder(Fetcher):
         return CliqueFinder.link_type_reversed.get(link_type)
 
     def check_link_type(self, clique, link_type, nodes_of_type,
-                        allow_implicit=False):
+                        allow_implicit=False) -> bool:
         # check if it's backwards
         link_type_reversed = self.get_link_type_reversed(link_type)
         # handle case of links like T<-->T
@@ -213,15 +245,16 @@ class CliqueFinder(Fetcher):
             matches = self.links.find_one(link_search_condition)
             use_reversed = True if matches else False
         if self_linked or not use_reversed:
-            self.check_link_type_forward(clique, link_type, nodes_of_type,
-                                         allow_implicit=allow_implicit)
+            return self.check_link_type_forward(clique, link_type,
+                                                nodes_of_type,
+                                                allow_implicit=allow_implicit)
         if self_linked or use_reversed:
-            self.check_link_type_back(clique, link_type, nodes_of_type,
-                                      allow_implicit=allow_implicit)
+            return self.check_link_type_back(clique, link_type, nodes_of_type,
+                                             allow_implicit=allow_implicit)
 
     def check_link_type_for_direction(self, clique, link_type, nodes_of_type,
                                       is_reversed=False,
-                                      allow_implicit=False):
+                                      allow_implicit=False) -> bool:
         if is_reversed:
             link_type = self.get_link_type_reversed(link_type)
         from_type = link_type[:link_type.index("-")]
@@ -230,7 +263,7 @@ class CliqueFinder(Fetcher):
         other_side = 'target' if not is_reversed else 'source'
         match_type = to_type if is_reversed else from_type
         if match_type not in nodes_of_type.keys():
-            return
+            return False
         other_side_type = to_type if not is_reversed else from_type
         nodes_to_add = set()
         for match_point in nodes_of_type[match_type]:
@@ -245,6 +278,7 @@ class CliqueFinder(Fetcher):
             nodes_of_type[other_side_type] = set()
         nodes_of_type[other_side_type] = \
             nodes_of_type[other_side_type] | nodes_to_add
+        return len(nodes_to_add) > 0
 
     def find_matches_for_point(self, match_point, clique, link_type,
                                side_to_match, other_side,
@@ -271,13 +305,15 @@ class CliqueFinder(Fetcher):
         return nodes_to_add
 
     def check_link_type_forward(self, clique, link_type, nodes_of_type,
-                                allow_implicit=False):
-        self.check_link_type_for_direction(clique, link_type, nodes_of_type,
-                                           is_reversed=False,
-                                           allow_implicit=allow_implicit)
+                                allow_implicit=False) -> bool:
+        return self.check_link_type_for_direction(clique, link_type,
+                                                  nodes_of_type,
+                                                  is_reversed=False,
+                                                  allow_implicit=allow_implicit)
 
     def check_link_type_back(self, clique, link_type, nodes_of_type,
-                             allow_implicit=False):
-        self.check_link_type_for_direction(clique, link_type, nodes_of_type,
-                                           is_reversed=True,
-                                           allow_implicit=allow_implicit)
+                             allow_implicit=False) -> bool:
+        return self.check_link_type_for_direction(clique, link_type,
+                                                  nodes_of_type,
+                                                  is_reversed=True,
+                                                  allow_implicit=allow_implicit)
diff --git a/app/discover/configuration.py b/app/discover/configuration.py
index c7bc0c0..9ec8f96 100644
--- a/app/discover/configuration.py
+++ b/app/discover/configuration.py
@@ -47,6 +47,10 @@ class Configuration(metaclass=Singleton):
     def get_env_name(self):
         return self.env_name
 
+    def get_env_type(self):
+        return 'OpenStack' if 'environment_type' not in self.environment \
+            else self.environment['environment_type']
+
     def update_env(self, values):
         self.collection.update_one({"name": self.env_name},
                                    {'$set': MongoAccess.encode_mongo_keys(values)})
diff --git a/app/discover/events/event_base.py b/app/discover/events/event_base.py
index 6b3b290..4b466e1 100644
--- a/app/discover/events/event_base.py
+++ b/app/discover/events/event_base.py
@@ -11,6 +11,7 @@ from abc import abstractmethod, ABC
 
 from discover.fetcher import Fetcher
 from utils.inventory_mgr import InventoryMgr
+from utils.origins import ScanOrigin, ScanOrigins
 
 
 class EventResult:
@@ -23,6 +24,8 @@ class EventResult:
         self.message = message
         self.related_object = related_object
         self.display_context = display_context
+        self.origin = ScanOrigin(origin_id=None,
+                                 origin_type=ScanOrigins.EVENT)
 
 
 class EventBase(Fetcher, ABC):
diff --git a/app/discover/events/event_instance_add.py b/app/discover/events/event_instance_add.py
index 4dd2b20..a8717a5 100644
--- a/app/discover/events/event_instance_add.py
+++ b/app/discover/events/event_instance_add.py
@@ -25,7 +25,7 @@ class EventInstanceAdd(EventBase):
 
         # scan instance
         scanner = Scanner()
-        scanner.set_env(env)
+        scanner.setup(env=env, origin=self.origin)
         scanner.scan("ScanInstancesRoot", instances_root,
                      limit_to_child_id=instance_id,
                      limit_to_child_type='instance')
diff --git a/app/discover/events/event_interface_add.py b/app/discover/events/event_interface_add.py
index e54bedb..f0ba569 100644
--- a/app/discover/events/event_interface_add.py
+++ b/app/discover/events/event_interface_add.py
@@ -30,7 +30,7 @@ class EventInterfaceAdd(EventBase):
 
     def add_gateway_port(self, env, project, network_name, router_doc, host_id):
         fetcher = CliFetchHostVservice()
-        fetcher.set_env(env)
+        fetcher.setup(env=env, origin=self.origin)
         router_id = router_doc['id']
         router = fetcher.get_vservice(host_id, router_id)
         device_id = decode_router_id(router_id)
@@ -101,7 +101,7 @@ class EventInterfaceAdd(EventBase):
         # add router-interface port document.
         if not ApiAccess.regions:
             fetcher = ApiFetchRegions()
-            fetcher.set_env(env)
+            fetcher.setup(env=env, origin=self.origin)
             fetcher.get(project_id)
         port_doc = EventSubnetAdd().add_port_document(env, port_id,
                                                       network_name=network_name)
@@ -134,7 +134,7 @@ class EventInterfaceAdd(EventBase):
         # update vservice-vnic, vnic-network,
         FindLinksForVserviceVnics().add_links(search={"parent_id": router_id})
         scanner = Scanner()
-        scanner.set_env(env)
+        scanner.setup(env=env, origin=self.origin)
 
         scanner.scan_cliques()
         self.log.info("Finished router-interface added.")
diff --git a/app/discover/events/event_port_add.py b/app/discover/events/event_port_add.py
index 9220015..e03db34 100644
--- a/app/discover/events/event_port_add.py
+++ b/app/discover/events/event_port_add.py
@@ -168,7 +168,7 @@ class EventPortAdd(EventBase):
                     "router": ('Gateways', router_name)}
 
         fetcher = CliFetchVserviceVnics()
-        fetcher.set_env(env)
+        fetcher.setup(env=env, origin=self.origin)
         namespace = 'q{}-{}'.format(object_type, object_id)
         vnic_documents = fetcher.handle_service(host['id'], namespace, enable_cache=False)
         if not vnic_documents:
@@ -258,7 +258,7 @@ class EventPortAdd(EventBase):
 
             # update instance
             instance_fetcher = ApiFetchHostInstances()
-            instance_fetcher.set_env(env)
+            instance_fetcher.setup(env=env, origin=self.origin)
             instance_docs = instance_fetcher.get(host_id + '-')
             instance = next(filter(lambda i: i['id'] == instance_id, instance_docs), None)
 
@@ -278,7 +278,7 @@ class EventPortAdd(EventBase):
                 # set ovs as default type.
                 vnic_fetcher = CliFetchInstanceVnics()
 
-            vnic_fetcher.set_env(env)
+            vnic_fetcher.setup(env=env, origin=self.origin)
             vnic_docs = vnic_fetcher.get(instance_id + '-')
             vnic = next(filter(lambda vnic: vnic['mac_address'] == mac_address, vnic_docs), None)
 
@@ -298,7 +298,7 @@ class EventPortAdd(EventBase):
             for fetcher in fetchers_implementing_add_links:
                 fetcher.add_links()
             scanner = Scanner()
-            scanner.set_env(env)
+            scanner.setup(env=env, origin=self.origin)
             scanner.scan_cliques()
 
         port_document = self.inv.get_by_id(env, port['id'])
diff --git a/app/discover/events/event_port_delete.py b/app/discover/events/event_port_delete.py
index 1e55870..937d8df 100644
--- a/app/discover/events/event_port_delete.py
+++ b/app/discover/events/event_port_delete.py
@@ -61,7 +61,7 @@ class EventPortDelete(EventDeleteBase):
             # update instance mac address.
             if port_doc['mac_address'] == instance_doc['mac_address']:
                 instance_fetcher = ApiFetchHostInstances()
-                instance_fetcher.set_env(env)
+                instance_fetcher.setup(env=env, origin=self.origin)
                 host_id = port_doc['binding:host_id']
                 instance_id = port_doc['device_id']
                 instance_docs = instance_fetcher.get(host_id + '-')
diff --git a/app/discover/events/event_router_add.py b/app/discover/events/event_router_add.py
index 1fb2244..0f8bc05 100644
--- a/app/discover/events/event_router_add.py
+++ b/app/discover/events/event_router_add.py
@@ -100,7 +100,7 @@ class EventRouterAdd(EventBase):
         host = self.inv.get_by_id(env, host_id)
 
         fetcher = CliFetchHostVservice()
-        fetcher.set_env(env)
+        fetcher.setup(env=env, origin=self.origin)
         router_doc = fetcher.get_vservice(host_id, router_id)
         gateway_info = router['external_gateway_info']
 
@@ -114,7 +114,7 @@ class EventRouterAdd(EventBase):
         # scan links and cliques
         FindLinksForVserviceVnics().add_links(search={"parent_id": router_id})
         scanner = Scanner()
-        scanner.set_env(env)
+        scanner.setup(env=env, origin=self.origin)
         scanner.scan_cliques()
         self.log.info("Finished router added.")
 
diff --git a/app/discover/events/event_router_update.py b/app/discover/events/event_router_update.py
index b63b224..f20f07e 100644
--- a/app/discover/events/event_router_update.py
+++ b/app/discover/events/event_router_update.py
@@ -60,7 +60,7 @@ class EventRouterUpdate(EventBase):
 
             # add gw_port_id info and port document.
             fetcher = CliFetchHostVservice()
-            fetcher.set_env(env)
+            fetcher.setup(env=env, origin=self.origin)
             router_vservice = fetcher.get_vservice(host_id, router_full_id)
             if router_vservice.get('gw_port_id'):
                 router_doc['gw_port_id'] = router_vservice['gw_port_id']
@@ -74,7 +74,7 @@ class EventRouterUpdate(EventBase):
 
         # update the cliques.
         scanner = Scanner()
-        scanner.set_env(env)
+        scanner.setup(env=env, origin=self.origin)
         scanner.scan_cliques()
         self.log.info("Finished router update.")
         return EventResult(result=True,
diff --git a/app/discover/events/event_subnet_add.py b/app/discover/events/event_subnet_add.py
index 4126e0c..0a91803 100644
--- a/app/discover/events/event_subnet_add.py
+++ b/app/discover/events/event_subnet_add.py
@@ -29,7 +29,7 @@ class EventSubnetAdd(EventBase):
         # document does not has project attribute. In this case, network_name should not be provided.
 
         fetcher = ApiFetchPort()
-        fetcher.set_env(env)
+        fetcher.setup(env=env, origin=self.origin)
         ports = fetcher.get(port_id)
 
         if ports:
@@ -133,7 +133,7 @@ class EventSubnetAdd(EventBase):
             # update network
             if not ApiAccess.regions:
                 fetcher = ApiFetchRegions()
-                fetcher.set_env(env)
+                fetcher.setup(env=env, origin=self.origin)
                 fetcher.get(project_id)
 
             self.log.info("add new subnet.")
@@ -146,7 +146,7 @@ class EventSubnetAdd(EventBase):
         FindLinksForVserviceVnics().add_links(search={"parent_id": "qdhcp-%s-vnics" % network_id})
 
         scanner = Scanner()
-        scanner.set_env(env)
+        scanner.setup(env=env, origin=self.origin)
         scanner.scan_cliques()
         self.log.info("Finished subnet added.")
         return EventResult(result=True,
diff --git a/app/discover/events/event_subnet_update.py b/app/discover/events/event_subnet_update.py
index 59b0afb..2c58e70 100644
--- a/app/discover/events/event_subnet_update.py
+++ b/app/discover/events/event_subnet_update.py
@@ -50,7 +50,7 @@ class EventSubnetUpdate(EventBase):
                 # make sure that self.regions is not empty.
                 if not ApiAccess.regions:
                     fetcher = ApiFetchRegions()
-                    fetcher.set_env(env)
+                    fetcher.setup(env=env, origin=self.origin)
                     fetcher.get(project_id)
 
                 self.log.info("add port binding to DHCP server.")
@@ -69,12 +69,12 @@ class EventSubnetUpdate(EventBase):
                     # add link for vservice - vnic
                     FindLinksForVserviceVnics().add_links(search={"id": "qdhcp-%s" % network_id})
                     scanner = Scanner()
-                    scanner.set_env(env)
+                    scanner.setup(env=env, origin=self.origin)
                     scanner.scan_cliques()
                     FindLinksForVserviceVnics(). \
                         add_links(search={"id": "qdhcp-%s" % network_id})
                     scanner = Scanner()
-                    scanner.set_env(env)
+                    scanner.setup(env=env, origin=self.origin)
                     scanner.scan_cliques()
 
             if subnet['enable_dhcp'] is False and subnets[key]['enable_dhcp']:
diff --git a/app/discover/fetcher.py b/app/discover/fetcher.py
index 8d7fdbb..707cd60 100644
--- a/app/discover/fetcher.py
+++ b/app/discover/fetcher.py
@@ -8,16 +8,21 @@
 # http://www.apache.org/licenses/LICENSE-2.0                                  #
 ###############################################################################
 from discover.configuration import Configuration
+from utils.origins import Origin
 from utils.logging.full_logger import FullLogger
 
 
 class Fetcher:
 
+    ENV_TYPE_KUBERNETES = 'Kubernetes'
+    ENV_TYPE_OPENSTACK = 'OpenStack'
+
     def __init__(self):
         super().__init__()
         self.env = None
         self.log = FullLogger()
         self.configuration = None
+        self.origin = None
 
     @staticmethod
     def escape(string):
@@ -25,11 +30,55 @@ class Fetcher:
 
     def set_env(self, env):
         self.env = env
-        self.log.set_env(env)
+        self.log.setup(env=env)
         self.configuration = Configuration()
 
+    def setup(self, env, origin: Origin = None):
+        self.set_env(env=env)
+        if origin:
+            self.origin = origin
+            self.log.setup(origin=origin)
+
     def get_env(self):
         return self.env
 
     def get(self, object_id):
         return None
+
+    def set_folder_parent(self,
+                          o: dict,
+                          object_type: str =None,
+                          master_parent_type: str =None,
+                          master_parent_id: str =None,
+                          parent_objects_name=None,
+                          parent_type: str =None,
+                          parent_id: str =None,
+                          parent_text: str =None):
+        if object_type:
+            o['type'] = object_type
+            if not parent_objects_name:
+                parent_objects_name = '{}s'.format(object_type)
+        if not master_parent_type:
+            self.log.error('set_folder_parent: must specify: '
+                           'master_parent_type, master_parent_id, '
+                           'parent_type', 'parent_id')
+            return
+        if not parent_objects_name and not parent_type:
+            self.log.error('set_folder_parent: must specify: '
+                           'either parent_objects_name (e.g. "vedges") '
+                           'or parent_type and parent_id')
+            return
+        if parent_objects_name and not parent_type:
+            parent_type = '{}_folder'.format(parent_objects_name)
+        if parent_objects_name and not parent_id:
+            parent_id = '{}-{}'.format(master_parent_id, parent_objects_name)
+        o.update({
+            'master_parent_type': master_parent_type,
+            'master_parent_id': master_parent_id,
+            'parent_type': parent_type,
+            'parent_id': parent_id
+        })
+        if parent_text:
+            o['parent_text'] = parent_text
+        elif parent_objects_name:
+            o['parent_text'] = parent_objects_name.capitalize()
diff --git a/app/discover/fetchers/api/api_fetch_availability_zones.py b/app/discover/fetchers/api/api_fetch_availability_zones.py
index 196893b..ad9550e 100644
--- a/app/discover/fetchers/api/api_fetch_availability_zones.py
+++ b/app/discover/fetchers/api/api_fetch_availability_zones.py
@@ -28,7 +28,7 @@ class ApiFetchAvailabilityZones(ApiAccess):
         # because the later does not inclde the "internal" zone in the results
         endpoint = self.get_region_url_nover(region, "nova")
         req_url = endpoint + "/v2/" + token["tenant"]["id"] + \
-                  "/os-availability-zone/detail"
+            "/os-availability-zone/detail"
         headers = {
             "X-Auth-Project-Id": project,
             "X-Auth-Token": token["id"]
@@ -45,11 +45,10 @@ class ApiFetchAvailabilityZones(ApiAccess):
         for doc in azs:
             doc["id"] = doc["zoneName"]
             doc["name"] = doc.pop("zoneName")
-            doc["master_parent_type"] = "region"
-            doc["master_parent_id"] = region
-            doc["parent_type"] = "availability_zones_folder"
-            doc["parent_id"] = region + "-availability_zones"
-            doc["parent_text"] = "Availability Zones"
+            self.set_folder_parent(doc, object_type="availability_zone",
+                                   master_parent_type="region",
+                                   master_parent_id=region,
+                                   parent_text="Availability Zones")
             doc["available"] = doc["zoneState"]["available"]
             doc.pop("zoneState")
             ret.append(doc)
diff --git a/app/discover/fetchers/api/api_fetch_network.py b/app/discover/fetchers/api/api_fetch_network.py
index 889b8a5..b253773 100644
--- a/app/discover/fetchers/api/api_fetch_network.py
+++ b/app/discover/fetchers/api/api_fetch_network.py
@@ -23,7 +23,8 @@ class ApiFetchNetwork(ApiAccess):
             return []
         ret = []
         for region in self.regions:
-            # TODO: refactor legacy code (Unresolved reference - self.get_for_region)
+            # TODO: refactor legacy code
+            # (Unresolved reference - self.get_for_region)
             ret.extend(self.get_for_region(region, token, project_id))
         return ret
 
@@ -37,7 +38,7 @@ class ApiFetchNetwork(ApiAccess):
             "X-Auth-Token": token["id"]
         }
         response = self.get_url(req_url, headers)
-        if not "network" in response:
+        if "network" not in response:
             return []
         network = response["network"]
         subnets = network['subnets']
@@ -60,13 +61,12 @@ class ApiFetchNetwork(ApiAccess):
         network["cidrs"] = cidrs
         network["subnet_ids"] = subnet_ids
 
-        network["master_parent_type"] = "project"
-        network["master_parent_id"] = network["tenant_id"]
-        network["parent_type"] = "networks_folder"
-        network["parent_id"] = network["tenant_id"] + "-networks"
-        network["parent_text"] = "Networks"
-        # set the 'network' attribute for network objects to the name of network,
-        # to allow setting constraint on network when creating network clique
+        self.set_folder_parent(network, object_type="network",
+                               master_parent_type="project",
+                               master_parent_id=network["tenant_id"])
+        # set the 'network' attribute for network objects to the name of
+        # network, to allow setting constraint on network when creating
+        # network clique
         network['network'] = network["id"]
         # get the project name
         project = self.inv.get_by_id(self.get_env(), network["tenant_id"])
diff --git a/app/discover/fetchers/api/api_fetch_networks.py b/app/discover/fetchers/api/api_fetch_networks.py
index 4b70f65..f76517a 100644
--- a/app/discover/fetchers/api/api_fetch_networks.py
+++ b/app/discover/fetchers/api/api_fetch_networks.py
@@ -34,7 +34,7 @@ class ApiFetchNetworks(ApiAccess):
             "X-Auth-Token": token["id"]
         }
         response = self.get_url(req_url, headers)
-        if not "networks" in response:
+        if "networks" not in response:
             return []
         networks = response["networks"]
         req_url = endpoint + "/v2.0/subnets"
@@ -46,7 +46,6 @@ class ApiFetchNetworks(ApiAccess):
             for s in subnets:
                 subnets_hash[s["id"]] = s
         for doc in networks:
-            doc["master_parent_type"] = "project"
             project_id = doc["tenant_id"]
             if not project_id:
                 # find project ID of admin project
@@ -57,12 +56,12 @@ class ApiFetchNetworks(ApiAccess):
                 if not project:
                     self.log.error("failed to find admin project in DB")
                 project_id = project["id"]
-            doc["master_parent_id"] = project_id
-            doc["parent_type"] = "networks_folder"
-            doc["parent_id"] = project_id + "-networks"
-            doc["parent_text"] = "Networks"
-            # set the 'network' attribute for network objects to the name of network,
-            # to allow setting constraint on network when creating network clique
+            self.set_folder_parent(doc, object_type='network',
+                                   master_parent_id=project_id,
+                                   master_parent_type='project')
+            # set the 'network' attribute for network objects to the name of
+            # network, to allow setting constraint on network when creating
+            # network clique
             doc['network'] = doc["id"]
             # get the project name
             project = self.inv.get_by_id(self.get_env(), project_id)
diff --git a/app/discover/fetchers/api/api_fetch_port.py b/app/discover/fetchers/api/api_fetch_port.py
index f8d9eeb..8de1452 100644
--- a/app/discover/fetchers/api/api_fetch_port.py
+++ b/app/discover/fetchers/api/api_fetch_port.py
@@ -43,11 +43,9 @@ class ApiFetchPort(ApiAccess):
             return []
 
         doc = response["port"]
-        doc["master_parent_type"] = "network"
-        doc["master_parent_id"] = doc["network_id"]
-        doc["parent_type"] = "ports_folder"
-        doc["parent_id"] = doc["network_id"] + "-ports"
-        doc["parent_text"] = "Ports"
+        self.set_folder_parent(doc, object_type="port",
+                               master_parent_type="network",
+                               master_parent_id=doc["network_id"])
         # get the project name
         net = self.inv.get_by_id(self.get_env(), doc["network_id"])
         if net:
diff --git a/app/discover/fetchers/api/api_fetch_ports.py b/app/discover/fetchers/api/api_fetch_ports.py
index f4c54a6..5e44c1b 100644
--- a/app/discover/fetchers/api/api_fetch_ports.py
+++ b/app/discover/fetchers/api/api_fetch_ports.py
@@ -38,11 +38,9 @@ class ApiFetchPorts(ApiAccess):
             return []
         ports = response["ports"]
         for doc in ports:
-            doc["master_parent_type"] = "network"
-            doc["master_parent_id"] = doc["network_id"]
-            doc["parent_type"] = "ports_folder"
-            doc["parent_id"] = doc["network_id"] + "-ports"
-            doc["parent_text"] = "Ports"
+            self.set_folder_parent(doc, object_type="port",
+                                   master_parent_type="network",
+                                   master_parent_id=doc["network_id"])
             # get the project name
             net = self.inv.get_by_id(self.get_env(), doc["network_id"])
             if net:
diff --git a/app/discover/fetchers/api/api_fetch_project_hosts.py b/app/discover/fetchers/api/api_fetch_project_hosts.py
index 2aeb24f..1059600 100644
--- a/app/discover/fetchers/api/api_fetch_project_hosts.py
+++ b/app/discover/fetchers/api/api_fetch_project_hosts.py
@@ -11,11 +11,11 @@ import json
 
 from discover.fetchers.api.api_access import ApiAccess
 from discover.fetchers.db.db_access import DbAccess
-from discover.fetchers.cli.cli_access import CliAccess
+from discover.fetchers.cli.cli_fetch_host_details import CliFetchHostDetails
 from utils.ssh_connection import SshError
 
 
-class ApiFetchProjectHosts(ApiAccess, DbAccess, CliAccess):
+class ApiFetchProjectHosts(ApiAccess, DbAccess, CliFetchHostDetails):
     def __init__(self):
         super(ApiFetchProjectHosts, self).__init__()
 
diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py
index c77b22a..68b81c8 100644
--- a/app/discover/fetchers/cli/cli_access.py
+++ b/app/discover/fetchers/cli/cli_access.py
@@ -17,7 +17,7 @@ from utils.logging.console_logger import ConsoleLogger
 from utils.ssh_conn import SshConn
 
 
-class CliAccess(BinaryConverter, Fetcher):
+class CliAccess(Fetcher, BinaryConverter):
     connections = {}
     ssh_cmd = "ssh -q -o StrictHostKeyChecking=no "
     call_count_per_con = {}
@@ -71,8 +71,9 @@ class CliAccess(BinaryConverter, Fetcher):
         self.cached_commands[cmd_path] = {"timestamp": curr_time, "result": ret}
         return ret
 
-    def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True):
-        out = self.run(cmd, ssh_to_host, enable_cache)
+    def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True,
+                        use_sudo=True):
+        out = self.run(cmd, ssh_to_host, enable_cache, use_sudo=use_sudo)
         if not out:
             return []
         # first try to split lines by whitespace
@@ -236,7 +237,7 @@ class CliAccess(BinaryConverter, Fetcher):
             self.find_matching_regexps(o, line, regexps)
         for regexp_tuple in regexps:
             name = regexp_tuple['name']
-            if 'name' not in o and 'default' in regexp_tuple:
+            if name not in o and 'default' in regexp_tuple:
                 o[name] = regexp_tuple['default']
 
     @staticmethod
@@ -247,4 +248,8 @@ class CliAccess(BinaryConverter, Fetcher):
             regex = re.compile(regex)
             matches = regex.search(line)
             if matches and name not in o:
-                o[name] = matches.group(1)
+                try:
+                    o[name] = matches.group(1)
+                except IndexError as e:
+                    self.log.error('failed to find group 1 in match, {}'
+                                   .format(str(regexp_tuple)))
diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
index 26cd603..81d164d 100644
--- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
@@ -27,8 +27,8 @@ class CliFetchHostPnics(CliAccess):
              'description': 'IPv6 Address'}
         ]
 
-    def get(self, id):
-        host_id = id[:id.rindex("-")]
+    def get(self, parent_id):
+        host_id = parent_id[:parent_id.rindex("-")]
         cmd = 'ls -l /sys/class/net | grep ^l | grep -v "/virtual/"'
         host = self.inv.get_by_id(self.get_env(), host_id)
         if not host:
@@ -39,7 +39,8 @@ class CliFetchHostPnics(CliAccess):
                            ", host: " + str(host))
             return []
         host_types = host["host_type"]
-        if "Network" not in host_types and "Compute" not in host_types:
+        accepted_host_types = ['Network', 'Compute', 'Kube-node']
+        if not [t for t in accepted_host_types if t in host_types]:
             return []
         interface_lines = self.run_fetch_lines(cmd, host_id)
         interfaces = []
diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py
index ff37569..ac04568 100644
--- a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py
+++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py
@@ -18,8 +18,8 @@ class CliFetchVconnectorsOvs(CliFetchVconnectors):
 
     def get_vconnectors(self, host):
         host_id = host['id']
-        lines = self.run_fetch_lines("brctl show", host_id)
-        headers = ["bridge_name", "bridge_id", "stp_enabled", "interfaces"]
+        lines = self.run_fetch_lines('brctl show', host_id)
+        headers = ['bridge_name', 'bridge_id', 'stp_enabled', 'interfaces']
         headers_count = len(headers)
         # since we hard-coded the headers list, remove the headers line
         del lines[:1]
@@ -31,26 +31,32 @@ class CliFetchVconnectorsOvs(CliFetchVconnectors):
         results = self.parse_cmd_result_with_whitespace(fixed_lines, headers, False)
         ret = []
         for doc in results:
-            doc["name"] = doc.pop("bridge_name")
-            doc["id"] = doc["name"] + "-" +  doc.pop("bridge_id")
-            doc["host"] = host_id
-            doc["connector_type"] = "bridge"
-            if "interfaces" in doc:
-                interfaces = {}
-                interface_names = doc["interfaces"].split(",")
-                for interface_name in interface_names:
-                    # find MAC address for this interface from ports list
-                    port_id_prefix = interface_name[3:]
-                    port = self.inv.find_items({
-                        "environment": self.get_env(),
-                        "type": "port",
-                        "binding:host_id": host_id,
-                        "id": {"$regex": r"^" + re.escape(port_id_prefix)}
-                    }, get_single=True)
-                    mac_address = '' if not port else port['mac_address']
-                    interface = {'name': interface_name, 'mac_address': mac_address}
-                    interfaces[interface_name] = interface
-                doc["interfaces"] = interfaces
-                doc['interfaces_names'] = list(interfaces.keys())
-                ret.append(doc)
+            doc['name'] = '{}-{}'.format(host_id, doc['bridge_name'])
+            doc['id'] = '{}-{}'.format(doc['name'], doc.pop('bridge_id'))
+            doc['host'] = host_id
+            doc['connector_type'] = 'bridge'
+            self.get_vconnector_interfaces(doc, host_id)
+            ret.append(doc)
         return ret
+
+    def get_vconnector_interfaces(self, doc, host_id):
+        if 'interfaces' not in doc:
+            doc['interfaces'] = {}
+            doc['interfaces_names'] = []
+            return
+        interfaces = {}
+        interface_names = doc['interfaces'].split(',')
+        for interface_name in interface_names:
+            # find MAC address for this interface from ports list
+            port_id_prefix = interface_name[3:]
+            port = self.inv.find_items({
+                'environment': self.get_env(),
+                'type': 'port',
+                'binding:host_id': host_id,
+                'id': {'$regex': r'^' + re.escape(port_id_prefix)}
+            }, get_single=True)
+            mac_address = '' if not port else port['mac_address']
+            interface = {'name': interface_name, 'mac_address': mac_address}
+            interfaces[interface_name] = interface
+        doc['interfaces'] = interfaces
+        doc['interfaces_names'] = list(interfaces.keys())
diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
index 3bc3a5b..0129d3b 100644
--- a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
+++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
@@ -66,17 +66,15 @@ class CliFetchVserviceVnics(CliAccess):
                     master_parent_id = "{}-{}".format(host, service)
                     current = {
                         "id": host + "-" + name,
-                        "type": "vnic",
                         "vnic_type": "vservice_vnic",
                         "host": host,
                         "name": name,
-                        "master_parent_type": "vservice",
-                        "master_parent_id": master_parent_id,
-                        "parent_type": "vnics_folder",
-                        "parent_id": "{}-vnics".format(master_parent_id),
-                        "parent_text": "vNICs",
                         "lines": []
                     }
+                    self.set_folder_parent(current, object_type="vnic",
+                                           master_parent_type="vservice",
+                                           master_parent_id=master_parent_id,
+                                           parent_text="vNICs")
                     interfaces.append(current)
                     self.handle_line(current, line_remainder)
             else:
diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py
index 85376ed..7721136 100644
--- a/app/discover/fetchers/db/db_fetch_oteps.py
+++ b/app/discover/fetchers/db/db_fetch_oteps.py
@@ -82,4 +82,4 @@ class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton):
                     interface = l.split(":")[1].strip()
 
         if vconnector:
-            doc["vconnector"] = vconnector
+            doc["vconnector"] = '{}-{}'.format(host_id, vconnector)
diff --git a/app/discover/link_finders/find_links_for_vedges.py b/app/discover/link_finders/find_links_for_vedges.py
index f9719b4..afabdbe 100644
--- a/app/discover/link_finders/find_links_for_vedges.py
+++ b/app/discover/link_finders/find_links_for_vedges.py
@@ -104,8 +104,6 @@ class FindLinksForVedges(FindLinks):
         if "pnic" in vedge:
             if pname != vedge["pnic"]:
                 return
-        elif self.configuration.has_network_plugin('VPP'):
-            pass
         pnic = self.inv.find_items({
             "environment": self.get_env(),
             "type": "host_pnic",
diff --git a/app/discover/scan.py b/app/discover/scan.py
index 49f37ff..fb5e833 100755
--- a/app/discover/scan.py
+++ b/app/discover/scan.py
@@ -22,6 +22,7 @@ from discover.scan_error import ScanError
 from discover.scanner import Scanner
 from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager
 from utils.constants import EnvironmentFeatures
+from utils.origins import ScanOrigin, ScanOrigins
 from utils.mongo_access import MongoAccess
 from utils.exceptions import ScanArgumentsError
 from utils.inventory_mgr import InventoryMgr
@@ -112,6 +113,7 @@ class ScanPlan:
 
 class ScanController(Fetcher):
     DEFAULTS = {
+        "_id": None,
         "env": "",
         "mongo_config": "",
         "type": "",
@@ -126,7 +128,8 @@ class ScanController(Fetcher):
         "cliques_only": False,
         "monitoring_setup_only": False,
         "clear": False,
-        "clear_all": False
+        "clear_all": False,
+        "scheduled": False
     }
 
     def __init__(self):
@@ -274,9 +277,13 @@ class ScanController(Fetcher):
         self.conf.use_env(env_name)
 
         # generate ScanObject Class and instance.
+        origin = ScanOrigin(origin_id=args['_id'],
+                            origin_type=ScanOrigins.SCHEDULED
+                                        if args["scheduled"]
+                                        else ScanOrigins.MANUAL)
         scanner = Scanner()
         scanner.log.set_loglevel(args['loglevel'])
-        scanner.set_env(env_name)
+        scanner.setup(env=env_name, origin=origin)
         scanner.found_errors[env_name] = False
 
         # decide what scanning operations to do
diff --git a/app/discover/scan_manager.py b/app/discover/scan_manager.py
index 91dd06c..6e31bbd 100644
--- a/app/discover/scan_manager.py
+++ b/app/discover/scan_manager.py
@@ -103,7 +103,8 @@ class ScanManager(Manager):
 
     def _build_scan_args(self, scan_request: dict):
         args = {
-            'mongo_config': self.args.mongo_config
+            'mongo_config': self.args.mongo_config,
+            'scheduled': True if scan_request.get('interval') else False
         }
 
         def set_arg(name_from: str, name_to: str = None):
@@ -113,6 +114,7 @@ class ScanManager(Manager):
             if val:
                 args[name_to] = val
 
+        set_arg("_id")
         set_arg("object_id", "id")
         set_arg("log_level", "loglevel")
         set_arg("environment", "env")
diff --git a/app/discover/scanner.py b/app/discover/scanner.py
index 8aac40b..8d36baf 100644
--- a/app/discover/scanner.py
+++ b/app/discover/scanner.py
@@ -10,6 +10,7 @@
 # base class for scanners
 
 import json
+
 import os
 import queue
 import traceback
@@ -27,9 +28,6 @@ from utils.ssh_connection import SshError
 
 class Scanner(Fetcher):
 
-    ENV_TYPE_OPENSTACK = 'OpenStack'
-    ENV_TYPE_KUBERNETES = 'Kubernetes'
-
     config = None
     environment = None
     env = None
@@ -92,11 +90,11 @@ class Scanner(Fetcher):
             else basic_cond
         if not env_cond:
             env_cond = basic_cond
-        if 'environment_type' not in env_cond:
+        if 'environment_type' not in env_cond.keys():
             env_cond.update(basic_cond)
         if not isinstance(env_cond, dict):
-            self.log.warn('illegal environment_condition given '
-                          'for type {}'.format(type_to_fetch['type']))
+            self.log.warn('Illegal environment_condition given '
+                          'for type {type}'.format(type=type_to_fetch['type']))
             return True
         conf = self.config.get_env_config()
         if 'environment_type' not in conf:
@@ -104,14 +102,24 @@ class Scanner(Fetcher):
         for attr, required_val in env_cond.items():
             if attr == "mechanism_drivers":
                 if "mechanism_drivers" not in conf:
-                    self.log.warn('illegal environment configuration: '
+                    self.log.warn('Illegal environment configuration: '
                                   'missing mechanism_drivers')
                     return False
                 if not isinstance(required_val, list):
                     required_val = [required_val]
-                return bool(set(required_val) & set(conf["mechanism_drivers"]))
-            elif attr not in conf or conf[attr] != required_val:
+                value_ok = bool(set(required_val) &
+                                set(conf["mechanism_drivers"]))
+                if not value_ok:
+                    return False
+            elif attr not in conf:
                 return False
+            else:
+                if isinstance(required_val, list):
+                    if conf[attr] not in required_val:
+                        return False
+                else:
+                    if conf[attr] != required_val:
+                        return False
         # no check failed
         return True
 
@@ -132,18 +140,20 @@ class Scanner(Fetcher):
         if not isinstance(fetcher, Fetcher):
             type_to_fetch['fetcher'] = fetcher()  # make it an instance
             fetcher = type_to_fetch["fetcher"]
-        fetcher.set_env(self.get_env())
+        fetcher.setup(env=self.get_env(), origin=self.origin)
 
         # get children_scanner instance
         children_scanner = type_to_fetch.get("children_scanner")
 
         escaped_id = fetcher.escape(str(obj_id)) if obj_id else obj_id
         self.log.info(
-            "scanning : type=%s, parent: (type=%s, name=%s, id=%s)",
-            type_to_fetch["type"],
-            parent.get('type', 'environment'),
-            parent.get('name', ''),
-            escaped_id)
+            "Scanning: type={type}, "
+            "parent: (type={parent_type}, "
+            "name={parent_name}, "
+            "id={parent_id})".format(type=type_to_fetch["type"],
+                                     parent_type=parent.get('type', 'environment'),
+                                     parent_name=parent.get('name', ''),
+                                     parent_id=escaped_id))
 
         # fetch OpenStack data from environment by CLI, API or MySQL
         # or physical devices data from ACI API
@@ -154,18 +164,21 @@ class Scanner(Fetcher):
             self.found_errors[self.get_env()] = True
             return []
         except Exception as e:
-            self.log.error("Error while scanning : " +
-                           "fetcher=%s, " +
-                           "type=%s, " +
-                           "parent: (type=%s, name=%s, id=%s), " +
-                           "error: %s",
-                           fetcher.__class__.__name__,
-                           type_to_fetch["type"],
-                           "environment" if "type" not in parent
-                           else parent["type"],
-                           "" if "name" not in parent else parent["name"],
-                           escaped_id,
-                           e)
+            self.log.error(
+                "Error while scanning: fetcher={fetcher}, type={type}, "
+                "parent: (type={parent_type}, name={parent_name}, "
+                "id={parent_id}), "
+                "error: {error}".format(fetcher=fetcher.__class__.__name__,
+                                        type=type_to_fetch["type"],
+                                        parent_type="environment"
+                                                    if "type" not in parent
+                                                    else parent["type"],
+                                        parent_name=""
+                                                    if "name" not in parent
+                                                    else parent["name"],
+                                        parent_id=escaped_id,
+                                        error=e))
+
             traceback.print_exc()
             raise ScanError(str(e))
 
@@ -232,14 +245,16 @@ class Scanner(Fetcher):
         self.log.info("Scan complete")
 
     def scan_links(self):
-        self.log.info("scanning for links")
+        self.log.info("Scanning for links")
         for fetcher in self.link_finders:
-            fetcher.set_env(self.get_env())
+            fetcher.setup(env=self.get_env(),
+                          origin=self.origin)
             fetcher.add_links()
 
     def scan_cliques(self):
         clique_scanner = CliqueFinder()
-        clique_scanner.set_env(self.get_env())
+        clique_scanner.setup(env=self.get_env(),
+                             origin=self.origin)
         clique_scanner.find_cliques()
 
     def deploy_monitoring_setup(self):