Calipso initial release for OPNFV

Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>

16 files changed, 1075 insertions, 0 deletions

diff --git a/app/discover/fetchers/cli/__init__.py b/app/discover/fetchers/cli/__init__.py
new file mode 100644
index 0000000..b0637e9
--- /dev/null
+++ b/app/discover/fetchers/cli/__init__.py
@@ -0,0 +1,9 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py
new file mode 100644
index 0000000..1db84ea
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_access.py
@@ -0,0 +1,206 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+import time
+
+from discover.fetcher import Fetcher
+from utils.binary_converter import BinaryConverter
+from utils.logging.console_logger import ConsoleLogger
+from utils.ssh_conn import SshConn
+
+
+class CliAccess(BinaryConverter, Fetcher):
+    connections = {}
+    ssh_cmd = "ssh -o StrictHostKeyChecking=no "
+    call_count_per_con = {}
+    max_call_count_per_con = 100
+    cache_lifetime = 60  # no. of seconds to cache results
+    cached_commands = {}
+
+    def __init__(self):
+        super().__init__()
+        self.log = ConsoleLogger()
+
+    @staticmethod
+    def is_gateway_host(ssh_to_host):
+        ssh_conn = SshConn(ssh_to_host)
+        return ssh_conn.is_gateway_host(ssh_to_host)
+
+    def run_on_gateway(self, cmd, ssh_to_host="", enable_cache=True,
+                       use_sudo=True):
+        self.run(cmd, ssh_to_host=ssh_to_host, enable_cache=enable_cache,
+                 on_gateway=True, use_sudo=use_sudo)
+
+    def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False,
+            ssh=None, use_sudo=True):
+        ssh_conn = ssh if ssh else SshConn(ssh_to_host)
+        if use_sudo and not cmd.strip().startswith("sudo "):
+            cmd = "sudo " + cmd
+        if not on_gateway and ssh_to_host \
+                and not ssh_conn.is_gateway_host(ssh_to_host):
+            cmd = self.ssh_cmd + ssh_to_host + " " + cmd
+        curr_time = time.time()
+        cmd_path = ssh_to_host + ',' + cmd
+        if enable_cache and cmd_path in self.cached_commands:
+            # try to re-use output from last call
+            cached = self.cached_commands[cmd_path]
+            if cached["timestamp"] + self.cache_lifetime < curr_time:
+                # result expired
+                self.cached_commands.pop(cmd_path, None)
+            else:
+                # result is good to use - skip the SSH call
+                self.log.info('CliAccess: ****** using cached result, ' +
+                              'host: ' + ssh_to_host + ', cmd: %s ******', cmd)
+                return cached["result"]
+
+        self.log.info('CliAccess: host: %s, cmd: %s', ssh_to_host, cmd)
+        ret = ssh_conn.exec(cmd)
+        self.cached_commands[cmd_path] = {"timestamp": curr_time, "result": ret}
+        return ret
+
+    def run_fetch_lines(self, cmd, ssh_to_host="", enable_cache=True):
+        out = self.run(cmd, ssh_to_host, enable_cache)
+        if not out:
+            return []
+        # first try to split lines by whitespace
+        ret = out.splitlines()
+        # if split by whitespace did not work, try splitting by "\\n"
+        if len(ret) == 1:
+            ret = [l for l in out.split("\\n") if l != ""]
+        return ret
+
+    # parse command output columns separated by whitespace
+    # since headers can contain whitespace themselves,
+    # it is the caller's responsibility to provide the headers
+    def parse_cmd_result_with_whitespace(self, lines, headers, remove_first):
+        if remove_first:
+            # remove headers line
+            del lines[:1]
+        results = [self.parse_line_with_ws(line, headers)
+                   for line in lines]
+        return results
+
+    # parse command output with "|" column separators and "-" row separators
+    def parse_cmd_result_with_separators(self, lines):
+        headers = self.parse_headers_line_with_separators(lines[1])
+        # remove line with headers and formatting lines above it and below it
+        del lines[:3]
+        # remove formatting line in the end
+        lines.pop()
+        results = [self.parse_content_line_with_separators(line, headers)
+                   for line in lines]
+        return results
+
+    # parse a line with columns separated by whitespace
+    def parse_line_with_ws(self, line, headers):
+        s = line if isinstance(line, str) else self.binary2str(line)
+        parts = [word.strip() for word in s.split() if word.strip()]
+        ret = {}
+        for i, p in enumerate(parts):
+            header = headers[i]
+            ret[header] = p
+        return ret
+
+    # parse a line with "|" column separators
+    def parse_line_with_separators(self, line):
+        s = self.binary2str(line)
+        parts = [word.strip() for word in s.split("|") if word.strip()]
+        # remove the ID field
+        del parts[:1]
+        return parts
+
+    def parse_headers_line_with_separators(self, line):
+        return self.parse_line_with_separators(line)
+
+    def parse_content_line_with_separators(self, line, headers):
+        content_parts = self.parse_line_with_separators(line)
+        content = {}
+        for i in range(0, len(content_parts)):
+            content[headers[i]] = content_parts[i]
+        return content
+
+    def merge_ws_spillover_lines(self, lines):
+        # with WS-separated output, extra output sometimes spills to next line
+        # detect that and add to the end of the previous line for our procesing
+        pending_line = None
+        fixed_lines = []
+        # remove headers line
+        for l in lines:
+            if l[0] == '\t':
+                # this is a spill-over line
+                if pending_line:
+                    # add this line to the end of the previous line
+                    pending_line = pending_line.strip() + "," + l.strip()
+            else:
+                # add the previous pending line to the fixed lines list
+                if pending_line:
+                    fixed_lines.append(pending_line)
+                # make current line the pending line
+                pending_line = l
+        if pending_line:
+            fixed_lines.append(pending_line)
+        return fixed_lines
+
+    """
+    given output lines from CLI command like 'ip -d link show',
+    find lines belonging to section describing a specific interface
+    parameters:
+    - lines: list of strings, output of command
+    - header_regexp: regexp marking the start of the section
+    - end_regexp: regexp marking the end of the section
+    """
+    def get_section_lines(self, lines, header_regexp, end_regexp):
+        if not lines:
+            return []
+        header_re = re.compile(header_regexp)
+        start_pos = None
+        # find start_pos of section
+        line_count = len(lines)
+        for line_num in range(0, line_count-1):
+            matches = header_re.match(lines[line_num])
+            if matches:
+                start_pos = line_num
+                break
+        if not start_pos:
+            return []
+        # find end of section
+        end_pos = line_count
+        end_re = re.compile(end_regexp)
+        for line_num in range(start_pos+1, end_pos-1):
+            matches = end_re.match(lines[line_num])
+            if matches:
+                end_pos = line_num
+                break
+        return lines[start_pos:end_pos]
+
+    def get_object_data(self, o, lines, regexps):
+        """
+        find object data in output lines from CLI command
+        parameters:
+        - o: object (dict), to which we'll add attributes with the data found
+        - lines: list of strings
+        - regexps: dict, keys are attribute names, values are regexp to match
+                    for finding the value of the attribute
+        """
+        for line in lines:
+            self.find_matching_regexps(o, line, regexps)
+        for regexp_tuple in regexps:
+            name = regexp_tuple['name']
+            if 'name' not in o and 'default' in regexp_tuple:
+                o[name] = regexp_tuple['default']
+
+    def find_matching_regexps(self, o, line, regexps):
+        for regexp_tuple in regexps:
+            name = regexp_tuple['name']
+            regex = regexp_tuple['re']
+            regex = re.compile(regex)
+            matches = regex.search(line)
+            if matches:
+                o[name] = matches.group(1)
diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
new file mode 100644
index 0000000..3516e25
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py
@@ -0,0 +1,122 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchHostPnics(CliAccess):
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+        self.ethtool_attr = re.compile('^\s+([^:]+):\s(.*)$')
+        self.regexps = [
+            {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'},
+            {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'},
+            {'name': 'IP Address', 're': '^\s*inet addr:?(\S+)\s.*$'},
+            {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'},
+            {'name': 'IPv6 Address', 're': '^\s*inet6 addr:\s*(\S+)(\s.*)?$'},
+            {'name': 'IPv6 Address', 're': '^\s*inet6 \s*(\S+)(\s.*)?$'}
+        ]
+
+    def get(self, id):
+        host_id = id[:id.rindex("-")]
+        cmd = 'ls -l /sys/class/net | grep ^l | grep -v "/virtual/"'
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host:
+            self.log.error("CliFetchHostPnics: host not found: " + host_id)
+            return []
+        if "host_type" not in host:
+            self.log.error("host does not have host_type: " + host_id +
+                           ", host: " + str(host))
+            return []
+        host_types = host["host_type"]
+        if "Network" not in host_types and "Compute" not in host_types:
+            return []
+        interface_lines = self.run_fetch_lines(cmd, host_id)
+        interfaces = []
+        for line in interface_lines:
+            interface_name = line[line.rindex('/')+1:]
+            interface_name = interface_name.strip()
+            # run ifconfig with specific interface name,
+            # since running it with no name yields a list without inactive pNICs
+            interface = self.find_interface_details(host_id, interface_name)
+            if interface:
+                interfaces.append(interface)
+        return interfaces
+
+    def find_interface_details(self, host_id, interface_name):
+        lines = self.run_fetch_lines("ifconfig " + interface_name, host_id)
+        interface = None
+        status_up = None
+        for line in [l for l in lines if l != '']:
+            tokens = None
+            if interface is None:
+                tokens = line.split()
+                name = tokens[0].strip('- :')
+                name = name.strip()
+                if name == interface_name:
+                    line_remainder = line.strip('-')[len(interface_name)+2:]
+                    line_remainder = line_remainder.strip(' :')
+                    id = interface_name
+                    interface = {
+                        "host": host_id,
+                        "name": id,
+                        "local_name": interface_name,
+                        "lines": []
+                    }
+                    self.handle_line(interface, line_remainder)
+                    if '<UP,' in line:
+                        status_up = True
+            if status_up is None:
+                if tokens is None:
+                    tokens = line.split()
+                if 'BROADCAST' in tokens:
+                    status_up = 'UP' in tokens
+            if interface:
+                self.handle_line(interface, line)
+        self.set_interface_data(interface)
+        interface['state'] = 'UP' if status_up else 'DOWN'
+        if 'id' not in interface:
+            interface['id'] = interface_name + '-unknown_mac'
+        return interface
+
+    def handle_line(self, interface, line):
+        self.find_matching_regexps(interface, line, self.regexps)
+        if 'mac_address' in interface:
+            interface["id"] = interface["name"] + "-" + interface["mac_address"]
+        interface["lines"].append(line.strip())
+
+    def set_interface_data(self, interface):
+        if not interface:
+            return
+        interface["data"] = "\n".join(interface["lines"])
+        interface.pop("lines", None)
+        ethtool_ifname = interface["local_name"]
+        if "@" in interface["local_name"]:
+            pos = interface["local_name"].index("@")
+            ethtool_ifname = ethtool_ifname[pos + 1:]
+        cmd = "ethtool " + ethtool_ifname
+        lines = self.run_fetch_lines(cmd, interface["host"])
+        attr = None
+        for line in lines[1:]:
+            matches = self.ethtool_attr.match(line)
+            if matches:
+                # add this attribute to the interface
+                attr = matches.group(1)
+                value = matches.group(2)
+                interface[attr] = value.strip()
+            else:
+                # add more values to the current attribute as an array
+                if isinstance(interface[attr], str):
+                    interface[attr] = [interface[attr], line.strip()]
+                else:
+                    interface[attr].append(line.strip())
diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py b/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py
new file mode 100644
index 0000000..69b6413
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_host_pnics_vpp.py
@@ -0,0 +1,44 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+
+from discover.fetcher import Fetcher
+from utils.inventory_mgr import InventoryMgr
+
+NAME_RE = '^[a-zA-Z]*GigabitEthernet'
+
+class CliFetchHostPnicsVpp(Fetcher):
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+        self.name_re = re.compile(NAME_RE)
+
+    def get(self, id):
+        host_id = id[:id.rindex("-")]
+        host_id = id[:host_id.rindex("-")]
+        vedges = self.inv.find_items({
+            "environment": self.get_env(),
+            "type": "vedge",
+            "host": host_id
+        })
+        ret = []
+        for vedge in vedges:
+            pnic_ports = vedge['ports']
+            for pnic_name in pnic_ports:
+                if not self.name_re.search(pnic_name):
+                    continue
+                pnic = pnic_ports[pnic_name]
+                pnic['host'] = host_id
+                pnic['id'] = host_id + "-pnic-" + pnic_name
+                pnic['type'] = 'pnic'
+                pnic['object_name'] = pnic_name
+                pnic['Link detected'] = 'yes' if pnic['state'] == 'up' else 'no'
+                ret.append(pnic)
+        return ret
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservice.py b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
new file mode 100644
index 0000000..9f8173f
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservice.py
@@ -0,0 +1,80 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+
+from discover.fetchers.cli.cli_access import CliAccess
+from discover.fetchers.db.db_access import DbAccess
+from discover.network_agents_list import NetworkAgentsList
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchHostVservice(CliAccess, DbAccess):
+    def __init__(self):
+        super(CliFetchHostVservice, self).__init__()
+        # match only DHCP agent and router (L3 agent)
+        self.type_re = re.compile("^q(dhcp|router)-")
+        self.inv = InventoryMgr()
+        self.agents_list = NetworkAgentsList()
+
+    def get_vservice(self, host_id, name_space):
+        result = {"local_service_id": name_space}
+        self.set_details(host_id, result)
+        return result
+
+    def set_details(self, host_id, r):
+        # keep the index without prefix
+        id_full = r["local_service_id"].strip()
+        prefix = id_full[1:id_full.index('-')]
+        id_clean = id_full[id_full.index('-') + 1:]
+        r["service_type"] = prefix
+        name = self.get_router_name(r, id_clean) if prefix == "router" \
+            else self.get_network_name(id_clean)
+        r["name"] = prefix + "-" + name
+        r["host"] = host_id
+        r["id"] = host_id + "-" + id_full
+        self.set_agent_type(r)
+
+    def get_network_name(self, id):
+        query = """
+                SELECT name
+                FROM {}.networks
+                WHERE id = %s
+                """.format(self.neutron_db)
+        results = self.get_objects_list_for_id(query, "router", id)
+        if not list(results):
+            return id
+        for db_row in results:
+            return db_row["name"]
+
+    def get_router_name(self, r, id):
+        query = """
+                SELECT *
+                FROM {}.routers
+                WHERE id = %s
+                """.format(self.neutron_db)
+        results = self.get_objects_list_for_id(query, "router", id.strip())
+        for db_row in results:
+            r.update(db_row)
+        return r["name"]
+
+    # dynamically create sub-folder for vService by type
+    def set_agent_type(self, o):
+        o["master_parent_id"] = o["host"] + "-vservices"
+        o["master_parent_type"] = "vservices_folder"
+        atype = o["service_type"]
+        agent = self.agents_list.get_type(atype)
+        try:
+            o["parent_id"] = o["master_parent_id"] + "-" + agent["type"] + "s"
+            o["parent_type"] = "vservice_" + agent["type"] + "s_folder"
+            o["parent_text"] = agent["folder_text"]
+        except KeyError:
+            o["parent_id"] = o["master_parent_id"] + "-" + "miscellenaous"
+            o["parent_type"] = "vservice_miscellenaous_folder"
+            o["parent_text"] = "Misc. services"
diff --git a/app/discover/fetchers/cli/cli_fetch_host_vservices.py b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
new file mode 100644
index 0000000..9b62dcb
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_host_vservices.py
@@ -0,0 +1,27 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.cli.cli_fetch_host_vservice import CliFetchHostVservice
+
+
+class CliFetchHostVservices(CliFetchHostVservice):
+    def __init__(self):
+        super(CliFetchHostVservices, self).__init__()
+
+    def get(self, host_id):
+        host = self.inv.get_single(self.get_env(), "host", host_id)
+        if "Network" not in host["host_type"]:
+            return []
+        services_ids = [l[:l.index(' ')] if ' ' in l else l
+                        for l in self.run_fetch_lines("ip netns", host_id)]
+        results = [{"local_service_id": s} for s in services_ids if self.type_re.match(s)]
+        for r in results:
+            self.set_details(host_id, r)
+        return results
+
diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics.py
new file mode 100644
index 0000000..22ac573
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics.py
@@ -0,0 +1,22 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.cli.cli_fetch_instance_vnics_base import CliFetchInstanceVnicsBase
+
+
+class CliFetchInstanceVnics(CliFetchInstanceVnicsBase):
+    def __init__(self):
+        super().__init__()
+
+    def set_vnic_properties(self, v, instance):
+        super().set_vnic_properties(v, instance)
+        v["source_bridge"] = v["source"]["@bridge"]
+
+    def get_vnic_name(self, v, instance):
+        return v["target"]["@dev"]
diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
new file mode 100644
index 0000000..4de1840
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_base.py
@@ -0,0 +1,68 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import xmltodict
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchInstanceVnicsBase(CliAccess):
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, id):
+        instance_uuid = id[:id.rindex('-')]
+        instance = self.inv.get_by_id(self.get_env(), instance_uuid)
+        if not instance:
+            return []
+        host = self.inv.get_by_id(self.get_env(), instance["host"])
+        if not host or "Compute" not in host["host_type"]:
+            return []
+        lines = self.run_fetch_lines("virsh list", instance["host"])
+        del lines[:2]  # remove header
+        virsh_ids = [l.split()[0] for l in lines if l > ""]
+        results = []
+        # Note: there are 2 ids here of instances with local names, which are
+        # not connected to the data we have thus far for the instance
+        # therefore, we will decide whether the instance is the correct one
+        # based on comparison of the uuid in the dumpxml output
+        for id in virsh_ids:
+            results.extend(self.get_vnics_from_dumpxml(id, instance))
+        return results
+
+    def get_vnics_from_dumpxml(self, id, instance):
+        xml_string = self.run("virsh dumpxml " + id, instance["host"])
+        if not xml_string.strip():
+            return []
+        response = xmltodict.parse(xml_string)
+        if instance["uuid"] != response["domain"]["uuid"]:
+            # this is the wrong instance - skip it
+            return []
+        try:
+            vnics = response["domain"]["devices"]["interface"]
+        except KeyError:
+            return []
+        if isinstance(vnics, dict):
+            vnics = [vnics]
+        for v in vnics:
+            self.set_vnic_properties(v, instance)
+        return vnics
+
+    def set_vnic_properties(self, v, instance):
+        v["name"] = self.get_vnic_name(v, instance)
+        v["id"] = v["name"]
+        v["vnic_type"] = "instance_vnic"
+        v["host"] = instance["host"]
+        v["instance_id"] = instance["id"]
+        v["instance_db_id"] = instance["_id"]
+        v["mac_address"] = v["mac"]["@address"]
+        instance["mac_address"] = v["mac_address"]
+        self.inv.set(instance)
diff --git a/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py
new file mode 100644
index 0000000..58facd2
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_instance_vnics_vpp.py
@@ -0,0 +1,18 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.cli.cli_fetch_instance_vnics_base import CliFetchInstanceVnicsBase
+
+
+class CliFetchInstanceVnicsVpp(CliFetchInstanceVnicsBase):
+    def __init__(self):
+        super().__init__()
+
+    def get_vnic_name(self, v, instance):
+        return instance["name"] + "-" + v["@type"] + "-" + v["mac"]["@address"]
diff --git a/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py
new file mode 100644
index 0000000..1e65a14
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_oteps_lxb.py
@@ -0,0 +1,86 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.cli.cli_access import CliAccess
+from discover.fetchers.db.db_access import DbAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchOtepsLxb(CliAccess, DbAccess):
+
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+
+    def get(self, parent_id):
+        vconnector = self.inv.get_by_id(self.get_env(), parent_id)
+        if not vconnector:
+            return []
+        configurations = vconnector['configurations']
+        tunneling_ip = configurations['tunneling_ip']
+        tunnel_types_used = configurations['tunnel_types']
+        if not tunnel_types_used:
+            return []
+        tunnel_type = tunnel_types_used[0]
+        if not tunnel_type:
+            return []
+        # check only interfaces with name matching tunnel type
+        ret = [i for i in vconnector['interfaces'].values()
+               if i['name'].startswith(tunnel_type + '-')]
+        for otep in ret:
+            otep['ip_address'] = tunneling_ip
+            otep['host'] = vconnector['host']
+            self.get_otep_ports(otep)
+            otep['id'] = otep['host'] + '-otep-' + otep['name']
+            otep['name'] = otep['id']
+            otep['vconnector'] = vconnector['name']
+            otep['overlay_type'] = tunnel_type
+            self.get_udp_port(otep)
+        return ret
+
+    """
+    fetch OTEP data from CLI command 'ip -d link show'
+    """
+    def get_otep_ports(self, otep):
+        cmd = 'ip -d link show'
+        lines = self.run_fetch_lines(cmd, otep['host'])
+        header_format = '[0-9]+: ' + otep['name'] + ':'
+        interface_lines = self.get_section_lines(lines, header_format, '\S')
+        otep['data'] = '\n'.join(interface_lines)
+        regexps = [
+            {'name': 'state', 're': ',UP,', 'default': 'DOWN'},
+            {'name': 'mac_address', 're': '.*\slink/ether\s(\S+)\s'},
+            {'name': 'mtu', 're': '.*\smtu\s(\S+)\s'},
+        ]
+        self.get_object_data(otep, interface_lines, regexps)
+        cmd = 'bridge fdb show'
+        dst_line_format = ' dev ' + otep['name'] + ' dst '
+        lines = self.run_fetch_lines(cmd, otep['host'])
+        lines = [l for l in lines if dst_line_format in l]
+        if lines:
+            l = lines[0]
+            otep['bridge dst'] = l[l.index(' dst ')+5:]
+        return otep
+
+    def get_udp_port(self, otep):
+        table_name = "neutron.ml2_" + otep['overlay_type'] + "_endpoints"
+        results = None
+        try:
+            results = self.get_objects_list_for_id(
+                """
+                SELECT udp_port
+                FROM {}
+                WHERE host = %s
+                """.format(table_name),
+                "vedge", otep['host'])
+        except Exception as e:
+            self.log.error('failed to fetch UDP port for OTEP: ' + str(e))
+        otep['udp_port'] = 0
+        for result in results:
+            otep['udp_port'] = result['udp_port']
diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors.py b/app/discover/fetchers/cli/cli_fetch_vconnectors.py
new file mode 100644
index 0000000..78b767a
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vconnectors.py
@@ -0,0 +1,40 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from abc import abstractmethod, ABCMeta
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+from utils.singleton import Singleton
+
+
+class ABCSingleton(ABCMeta, Singleton):
+    pass
+
+
+class CliFetchVconnectors(CliAccess, metaclass=ABCSingleton):
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+
+    @abstractmethod
+    def get_vconnectors(self, host):
+        raise NotImplementedError("Subclass must override get_vconnectors()")
+
+    def get(self, id):
+        host_id = id[:id.rindex('-')]
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host:
+            self.log.error("CliFetchVconnectors: host not found: " + host_id)
+            return []
+        if "host_type" not in host:
+            self.log.error("host does not have host_type: " + host_id + \
+                           ", host: " + str(host))
+            return []
+        return self.get_vconnectors(host)
diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py
new file mode 100644
index 0000000..648dc63
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_lxb.py
@@ -0,0 +1,35 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import json
+
+from discover.fetchers.cli.cli_fetch_vconnectors_ovs import CliFetchVconnectorsOvs
+from discover.fetchers.db.db_access import DbAccess
+
+
+class CliFetchVconnectorsLxb(CliFetchVconnectorsOvs, DbAccess):
+
+    def __init__(self):
+        super().__init__()
+
+    def get(self, id):
+        ret = super().get(id)
+        for doc in ret:
+            query = """
+              SELECT configurations
+              FROM {}.agents
+              WHERE agent_type="Linux bridge agent" AND host = %s
+            """.format(self.neutron_db)
+            host = doc['host']
+            matches = self.get_objects_list_for_id(query, '', host)
+            if not matches:
+                raise ValueError('No Linux bridge agent in DB for host: {}'.format(host))
+            agent = matches[0]
+            doc['configurations'] = json.loads(agent['configurations'])
+        return ret
diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py
new file mode 100644
index 0000000..ff37569
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_ovs.py
@@ -0,0 +1,56 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+
+from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors
+
+
+class CliFetchVconnectorsOvs(CliFetchVconnectors):
+    def __init__(self):
+        super().__init__()
+
+    def get_vconnectors(self, host):
+        host_id = host['id']
+        lines = self.run_fetch_lines("brctl show", host_id)
+        headers = ["bridge_name", "bridge_id", "stp_enabled", "interfaces"]
+        headers_count = len(headers)
+        # since we hard-coded the headers list, remove the headers line
+        del lines[:1]
+
+        # intefaces can spill to next line - need to detect that and add
+        # them to the end of the previous line for our procesing
+        fixed_lines = self.merge_ws_spillover_lines(lines)
+
+        results = self.parse_cmd_result_with_whitespace(fixed_lines, headers, False)
+        ret = []
+        for doc in results:
+            doc["name"] = doc.pop("bridge_name")
+            doc["id"] = doc["name"] + "-" +  doc.pop("bridge_id")
+            doc["host"] = host_id
+            doc["connector_type"] = "bridge"
+            if "interfaces" in doc:
+                interfaces = {}
+                interface_names = doc["interfaces"].split(",")
+                for interface_name in interface_names:
+                    # find MAC address for this interface from ports list
+                    port_id_prefix = interface_name[3:]
+                    port = self.inv.find_items({
+                        "environment": self.get_env(),
+                        "type": "port",
+                        "binding:host_id": host_id,
+                        "id": {"$regex": r"^" + re.escape(port_id_prefix)}
+                    }, get_single=True)
+                    mac_address = '' if not port else port['mac_address']
+                    interface = {'name': interface_name, 'mac_address': mac_address}
+                    interfaces[interface_name] = interface
+                doc["interfaces"] = interfaces
+                doc['interfaces_names'] = list(interfaces.keys())
+                ret.append(doc)
+        return ret
diff --git a/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py
new file mode 100644
index 0000000..479e1db
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vconnectors_vpp.py
@@ -0,0 +1,64 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+from discover.fetchers.cli.cli_fetch_vconnectors import CliFetchVconnectors
+
+
+class CliFetchVconnectorsVpp(CliFetchVconnectors):
+    def __init__(self):
+        super().__init__()
+
+    def get_vconnectors(self, host):
+        lines = self.run_fetch_lines("vppctl show mode", host['id'])
+        vconnectors = {}
+        for l in lines:
+            if not l.startswith('l2 bridge'):
+                continue
+            line_parts = l.split(' ')
+            name = line_parts[2]
+            bd_id = line_parts[4]
+            if bd_id in vconnectors:
+                vconnector = vconnectors[bd_id]
+            else:
+                vconnector = {
+                    'host': host['id'],
+                    'id': host['id'] + '-vconnector-' + bd_id,
+                    'bd_id': bd_id,
+                    'name': "bridge-domain-" + bd_id,
+                    'interfaces': {},
+                    'interfaces_names': []
+                }
+                vconnectors[bd_id] = vconnector
+            interface = self.get_interface_details(host, name)
+            if interface:
+                vconnector['interfaces'][name] = interface
+                vconnector['interfaces_names'].append(name)
+        return list(vconnectors.values())
+
+    def get_interface_details(self, host, name):
+        # find vconnector interfaces
+        cmd = "vppctl show hardware-int " + name
+        interface_lines = self.run_fetch_lines(cmd, host['id'])
+        # remove header line
+        interface_lines.pop(0)
+        interface = None
+        for l in interface_lines:
+            if not l.strip():
+                continue  # ignore empty lines
+            if not l.startswith(' '):
+                details = l.split()
+                interface = {
+                    "name": details[0],
+                    "hardware": details[3],
+                    "state": details[2],
+                    "id": details[1],
+                }
+            elif l.startswith('  Ethernet address '):
+                interface['mac_address'] = l[l.rindex(' ') + 1:]
+        return interface
diff --git a/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py
new file mode 100644
index 0000000..f9c622d
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vpp_vedges.py
@@ -0,0 +1,58 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# Copyright 2016 cisco Corporation
+#oslo related message handling
+
+from oslo_serialization import jsonutils
+from oslo_utils import uuidutils
+import yaml
+
+from neutronclient.tests.functional import base
+
+
+class TestCLIFormatter(base.ClientTestBase):
+
+## old stuff ..not related to vpp..disregard
+    def setUp(self):
+        super(TestCLIFormatter, self).setUp()
+        self.net_name = 'net-%s' % uuidutils.generate_uuid()
+        self.addCleanup(self.neutron, 'net-delete %s' % self.net_name)
+
+    def _create_net(self, fmt, col_attrs):
+        params = ['-c %s' % attr for attr in col_attrs]
+        params.append('-f %s' % fmt)
+        params.append(self.net_name)
+        param_string = ' '.join(params)
+        return self.neutron('net-create', params=param_string)
+
+    def test_net_create_with_json_formatter(self):
+        result = self._create_net('json', ['name', 'admin_state_up'])
+        self.assertDictEqual({'name': self.net_name,
+                              'admin_state_up': True},
+                             jsonutils.loads(result))
+
+    def test_net_create_with_yaml_formatter(self):
+        result = self._create_net('yaml', ['name', 'admin_state_up'])
+        self.assertDictEqual({'name': self.net_name,
+                              'admin_state_up': True},
+                             yaml.load(result))
+
+    def test_net_create_with_value_formatter(self):
+        # NOTE(amotoki): In 'value' formatter, there is no guarantee
+        # in the order of attribute, so we use one attribute in this test.
+        result = self._create_net('value', ['name'])
+        self.assertEqual(self.net_name, result.strip())
+
+    def test_net_create_with_shell_formatter(self):
+        result = self._create_net('shell', ['name', 'admin_state_up'])
+        result_lines = set(result.strip().split('\n'))
+        self.assertSetEqual(set(['name="%s"' % self.net_name,
+                                 'admin_state_up="True"']),
+result_lines)
diff --git a/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
new file mode 100644
index 0000000..44ac8d6
--- /dev/null
+++ b/app/discover/fetchers/cli/cli_fetch_vservice_vnics.py
@@ -0,0 +1,140 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+import re
+
+from discover.fetchers.cli.cli_access import CliAccess
+from utils.inventory_mgr import InventoryMgr
+
+
+class CliFetchVserviceVnics(CliAccess):
+    def __init__(self):
+        super().__init__()
+        self.inv = InventoryMgr()
+        self.if_header = re.compile('^[-]?(\S+)\s+(.*)$')
+        self.regexps = [
+            {'name': 'mac_address', 're': '^.*\sHWaddr\s(\S+)(\s.*)?$'},
+            {'name': 'mac_address', 're': '^.*\sether\s(\S+)(\s.*)?$'},
+            {'name': 'netmask', 're': '^.*\sMask:\s?([0-9.]+)(\s.*)?$'},
+            {'name': 'netmask', 're': '^.*\snetmask\s([0-9.]+)(\s.*)?$'},
+            {'name': 'IP Address', 're': '^\s*inet addr:(\S+)\s.*$'},
+            {'name': 'IP Address', 're': '^\s*inet ([0-9.]+)\s.*$'},
+            {'name': 'IPv6 Address',
+             're': '^\s*inet6 addr: ?\s*([0-9a-f:/]+)(\s.*)?$'},
+            {'name': 'IPv6 Address',
+             're': '^\s*inet6 \s*([0-9a-f:/]+)(\s.*)?$'}
+        ]
+
+    def get(self, host_id):
+        host = self.inv.get_by_id(self.get_env(), host_id)
+        if not host:
+            self.log.error("host not found: " + host_id)
+            return []
+        if "host_type" not in host:
+            self.log.error("host does not have host_type: " + host_id +
+                           ", host: " + str(host))
+            return []
+        if "Network" not in host["host_type"]:
+            return []
+        lines = self.run_fetch_lines("ip netns", host_id)
+        ret = []
+        for l in [l for l in lines
+                  if l.startswith("qdhcp") or l.startswith("qrouter")]:
+            service = l.strip()
+            service = service if ' ' not in service \
+                else service[:service.index(' ')]
+            ret.extend(self.handle_service(host_id, service))
+        return ret
+
+    def handle_service(self, host, service, enable_cache=True):
+        cmd = "ip netns exec " + service + " ifconfig"
+        lines = self.run_fetch_lines(cmd, host, enable_cache)
+        interfaces = []
+        current = None
+        for line in lines:
+            matches = self.if_header.match(line)
+            if matches:
+                if current:
+                    self.set_interface_data(current)
+                name = matches.group(1).strip(":")
+                # ignore 'lo' interface
+                if name == 'lo':
+                    current = None
+                else:
+                    line_remainder = matches.group(2)
+                    vservice_id = host + "-" + service
+                    current = {
+                        "id": host + "-" + name,
+                        "type": "vnic",
+                        "vnic_type": "vservice_vnic",
+                        "host": host,
+                        "name": name,
+                        "master_parent_type": "vservice",
+                        "master_parent_id": vservice_id,
+                        "parent_type": "vnics_folder",
+                        "parent_id": vservice_id + "-vnics",
+                        "parent_text": "vNICs",
+                        "lines": []
+                    }
+                    interfaces.append(current)
+                    self.handle_line(current, line_remainder)
+            else:
+                if current:
+                    self.handle_line(current, line)
+        if current:
+            self.set_interface_data(current)
+        return interfaces
+
+    def handle_line(self, interface, line):
+        self.find_matching_regexps(interface, line, self.regexps)
+        interface["lines"].append(line.strip())
+
+    def set_interface_data(self, interface):
+        if not interface or 'IP Address' not in interface or 'netmask' not in interface:
+            return
+
+        interface["data"] = "\n".join(interface.pop("lines", None))
+        interface["cidr"] = self.get_cidr_for_vnic(interface)
+        network = self.inv.get_by_field(self.get_env(), "network", "cidrs",
+                                        interface["cidr"], get_single=True)
+        if not network:
+            return
+        interface["network"] = network["id"]
+        # set network for the vservice, to check network on clique creation
+        vservice = self.inv.get_by_id(self.get_env(),
+                                      interface["master_parent_id"])
+        network_id = network["id"]
+        if "network" not in vservice:
+            vservice["network"] = list()
+        if network_id not in vservice["network"]:
+            vservice["network"].append(network_id)
+        self.inv.set(vservice)
+
+    # find CIDR string by IP address and netmask
+    def get_cidr_for_vnic(self, vnic):
+        if "IP Address" not in vnic:
+            vnic["IP Address"] = "No IP Address"
+            return "No IP Address"
+        ipaddr = vnic["IP Address"].split('.')
+        netmask = vnic["netmask"].split('.')
+
+        # calculate network start
+        net_start = []
+        for pos in range(0, 4):
+            net_start.append(str(int(ipaddr[pos]) & int(netmask[pos])))
+
+        cidr_string = '.'.join(net_start) + '/'
+        cidr_string = cidr_string + self.get_net_size(netmask)
+        return cidr_string
+
+    def get_net_size(self, netmask):
+        binary_str = ''
+        for octet in netmask:
+            binary_str += bin(int(octet))[2:].zfill(8)
+        return str(len(binary_str.rstrip('0')))