From d32f75145676bacefde0d08a14680a5984623451 Mon Sep 17 00:00:00 2001 From: Koren Lev Date: Fri, 29 Sep 2017 01:38:18 +0300 Subject: release 1.0 calipso for opnfv apex Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2 Signed-off-by: Koren Lev --- app/discover/event_manager.py | 21 ++-- app/discover/events/event_interface_add.py | 7 +- app/discover/events/event_interface_delete.py | 3 +- app/discover/events/event_router_add.py | 2 +- app/discover/events/event_router_delete.py | 3 +- app/discover/events/event_router_update.py | 2 +- app/discover/events/event_subnet_add.py | 4 +- app/discover/events/event_subnet_update.py | 5 +- app/discover/events/listeners/default_listener.py | 16 +-- app/discover/events/listeners/listener_base.py | 14 +++ app/discover/fetcher_new.py | 30 ----- app/discover/fetchers/cli/cli_access.py | 62 ++++++++-- .../fetchers/cli/cli_fetch_bond_host_pnics.py | 134 +++++++++++++++++++++ app/discover/fetchers/cli/cli_fetch_host_pnics.py | 26 ++-- .../fetchers/cli/cli_fetch_host_vservice.py | 22 ++-- .../fetchers/cli/cli_fetch_host_vservices.py | 2 +- .../fetchers/cli/cli_fetch_instance_vnics_base.py | 2 +- .../fetchers/cli/cli_fetch_vservice_vnics.py | 8 +- app/discover/fetchers/db/db_access.py | 43 +++++-- .../fetchers/db/db_fetch_host_network_agents.py | 3 +- app/discover/fetchers/db/db_fetch_oteps.py | 4 +- app/discover/fetchers/db/db_fetch_vedges_ovs.py | 63 ++++++---- app/discover/link_finders/__init__.py | 10 ++ app/discover/link_finders/find_links_for_pnics.py | 12 ++ .../link_finders/find_links_for_vconnectors.py | 3 +- .../link_finders/find_links_for_vservice_vnics.py | 16 +-- app/discover/scanner.py | 2 +- 27 files changed, 375 insertions(+), 144 deletions(-) delete mode 100644 app/discover/fetcher_new.py create mode 100644 app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py (limited to 'app/discover') diff --git a/app/discover/event_manager.py b/app/discover/event_manager.py index 6a56912..e2f8282 100644 --- a/app/discover/event_manager.py +++ b/app/discover/event_manager.py @@ -40,12 +40,18 @@ class EventManager(Manager): } LISTENERS = { - 'Mirantis-6.0': DefaultListener, - 'Mirantis-7.0': DefaultListener, - 'Mirantis-8.0': DefaultListener, - 'RDO-Mitaka': DefaultListener, - 'RDO-Liberty': DefaultListener, - 'Apex-Euphrates': DefaultListener, + 'Mirantis': { + '6.0': DefaultListener, + '7.0': DefaultListener, + '8.0': DefaultListener, + }, + 'RDO': { + 'Mitaka': DefaultListener, + 'Liberty': DefaultListener, + }, + 'Apex': { + 'Euphrates': DefaultListener, + }, } def __init__(self): @@ -105,7 +111,8 @@ class EventManager(Manager): def get_listener(self, env: str): env_config = self.inv.get_env_config(env) - return self.LISTENERS.get(env_config.get('distribution')) + return (self.LISTENERS.get(env_config.get('distribution'), {}) + .get(env_config.get('distribution_version'))) def listen_to_events(self, listener: ListenerBase, env_name: str, process_vars: dict): listener.listen({ diff --git a/app/discover/events/event_interface_add.py b/app/discover/events/event_interface_add.py index 698559c..e54bedb 100644 --- a/app/discover/events/event_interface_add.py +++ b/app/discover/events/event_interface_add.py @@ -83,11 +83,12 @@ class EventInterfaceAdd(EventBase): def handle(self, env, values): interface = values['payload']['router_interface'] + project_id = values['_context_project_id'] project = values['_context_project_name'] host_id = values["publisher_id"].replace("network.", "", 1) port_id = interface['port_id'] subnet_id = interface['subnet_id'] - router_id = encode_router_id(host_id, interface['id']) + router_id = encode_router_id(interface['id']) network_document = self.inv.get_by_field(env, "network", "subnet_ids", subnet_id, get_single=True) @@ -98,10 +99,10 @@ class EventInterfaceAdd(EventBase): network_id = network_document['id'] # add router-interface port document. - if len(ApiAccess.regions) == 0: + if not ApiAccess.regions: fetcher = ApiFetchRegions() fetcher.set_env(env) - fetcher.get(None) + fetcher.get(project_id) port_doc = EventSubnetAdd().add_port_document(env, port_id, network_name=network_name) diff --git a/app/discover/events/event_interface_delete.py b/app/discover/events/event_interface_delete.py index b1df978..f4ec400 100644 --- a/app/discover/events/event_interface_delete.py +++ b/app/discover/events/event_interface_delete.py @@ -18,8 +18,7 @@ class EventInterfaceDelete(EventDeleteBase): def handle(self, env, values): interface = values['payload']['router_interface'] port_id = interface['port_id'] - host_id = values["publisher_id"].replace("network.", "", 1) - router_id = encode_router_id(host_id, interface['id']) + router_id = encode_router_id(interface['id']) # update router document port_doc = self.inv.get_by_id(env, port_id) diff --git a/app/discover/events/event_router_add.py b/app/discover/events/event_router_add.py index 3c1c9e2..1fb2244 100644 --- a/app/discover/events/event_router_add.py +++ b/app/discover/events/event_router_add.py @@ -96,7 +96,7 @@ class EventRouterAdd(EventBase): router = values['payload']['router'] host_id = values["publisher_id"].replace("network.", "", 1) project_id = values['_context_project_id'] - router_id = encode_router_id(host_id, router['id']) + router_id = encode_router_id(router['id']) host = self.inv.get_by_id(env, host_id) fetcher = CliFetchHostVservice() diff --git a/app/discover/events/event_router_delete.py b/app/discover/events/event_router_delete.py index 65072d6..d0bd645 100644 --- a/app/discover/events/event_router_delete.py +++ b/app/discover/events/event_router_delete.py @@ -21,7 +21,6 @@ class EventRouterDelete(EventDeleteBase): self.log.error("Publisher_id is not in event values. Aborting router delete") return EventResult(result=False, retry=False) - host_id = values['publisher_id'].replace('network.', '', 1) if 'router_id' in payload: router_id = payload['router_id'] elif 'id' in payload: @@ -33,5 +32,5 @@ class EventRouterDelete(EventDeleteBase): self.log.error("Router id is not in payload. Aborting router delete") return EventResult(result=False, retry=False) - router_full_id = encode_router_id(host_id, router_id) + router_full_id = encode_router_id(router_id) return self.delete_handler(env, router_full_id, "vservice") diff --git a/app/discover/events/event_router_update.py b/app/discover/events/event_router_update.py index cfbbf58..b63b224 100644 --- a/app/discover/events/event_router_update.py +++ b/app/discover/events/event_router_update.py @@ -26,7 +26,7 @@ class EventRouterUpdate(EventBase): host_id = values["publisher_id"].replace("network.", "", 1) router_id = payload['id'] if 'id' in payload else router['id'] - router_full_id = encode_router_id(host_id, router_id) + router_full_id = encode_router_id(router_id) router_doc = self.inv.get_by_id(env, router_full_id) if not router_doc: self.log.info("Router document not found, aborting router updating") diff --git a/app/discover/events/event_subnet_add.py b/app/discover/events/event_subnet_add.py index fcae5fd..4126e0c 100644 --- a/app/discover/events/event_subnet_add.py +++ b/app/discover/events/event_subnet_add.py @@ -131,10 +131,10 @@ class EventSubnetAdd(EventBase): # Check DHCP enable, if true, scan network. if subnet['enable_dhcp'] is True: # update network - if len(ApiAccess.regions) == 0: + if not ApiAccess.regions: fetcher = ApiFetchRegions() fetcher.set_env(env) - fetcher.get(None) + fetcher.get(project_id) self.log.info("add new subnet.") host_id = notification["publisher_id"].replace("network.", "", 1) diff --git a/app/discover/events/event_subnet_update.py b/app/discover/events/event_subnet_update.py index 3529f0d..59b0afb 100644 --- a/app/discover/events/event_subnet_update.py +++ b/app/discover/events/event_subnet_update.py @@ -23,6 +23,7 @@ class EventSubnetUpdate(EventBase): def handle(self, env, notification): # check for network document. subnet = notification['payload']['subnet'] + project_id = notification['_context_project_id'] project = notification['_context_project_name'] host_id = notification['publisher_id'].replace('network.', '', 1) subnet_id = subnet['id'] @@ -47,10 +48,10 @@ class EventSubnetUpdate(EventBase): network_document['name']) # make sure that self.regions is not empty. - if len(ApiAccess.regions) == 0: + if not ApiAccess.regions: fetcher = ApiFetchRegions() fetcher.set_env(env) - fetcher.get(None) + fetcher.get(project_id) self.log.info("add port binding to DHCP server.") port_id = DbFetchPort(). \ diff --git a/app/discover/events/listeners/default_listener.py b/app/discover/events/listeners/default_listener.py index 54453a7..273f3e3 100755 --- a/app/discover/events/listeners/default_listener.py +++ b/app/discover/events/listeners/default_listener.py @@ -30,17 +30,19 @@ from monitoring.setup.monitoring_setup_manager import MonitoringSetupManager from utils.constants import OperationalStatus, EnvironmentFeatures from utils.inventory_mgr import InventoryMgr from utils.logging.full_logger import FullLogger +from utils.logging.logger import Logger from utils.mongo_access import MongoAccess -from utils.string_utils import stringify_datetime from utils.util import SignalHandler, setup_args class DefaultListener(ListenerBase, ConsumerMixin): SOURCE_SYSTEM = "OpenStack" - COMMON_METADATA_FILE = "events.json" + LOG_FILENAME = "default_listener.log" + LOG_LEVEL = Logger.INFO + DEFAULTS = { "env": "Mirantis-Liberty", "mongo_config": "", @@ -92,7 +94,7 @@ class DefaultListener(ListenerBase, ConsumerMixin): return False, None def process_task(self, body, message): - received_timestamp = stringify_datetime(datetime.datetime.now()) + received_timestamp = datetime.datetime.now() processable, event_data = self._extract_event_data(body) # If env listener can't process the message # or it's not intended for env listener to handle, @@ -100,7 +102,7 @@ class DefaultListener(ListenerBase, ConsumerMixin): if processable and event_data["event_type"] in self.handler.handlers: event_result = self.handle_event(event_data["event_type"], event_data) - finished_timestamp = stringify_datetime(datetime.datetime.now()) + finished_timestamp = datetime.datetime.now() self.save_message(message_body=event_data, result=event_result, started=received_timestamp, @@ -143,8 +145,8 @@ class DefaultListener(ListenerBase, ConsumerMixin): # 'Retry' flag specifies if the error is recoverable or not # 'Retry' flag is checked only is 'result' is False def handle_event(self, event_type: str, notification: dict) -> EventResult: - print("Got notification.\nEvent_type: {}\nNotification:\n{}". - format(event_type, notification)) + self.log.error("Got notification.\nEvent_type: {}\nNotification:\n{}". + format(event_type, notification)) try: result = self.handler.handle(event_name=event_type, notification=notification) @@ -154,7 +156,7 @@ class DefaultListener(ListenerBase, ConsumerMixin): return EventResult(result=False, retry=False) def save_message(self, message_body: dict, result: EventResult, - started: str, finished: str): + started: datetime, finished: datetime): try: message = Message( msg_id=message_body.get('message_id'), diff --git a/app/discover/events/listeners/listener_base.py b/app/discover/events/listeners/listener_base.py index 7052dc9..4ff4e57 100644 --- a/app/discover/events/listeners/listener_base.py +++ b/app/discover/events/listeners/listener_base.py @@ -7,11 +7,25 @@ # which accompanies this distribution, and is available at # # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### +import os from abc import ABC, abstractmethod +from utils.logging.console_logger import ConsoleLogger +from utils.logging.file_logger import FileLogger +from utils.logging.logger import Logger + class ListenerBase(ABC): + LOG_FILENAME = "listener_base.log" + LOG_LEVEL = Logger.WARNING + + def __init__(self): + super().__init__() + self.log_file = os.path.join(FileLogger.LOG_DIRECTORY, + self.LOG_FILENAME) + self.log = ConsoleLogger(level=Logger.INFO) + @staticmethod @abstractmethod def listen(self): diff --git a/app/discover/fetcher_new.py b/app/discover/fetcher_new.py deleted file mode 100644 index f545554..0000000 --- a/app/discover/fetcher_new.py +++ /dev/null @@ -1,30 +0,0 @@ -############################################################################### -# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # -# and others # -# # -# All rights reserved. This program and the accompanying materials # -# are made available under the terms of the Apache License, Version 2.0 # -# which accompanies this distribution, and is available at # -# http://www.apache.org/licenses/LICENSE-2.0 # -############################################################################### -from discover.fetcher import Fetcher -##old stuff -class FetchHostObjectTypes(Fetcher): - - - def get(self, parent): - ret = { - "type": "host_object_type", - "id": "", - "parent": parent, - "rows": [ - {"id": "instances_root", "text": "Instances", "descendants": 1}, - {"id": "networks_root", "text": "Networks", "descendants": 1}, - {"id": "pnics_root", "text": "pNICs", "descendants": 1}, - {"id": "vservices_root", "text": "vServices", "descendants": 1} - ] - } - return ret - - ## old/moved - diff --git a/app/discover/fetchers/cli/cli_access.py b/app/discover/fetchers/cli/cli_access.py index 275a3e8..c77b22a 100644 --- a/app/discover/fetchers/cli/cli_access.py +++ b/app/discover/fetchers/cli/cli_access.py @@ -12,6 +12,7 @@ import time from discover.fetcher import Fetcher from utils.binary_converter import BinaryConverter +from utils.cli_dist_translator import CliDistTranslator from utils.logging.console_logger import ConsoleLogger from utils.ssh_conn import SshConn @@ -41,11 +42,16 @@ class CliAccess(BinaryConverter, Fetcher): def run(self, cmd, ssh_to_host="", enable_cache=True, on_gateway=False, ssh=None, use_sudo=True): ssh_conn = ssh if ssh else SshConn(ssh_to_host) - if use_sudo and not cmd.strip().startswith("sudo "): - cmd = "sudo " + cmd - if not on_gateway and ssh_to_host \ - and not ssh_conn.is_gateway_host(ssh_to_host): - cmd = self.ssh_cmd + ssh_to_host + " " + cmd + commands = self.adapt_cmd_to_env(ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host) + out = '' + for c in commands: + out += self.run_single_command(c, ssh_conn, ssh_to_host, + enable_cache=enable_cache) + return out + + def run_single_command(self, cmd, ssh_conn, ssh_to_host="", + enable_cache=True): curr_time = time.time() cmd_path = ssh_to_host + ',' + cmd if enable_cache and cmd_path in self.cached_commands: @@ -73,9 +79,44 @@ class CliAccess(BinaryConverter, Fetcher): ret = out.splitlines() # if split by whitespace did not work, try splitting by "\\n" if len(ret) == 1: - ret = [l for l in out.split("\\n") if l != ""] + ret = [line for line in out.split("\\n") if line != ""] return ret + MULTI_COMMAND_SEPARATOR = ';;;' + + @staticmethod + def handle_split_cmd(cmd: str): + if CliAccess.MULTI_COMMAND_SEPARATOR in cmd: + return cmd.split(CliAccess.MULTI_COMMAND_SEPARATOR) + return [cmd] + + def adapt_cmd_to_env(self, ssh_conn, cmd, use_sudo, on_gateway, + ssh_to_host): + cmd = self.adapt_cmd_to_dist(cmd) + commands = self.handle_split_cmd(cmd) + return [self.adapt_cmd_to_environment(c, use_sudo, on_gateway, + ssh_to_host, ssh_conn) + for c in commands] + + def adapt_cmd_to_environment(self, cmd, use_sudo, on_gateway, ssh_to_host, + ssh_conn): + if self.configuration.environment["distribution"] == "Mercury": + use_sudo = False + if use_sudo and not cmd.strip().startswith("sudo "): + cmd = "sudo " + cmd + if not on_gateway and ssh_to_host \ + and not ssh_conn.is_gateway_host(ssh_to_host): + cmd = self.ssh_cmd + ssh_to_host + " " + cmd + return cmd + + def adapt_cmd_to_dist(self, cmd): + env_conf = self.configuration.get_env_config() + dist = env_conf.get('distribution') + dist_version = env_conf.get('distribution_version') + translator = CliDistTranslator(dist, dist_version=dist_version) + cmd = translator.translate(cmd) + return cmd + # parse command output columns separated by whitespace # since headers can contain whitespace themselves, # it is the caller's responsibility to provide the headers @@ -126,7 +167,8 @@ class CliAccess(BinaryConverter, Fetcher): content[headers[i]] = content_parts[i] return content - def merge_ws_spillover_lines(self, lines): + @staticmethod + def merge_ws_spillover_lines(lines): # with WS-separated output, extra output sometimes spills to next line # detect that and add to the end of the previous line for our procesing pending_line = None @@ -156,7 +198,8 @@ class CliAccess(BinaryConverter, Fetcher): - header_regexp: regexp marking the start of the section - end_regexp: regexp marking the end of the section """ - def get_section_lines(self, lines, header_regexp, end_regexp): + @staticmethod + def get_section_lines(lines, header_regexp, end_regexp): if not lines: return [] header_re = re.compile(header_regexp) @@ -196,7 +239,8 @@ class CliAccess(BinaryConverter, Fetcher): if 'name' not in o and 'default' in regexp_tuple: o[name] = regexp_tuple['default'] - def find_matching_regexps(self, o, line, regexps): + @staticmethod + def find_matching_regexps(o, line, regexps): for regexp_tuple in regexps: name = regexp_tuple['name'] regex = regexp_tuple['re'] diff --git a/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py new file mode 100644 index 0000000..77f149f --- /dev/null +++ b/app/discover/fetchers/cli/cli_fetch_bond_host_pnics.py @@ -0,0 +1,134 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from collections import deque + +from discover.fetchers.cli.cli_access import CliAccess +from utils.inventory_mgr import InventoryMgr + + +class CliFetchBondHostPnics(CliAccess): + BOND_DIR = '/proc/net/bonding/' + SLAVE_INTERFACE_HEADER = 'Slave Interface: ' + + def __init__(self): + super().__init__() + self.inv = InventoryMgr() + + def get(self, parent_id: str): + self.log.info('CliFetchBondHostPnics: checking under {}' + .format(parent_id)) + host_id = parent_id[:parent_id.rindex('-')] + cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR) + host = self.inv.get_by_id(self.get_env(), host_id) + if not host: + self.log.error('CliFetchBondHostPnics: host not found: ' + host_id) + return [] + host_types = host['host_type'] + if 'Network' not in host_types and 'Compute' not in host_types: + return [] + lines = self.run_fetch_lines(cmd, host_id) + if lines and 'No such file or directory' in lines[0]: + return [] # no bonds so directory does not exist + bonds = [] + for line in lines: + bond = self.get_bond_details(host_id, line) + if bond: + bonds.append(bond) + return bonds + + def get_bond_details(self, host_id: str, interface_name: str) -> dict: + lines = self.run_fetch_lines('cat {}{}' + .format(self.BOND_DIR, interface_name), + host_id) + status, mac_address = \ + self.get_bond_status_and_mac_address(host_id, interface_name) + interface_id = '{}-{}'.format(interface_name, mac_address) + interface = { + 'host': host_id, + 'name': interface_name, + 'id': interface_id, + 'local_name': interface_name, + 'mac_address': mac_address, + 'Link detected': 'yes' if status == 'up' else 'no', + 'EtherChannel': True, + 'EtherChannel Master': '', + 'members': {} + } + # keep stack of info objects to support multi-level info + info_objects = deque([interface]) + for line in [line for line in lines if line != '']: + if line.startswith(self.SLAVE_INTERFACE_HEADER): + name = line[line.index(':')+1:].strip() + slave = { + 'name': name, + 'EtherChannel Master': interface_id + } + # remove any pending info objects, keep only interface + info_objects = deque([interface]) + info_objects.append(slave) + interface['members'][name] = slave + elif line.rstrip(':').lower().endswith('info'): + # move to lower level info object + info_name = line.rstrip(':') + upper_info_obj = info_objects[-1] + info_obj = {} + upper_info_obj[info_name] = info_obj + info_objects.append(info_obj) + else: + self.get_attribute_from_line(info_objects[-1], line) + for slave in list(interface['members'].values()): + self.set_slave_host_pnic_bond_attributes(host_id, slave, + interface_id) + return interface + + def get_bond_status_and_mac_address(self, host_id: str, name: str): + output = self.run_fetch_lines('ip link show {}'.format(name), host_id) + status_line = output[0] + status = status_line[status_line.index(' state ') + len(' state '):] + status = status[:status.index(' ')] + matches = [line.strip() for line in output if 'link/ether' in line] + if not matches: + self.log.error('Failed to find line with MAC address ' + 'for bond {} (host: {})' + .format(name, host_id)) + tokens = matches[0].split() + if len(tokens) < 2: + self.log.error('Failed to find MAC address in line: {}' + .format(matches[0])) + mac_address = tokens[1] + return status.lower(), mac_address + + def get_attribute_from_line(self, obj: dict, line: str): + if ':' not in line: + self.log.error('object {}: failed to find ":" in line: {}' + .format(obj['name'], line)) + return + attr = line[:line.index(':')] + value = line[len(attr)+1:] + obj[attr.strip()] = value.strip() + + def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id): + pnic = self.inv.find_one({ + 'environment': self.get_env(), + 'host': host, + 'type': 'host_pnic', + 'name': slave['name'] + }) + if not pnic: + self.log.error('unable to find slave pNIC {} under bond {}' + .format(slave_id, interface_id)) + return + mac_address = pnic['mac_address'] + slave_id = '{}-{}'.format(slave.get('name', ''), mac_address) + slave['mac_address'] = mac_address + slave['id'] = slave_id + pnic['EtherChannel'] = True + pnic['EtherChannel Master'] = interface_id + self.inv.set(pnic) diff --git a/app/discover/fetchers/cli/cli_fetch_host_pnics.py b/app/discover/fetchers/cli/cli_fetch_host_pnics.py index 5df4d3b..4af3ebc 100644 --- a/app/discover/fetchers/cli/cli_fetch_host_pnics.py +++ b/app/discover/fetchers/cli/cli_fetch_host_pnics.py @@ -67,21 +67,17 @@ class CliFetchHostPnics(CliAccess): tokens = None if interface is None: tokens = line.split() - name = tokens[0].strip('- :') - name = name.strip() - if name == interface_name: - line_remainder = line.strip('-')[len(interface_name)+2:] - line_remainder = line_remainder.strip(' :') - id = interface_name - interface = { - "host": host_id, - "name": id, - "local_name": interface_name, - "lines": [] - } - self.handle_line(interface, line_remainder) - if '= 25) + DbAccess.query_count_per_con += 1 + cursor = DbAccess.conn.cursor(dictionary=True) + try: + res = method(self, *args, cursor=cursor, **kwargs) + DbAccess.conn.commit() + return res + except: + DbAccess.conn.rollback() + raise + finally: + cursor.close() + return wrap + + class DbAccess(Fetcher): conn = None query_count_per_con = 0 @@ -47,10 +66,9 @@ class DbAccess(Fetcher): return DbAccess.query_count_per_con = 0 - @staticmethod - def get_neutron_db_name(): + @with_cursor + def get_neutron_db_name(self, cursor=None): # check if DB schema 'neutron' exists - cursor = DbAccess.conn.cursor(dictionary=True) cursor.execute('SHOW DATABASES') matches = [row.get('Database', '') for row in cursor if 'neutron' in row.get('Database', '')] @@ -68,6 +86,8 @@ class DbAccess(Fetcher): self.log.info("DbAccess: ****** forcing reconnect, " + "query count: %s ******", DbAccess.query_count_per_con) + DbAccess.conn.commit() + DbAccess.conn.close() DbAccess.conn = None self.conf = self.config.get("mysql") cnf = self.conf @@ -76,16 +96,15 @@ class DbAccess(Fetcher): cnf["user"], cnf["pwd"], cnf["schema"]) - def get_objects_list_for_id(self, query, object_type, id): - self.connect_to_db(DbAccess.query_count_per_con >= 25) - DbAccess.query_count_per_con += 1 + @with_cursor + def get_objects_list_for_id(self, query, object_type, object_id, + cursor=None): self.log.debug("query count: %s, running query:\n%s\n", str(DbAccess.query_count_per_con), query) - cursor = DbAccess.conn.cursor(dictionary=True) try: - if id: - cursor.execute(query, [str(id)]) + if object_id: + cursor.execute(query, [str(object_id)]) else: cursor.execute(query) except (AttributeError, mysql.connector.errors.OperationalError) as e: @@ -93,13 +112,13 @@ class DbAccess(Fetcher): self.connect_to_db(True) # try again to run the query cursor = DbAccess.conn.cursor(dictionary=True) - if id: - cursor.execute(query, [str(id)]) + if object_id: + cursor.execute(query, [str(object_id)]) else: cursor.execute(query) rows = [] - for row in cursor: + for row in cursor.fetchall(): rows.append(row) return rows diff --git a/app/discover/fetchers/db/db_fetch_host_network_agents.py b/app/discover/fetchers/db/db_fetch_host_network_agents.py index c323573..7d415f2 100644 --- a/app/discover/fetchers/db/db_fetch_host_network_agents.py +++ b/app/discover/fetchers/db/db_fetch_host_network_agents.py @@ -27,9 +27,8 @@ class DbFetchHostNetworkAgents(DbAccess): host_id = id[:-1 * len("-network_agents")] results = self.get_objects_list_for_id(query, "network_agent", host_id) mechanism_drivers = self.env_config['mechanism_drivers'] - id_prefix = mechanism_drivers[0] if mechanism_drivers else 'network_agent' for o in results: o["configurations"] = json.loads(o["configurations"]) o["name"] = o["binary"] - o['id'] = id_prefix + '-' + o['id'] + o['id'] = o['name'] + '-' + o['id'] return results diff --git a/app/discover/fetchers/db/db_fetch_oteps.py b/app/discover/fetchers/db/db_fetch_oteps.py index 3e3f4e1..f7eb8bd 100644 --- a/app/discover/fetchers/db/db_fetch_oteps.py +++ b/app/discover/fetchers/db/db_fetch_oteps.py @@ -35,7 +35,9 @@ class DbFetchOteps(DbAccess, CliAccess, metaclass=Singleton): table_name = "{}.ml2_{}_endpoints".format(self.neutron_db, tunnel_type) env_config = self.config.get_env_config() distribution = env_config["distribution"] - if distribution == "Canonical-icehouse": + distribution_version = env_config["distribution_version"] + dist_ver = "{}-{}".format(distribution, distribution_version) + if dist_ver == "Canonical-icehouse": # for Icehouse, we only get IP address from the DB, so take the # host IP address and from the host data in Mongo host = self.inv.get_by_id(self.get_env(), host_id) diff --git a/app/discover/fetchers/db/db_fetch_vedges_ovs.py b/app/discover/fetchers/db/db_fetch_vedges_ovs.py index 838ccb9..f516d10 100644 --- a/app/discover/fetchers/db/db_fetch_vedges_ovs.py +++ b/app/discover/fetchers/db/db_fetch_vedges_ovs.py @@ -24,8 +24,8 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$") self.port_line_header_prefix = " " * 8 + "Port " - def get(self, id): - host_id = id[:id.rindex('-')] + def get(self, parent_id): + host_id = parent_id[:parent_id.rindex('-')] results = self.get_objects_list_for_id( """ SELECT * @@ -66,11 +66,11 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): if not port_matches: continue port = {} - id = port_matches.group(1) + port_id = port_matches.group(1) name = port_matches.group(2) is_internal = port_matches.group(3) == " (internal)" port["internal"] = is_internal - port["id"] = id + port["id"] = port_id port["name"] = name ports[name] = port return ports @@ -106,7 +106,7 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): if "tunneling_ip" not in doc["configurations"]: return {} if not doc["configurations"]["tunneling_ip"]: - self.get_bridge_pnic(doc) + self.get_pnics(doc) return {} # read the 'br-tun' interface ports @@ -148,31 +148,48 @@ class DbFetchVedgesOvs(DbAccess, CliAccess, metaclass=Singleton): tunnel_ports[port["name"]] = port return tunnel_ports - def get_bridge_pnic(self, doc): - conf = doc["configurations"] - if "bridge_mappings" not in conf or not conf["bridge_mappings"]: - return - for v in conf["bridge_mappings"].values(): br = v - ifaces_list_lines = self.run_fetch_lines("ovs-vsctl list-ifaces " + br, - doc["host"]) - br_pnic_postfix = br + "--br-" - interface = "" + def get_pnics(self, vedge) -> dict: + bridges = vedge["configurations"].get("bridge_mappings", {}) + pnics = {} + for bridge in bridges.values(): + self.get_bridge_pnic(pnics, vedge, bridge) + return pnics + + MIRANTIS_DIST = "Mirantis" + + def get_bridge_pnic(self, pnics: dict, vedge: dict, bridge: dict): + cmd = "ovs-vsctl list-ifaces {}".format(bridge) + ifaces_list_lines = self.run_fetch_lines(cmd, vedge["host"]) + env_config = self.configuration.get_env_config() + distribution = env_config.get("distribution") + dist_version = env_config.get("distribution_version") + use_br_postfix = distribution == self.MIRANTIS_DIST and \ + dist_version in ["6.0", "7.0", "8.0"] for l in ifaces_list_lines: - if l.startswith(br_pnic_postfix): - interface = l[len(br_pnic_postfix):] - break - if not interface: - return - doc["pnic"] = interface + if use_br_postfix: + br_pnic_postfix = "{}--br-".format(bridge) + interface = l[len(br_pnic_postfix):] \ + if l.startswith(br_pnic_postfix) \ + else "" + else: + interface = l + if interface: + pnic = self.find_pnic_for_interface(vedge, interface) + if pnic: + pnics[pnic["name"]] = pnic + + def find_pnic_for_interface(self, vedge, interface): # add port ID to pNIC pnic = self.inv.find_items({ "environment": self.get_env(), "type": "host_pnic", - "host": doc["host"], + "host": vedge["host"], "name": interface }, get_single=True) if not pnic: return - port = doc["ports"][interface] - pnic["port_id"] = port["id"] + vedge["pnic"] = interface + port = vedge["ports"].get(interface, {}) + pnic["port_id"] = port.get("id", "") self.inv.set(pnic) + return pnic diff --git a/app/discover/link_finders/__init__.py b/app/discover/link_finders/__init__.py index e69de29..1e85a2a 100644 --- a/app/discover/link_finders/__init__.py +++ b/app/discover/link_finders/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/discover/link_finders/find_links_for_pnics.py b/app/discover/link_finders/find_links_for_pnics.py index 1f02426..94eba7b 100644 --- a/app/discover/link_finders/find_links_for_pnics.py +++ b/app/discover/link_finders/find_links_for_pnics.py @@ -41,6 +41,18 @@ class FindLinksForPnics(FindLinks): def add_pnic_network_links(self, pnic): host = pnic["host"] + if self.configuration.get_env_config()['type_drivers'] == "vlan": + # take this pnic only if we can find matching vedge-pnic links + matches = self.inv.find({ + "environment": self.get_env(), + "link_type": "vedge-host_pnic", + "host": host, + "target_id": pnic["id"]}, + projection={"_id": 1}, + collection="links", + get_single=True) + if not matches: + return # find ports for that host, and fetch just the network ID ports = self.inv.find_items({ "environment": self.get_env(), diff --git a/app/discover/link_finders/find_links_for_vconnectors.py b/app/discover/link_finders/find_links_for_vconnectors.py index edb351a..0703cd8 100644 --- a/app/discover/link_finders/find_links_for_vconnectors.py +++ b/app/discover/link_finders/find_links_for_vconnectors.py @@ -31,7 +31,8 @@ class FindLinksForVconnectors(FindLinks): is_ovs = mechanism_drivers and mechanism_drivers[0] == 'OVS' if is_ovs: # interface ID for OVS - vnic = self.inv.get_by_id(self.get_env(), interface_name) + vnic_id = "{}-{}".format(vconnector["host"], interface_name) + vnic = self.inv.get_by_id(self.get_env(), vnic_id) else: # interface ID for VPP - match interface MAC address to vNIC MAC interface = vconnector['interfaces'][interface_name] diff --git a/app/discover/link_finders/find_links_for_vservice_vnics.py b/app/discover/link_finders/find_links_for_vservice_vnics.py index ca9bc4a..f975c92 100644 --- a/app/discover/link_finders/find_links_for_vservice_vnics.py +++ b/app/discover/link_finders/find_links_for_vservice_vnics.py @@ -33,11 +33,6 @@ class FindLinksForVserviceVnics(FindLinks): host = self.inv.get_by_id(self.get_env(), v["host"]) if "Network" not in host["host_type"]: return - if "network" not in v: - return - network = self.inv.get_by_id(self.get_env(), v["network"]) - if network == []: - return vservice_id = v["parent_id"] vservice_id = vservice_id[:vservice_id.rindex('-')] vservice = self.inv.get_by_id(self.get_env(), vservice_id) @@ -46,7 +41,14 @@ class FindLinksForVserviceVnics(FindLinks): target = v["_id"] target_id = v["id"] link_type = "vservice-vnic" - link_name = network["name"] + extra_attributes = None + if "network" in v: + network = self.inv.get_by_id(self.get_env(), v["network"]) + link_name = network["name"] + extra_attributes = {'network': v['network']} + else: + link_name = "{}-{}".format(vservice["object_name"], + v["object_name"]) state = "up" # TBD link_weight = 0 # TBD self.create_link(self.get_env(), @@ -54,4 +56,4 @@ class FindLinksForVserviceVnics(FindLinks): target, target_id, link_type, link_name, state, link_weight, host=v["host"], - extra_attributes={'network': v['network']}) + extra_attributes=extra_attributes) diff --git a/app/discover/scanner.py b/app/discover/scanner.py index d1323bd..1fbcc68 100644 --- a/app/discover/scanner.py +++ b/app/discover/scanner.py @@ -240,7 +240,7 @@ class Scanner(Fetcher): run_app_path = conf.get('run_app_path', '') if not run_app_path: run_app_path = conf.get('app_path', '/etc/calipso') - return run_app_path + return run_app_path def load_scanners_metadata(self): parser = ScanMetadataParser(self.inv) -- cgit 1.2.3-korg