From 648a394f7a318443dfd82f790f83a79616c26905 Mon Sep 17 00:00:00 2001 From: yayogev Date: Tue, 27 Feb 2018 17:00:05 +0200 Subject: US3541 merge various fixes to OPNFV branch timestamp of last commit tt was merged: 26-Jan-2018 16:25. Change-Id: I7b0bf7885d7d0badb81c794a52c480b905d78459 Signed-off-by: yayogev --- app/utils/dict_naming_converter.py | 35 ++++++++++----- app/utils/inventory_mgr.py | 71 ++++++++++++++++-------------- app/utils/logging/console_logger.py | 1 - app/utils/logging/full_logger.py | 44 ++++++++++++++---- app/utils/logging/logger.py | 7 +-- app/utils/logging/message_logger.py | 14 ++++++ app/utils/logging/mongo_logging_handler.py | 20 ++++++++- 7 files changed, 135 insertions(+), 57 deletions(-) (limited to 'app/utils') diff --git a/app/utils/dict_naming_converter.py b/app/utils/dict_naming_converter.py index 91fea2e..d0f8d42 100644 --- a/app/utils/dict_naming_converter.py +++ b/app/utils/dict_naming_converter.py @@ -8,6 +8,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################### from bson.objectid import ObjectId +from datetime import datetime class DictNamingConverter: @@ -20,21 +21,33 @@ class DictNamingConverter: # Returns: # Dictionary with the new keys. @staticmethod - def change_dict_naming_convention(d, cf): + def change_dict_naming_convention(d, cf, level: int=0): new = {} + change_convention = DictNamingConverter.change_dict_naming_convention if not d: return d - if isinstance(d, str): + if isinstance(d, str) or isinstance(d, int) or isinstance(d, float) \ + or isinstance(d, bool) or isinstance(d, datetime): return d if isinstance(d, ObjectId): return d - for k, v in d.items(): - new_v = v - if isinstance(v, dict): - new_v = DictNamingConverter.change_dict_naming_convention(v, cf) - elif isinstance(v, list): - new_v = list() - for x in v: - new_v.append(DictNamingConverter.change_dict_naming_convention(x, cf)) - new[cf(k)] = new_v + if isinstance(d, object) and not isinstance(d, dict): + for k in dir(d): + if k.startswith('_'): + continue + v = getattr(d, k) + if callable(v): + continue + new[cf(k)] = change_convention(v, cf, level+1) + if isinstance(d, dict): + for k, v in d.items(): + new_v = v + if isinstance(v, dict): + new_v = change_convention(v, cf, level+1) + elif isinstance(v, list): + new_v = list() + for x in v: + list_val = change_convention(x, cf, level+1) + new_v.append(list_val) + new[cf(k)] = new_v return new diff --git a/app/utils/inventory_mgr.py b/app/utils/inventory_mgr.py index bbc5542..97b6cd4 100644 --- a/app/utils/inventory_mgr.py +++ b/app/utils/inventory_mgr.py @@ -389,38 +389,9 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): parent_id_path = parent.get("id_path", "/{}".format(environment)) parent_name_path = parent.get("name_path", "/{}".format(environment)) - try: - # case of dynamic folder added by need - master_parent_type = o["master_parent_type"] - master_parent_id = o["master_parent_id"] - master_parent = self.get_by_id(environment, master_parent_id) - if not master_parent: - self.log.error("failed to find master parent " + - master_parent_id) + if 'master_parent_type' in o: + if not self.create_parent_folder(o, parent): return False - folder_id_path = "/".join((master_parent["id_path"], - o["parent_id"])) - folder_name_path = "/".join((master_parent["name_path"], - o["parent_text"])) - folder = { - "environment": parent["environment"], - "parent_id": master_parent_id, - "parent_type": master_parent_type, - "id": o["parent_id"], - "id_path": folder_id_path, - "show_in_tree": True, - "name_path": folder_name_path, - "name": o["parent_id"], - "type": o["parent_type"], - "text": o["parent_text"] - } - # remove master_parent_type & master_parent_id after use, - # as they're there just ro help create the dynamic folder - o.pop("master_parent_type", True) - o.pop("master_parent_id", True) - self.set(folder) - except KeyError: - pass if o.get("text"): o["name"] = o["text"] @@ -459,6 +430,42 @@ class InventoryMgr(MongoAccess, metaclass=Singleton): if "create_object" not in o or o["create_object"]: # add/update object in DB self.set(o) - if self.is_feature_supported(environment, EnvironmentFeatures.MONITORING): + if self.is_feature_supported(environment, + EnvironmentFeatures.MONITORING): self.monitoring_setup_manager.create_setup(o) return True + + def create_parent_folder(self, o, parent) -> bool: + # case of dynamic folder added by need + master_parent_type = o["master_parent_type"] + master_parent_id = o["master_parent_id"] + env_path = '/{}'.format(parent['environment']) + master_parent = {'id_path': env_path, 'name_path': env_path} \ + if master_parent_type == 'environment' \ + else self.get_by_id(o['environment'], master_parent_id) + if not master_parent: + self.log.error("failed to find master parent " + + master_parent_id) + return False + folder_id_path = "/".join((master_parent['id_path'], + o["parent_id"])) + folder_name_path = "/".join((master_parent["name_path"], + o["parent_text"])) + folder = { + "environment": parent["environment"], + "parent_id": master_parent_id, + "parent_type": master_parent_type, + "id": o["parent_id"], + "id_path": folder_id_path, + "show_in_tree": True, + "name_path": folder_name_path, + "name": o["parent_id"], + "type": o["parent_type"], + "text": o["parent_text"] + } + # remove master_parent_type & master_parent_id after use, + # as they're there just ro help create the dynamic folder + o.pop("master_parent_type", True) + o.pop("master_parent_id", True) + self.set(folder) + return True diff --git a/app/utils/logging/console_logger.py b/app/utils/logging/console_logger.py index bb8b2ed..b1008e4 100644 --- a/app/utils/logging/console_logger.py +++ b/app/utils/logging/console_logger.py @@ -18,4 +18,3 @@ class ConsoleLogger(Logger): super().__init__(logger_name="{}-Console".format(self.PROJECT_NAME), level=level) self.add_handler(logging.StreamHandler()) - diff --git a/app/utils/logging/full_logger.py b/app/utils/logging/full_logger.py index 411eceb..f6fe5fa 100644 --- a/app/utils/logging/full_logger.py +++ b/app/utils/logging/full_logger.py @@ -10,34 +10,62 @@ import logging import logging.handlers +from utils.origins import Origin from utils.logging.logger import Logger from utils.logging.mongo_logging_handler import MongoLoggingHandler class FullLogger(Logger): - def __init__(self, env: str = None, log_file: str = None, - level: str = Logger.default_level): + def __init__(self, env: str = None, origin: Origin = None, + log_file: str = None, level: str = Logger.default_level): super().__init__(logger_name="{}-Full".format(self.PROJECT_NAME), level=level) + self.env = env + self.origin = origin # Console handler self.add_handler(logging.StreamHandler()) # Message handler - self.add_handler(MongoLoggingHandler(env, self.level)) + self.add_handler(MongoLoggingHandler(env=env, origin=origin, + level=self.level)) # File handler if log_file: self.add_handler(logging.handlers.WatchedFileHandler(log_file)) + def _get_message_handler(self): + defined_handlers = [h for h in self.log.handlers + if isinstance(h, MongoLoggingHandler)] + return defined_handlers[0] if defined_handlers else None + # Make sure we update MessageHandler with new env def set_env(self, env): - super().set_env(env) + self.env = env - defined_handler = [h for h in self.log.handlers - if isinstance(h, MongoLoggingHandler)] - if defined_handler: - defined_handler[0].env = env + handler = self._get_message_handler() + if handler: + handler.env = env else: self.add_handler(MongoLoggingHandler(env, self.level)) + + def set_origin(self, origin: Origin): + self.origin = origin + + handler = self._get_message_handler() + if handler: + handler.origin = origin + else: + self.add_handler(MongoLoggingHandler(env=self.env, + level=self.level, + origin=origin)) + + def setup(self, **kwargs): + env = kwargs.get('env') + if env and self.env != env: + self.set_env(env) + + origin = kwargs.get('origin') + if origin and self.origin != origin: + self.set_origin(origin) \ No newline at end of file diff --git a/app/utils/logging/logger.py b/app/utils/logging/logger.py index 316d3fd..9628040 100644 --- a/app/utils/logging/logger.py +++ b/app/utils/logging/logger.py @@ -34,11 +34,12 @@ class Logger(ABC): level=level) self.log.propagate = False self.set_loglevel(level) - self.env = None self.level = level - def set_env(self, env): - self.env = env + # Subclasses should override this method + # to perform runtime changes to handlers, etc. + def setup(self, **kwargs): + pass @staticmethod def check_level(level): diff --git a/app/utils/logging/message_logger.py b/app/utils/logging/message_logger.py index 02e098f..d433a0f 100644 --- a/app/utils/logging/message_logger.py +++ b/app/utils/logging/message_logger.py @@ -18,4 +18,18 @@ class MessageLogger(Logger): def __init__(self, env: str = None, level: str = None): super().__init__(logger_name="{}-Message".format(self.PROJECT_NAME), level=level) + self.env = env self.add_handler(MongoLoggingHandler(env, self.level)) + + def set_env(self, env): + self.env = env + + if self.log.handlers: + self.log.handlers[0].env = env + else: + self.add_handler(MongoLoggingHandler(env, self.level)) + + def setup(self, **kwargs): + env = kwargs.get('env') + if env and self.env != env: + self.set_env(env) diff --git a/app/utils/logging/mongo_logging_handler.py b/app/utils/logging/mongo_logging_handler.py index ffb6f85..3929e02 100644 --- a/app/utils/logging/mongo_logging_handler.py +++ b/app/utils/logging/mongo_logging_handler.py @@ -11,9 +11,9 @@ import datetime import logging from messages.message import Message +from utils.origins import Origin from utils.inventory_mgr import InventoryMgr from utils.logging.logger import Logger -from utils.string_utils import stringify_datetime class MongoLoggingHandler(logging.Handler): @@ -22,11 +22,12 @@ class MongoLoggingHandler(logging.Handler): """ SOURCE_SYSTEM = 'Calipso' - def __init__(self, env: str, level: str): + def __init__(self, env: str, level: str, origin: Origin = None): super().__init__(Logger.get_numeric_level(level)) self.str_level = level self.env = env self.inv = None + self.origin = origin def emit(self, record): # Try to invoke InventoryMgr for logging @@ -46,7 +47,22 @@ class MongoLoggingHandler(logging.Handler): d = now - datetime.datetime(1970, 1, 1) timestamp_id = '{}.{}.{}'.format(d.days, d.seconds, d.microseconds) source = self.SOURCE_SYSTEM + message = Message(msg_id=timestamp_id, env=self.env, source=source, msg=Logger.formatter.format(record), ts=now, level=record.levelname) + if self.origin: + message.extra['origin_id'] = ( + str(self.origin.origin_id) + if self.origin.origin_id + else None + ) + message.extra['origin_type'] = ( + self.origin.origin_type.value + if self.origin.origin_type + else None + ) + for extra_field in self.origin.extra: + message.extra[extra_field] = getattr(self.origin, extra_field) + self.inv.collections['messages'].insert_one(message.get()) \ No newline at end of file -- cgit 1.2.3-korg