From 7e83d0876ddb84a45e130eeba28bc40ef53c074b Mon Sep 17 00:00:00 2001 From: Yaron Yogev Date: Thu, 27 Jul 2017 09:02:54 +0300 Subject: Calipso initial release for OPNFV Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88 Signed-off-by: Yaron Yogev --- app/utils/__init__.py | 10 + app/utils/binary_converter.py | 27 ++ app/utils/config_file.py | 48 ++++ app/utils/constants.py | 37 +++ app/utils/deep_merge.py | 77 +++++ app/utils/dict_naming_converter.py | 40 +++ app/utils/exceptions.py | 13 + app/utils/inventory_mgr.py | 445 +++++++++++++++++++++++++++++ app/utils/logging/__init__.py | 10 + app/utils/logging/console_logger.py | 21 ++ app/utils/logging/file_logger.py | 23 ++ app/utils/logging/full_logger.py | 47 +++ app/utils/logging/logger.py | 99 +++++++ app/utils/logging/message_logger.py | 21 ++ app/utils/logging/mongo_logging_handler.py | 53 ++++ app/utils/metadata_parser.py | 83 ++++++ app/utils/mongo_access.py | 137 +++++++++ app/utils/singleton.py | 16 ++ app/utils/special_char_converter.py | 32 +++ app/utils/ssh_conn.py | 94 ++++++ app/utils/ssh_connection.py | 217 ++++++++++++++ app/utils/string_utils.py | 59 ++++ app/utils/util.py | 172 +++++++++++ 23 files changed, 1781 insertions(+) create mode 100644 app/utils/__init__.py create mode 100644 app/utils/binary_converter.py create mode 100644 app/utils/config_file.py create mode 100644 app/utils/constants.py create mode 100644 app/utils/deep_merge.py create mode 100644 app/utils/dict_naming_converter.py create mode 100644 app/utils/exceptions.py create mode 100644 app/utils/inventory_mgr.py create mode 100644 app/utils/logging/__init__.py create mode 100644 app/utils/logging/console_logger.py create mode 100644 app/utils/logging/file_logger.py create mode 100644 app/utils/logging/full_logger.py create mode 100644 app/utils/logging/logger.py create mode 100644 app/utils/logging/message_logger.py create mode 100644 app/utils/logging/mongo_logging_handler.py create mode 100644 app/utils/metadata_parser.py create mode 100644 app/utils/mongo_access.py create mode 100644 app/utils/singleton.py create mode 100644 app/utils/special_char_converter.py create mode 100644 app/utils/ssh_conn.py create mode 100644 app/utils/ssh_connection.py create mode 100644 app/utils/string_utils.py create mode 100644 app/utils/util.py (limited to 'app/utils') diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/utils/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/utils/binary_converter.py b/app/utils/binary_converter.py new file mode 100644 index 0000000..70d4e40 --- /dev/null +++ b/app/utils/binary_converter.py @@ -0,0 +1,27 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from utils.logging.console_logger import ConsoleLogger + + +class BinaryConverter: + + def __init__(self): + super().__init__() + self.log = ConsoleLogger() + + def binary2str(self, txt): + if not isinstance(txt, bytes): + return str(txt) + try: + s = txt.decode("utf-8") + except TypeError: + s = str(txt) + return s + diff --git a/app/utils/config_file.py b/app/utils/config_file.py new file mode 100644 index 0000000..1982acc --- /dev/null +++ b/app/utils/config_file.py @@ -0,0 +1,48 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os + + +class ConfigFile: + + def __init__(self, file_path): + super().__init__() + if not os.path.isfile(file_path): + raise ValueError("config file doesn't exist in " + "the system: {0}" + .format(file_path)) + self.config_file = file_path + + def read_config(self): + params = {} + try: + with open(self.config_file) as f: + for line in f: + line = line.strip() + if line.startswith("#") or " " not in line: + continue + index = line.index(" ") + key = line[: index].strip() + value = line[index + 1:].strip() + if value: + params[key] = value + except Exception as e: + raise e + return params + + @staticmethod + def get(file_name): + # config file is taken from app/config by default + # look in the current work directory to get the + # config path + python_path = os.environ['PYTHONPATH'] + if os.pathsep in python_path: + python_path = python_path.split(os.pathsep)[0] + return python_path + '/config/' + file_name diff --git a/app/utils/constants.py b/app/utils/constants.py new file mode 100644 index 0000000..7aa0343 --- /dev/null +++ b/app/utils/constants.py @@ -0,0 +1,37 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from enum import Enum + + +class StringEnum(Enum): + def __str__(self): + return str(self.value) + + def __repr__(self): + return repr(self.value) + + +class ScanStatus(StringEnum): + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +class OperationalStatus(StringEnum): + STOPPED = "stopped" + RUNNING = "running" + ERROR = "error" + + +class EnvironmentFeatures(StringEnum): + SCANNING = "scanning" + MONITORING = "monitoring" + LISTENING = "listening" diff --git a/app/utils/deep_merge.py b/app/utils/deep_merge.py new file mode 100644 index 0000000..acb54ff --- /dev/null +++ b/app/utils/deep_merge.py @@ -0,0 +1,77 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +""" +Do a deep merge of dictionariesi, +recursively merging dictionaries with boltons.iterutils.remap + +Taken from: +https://gist.github.com/mahmoud/db02d16ac89fa401b968 + + +This is an extension of the technique first detailed here: +http://sedimental.org/remap.html#add_common_keys +In short, it calls remap on each container, back to front, +using the accumulating previous values as the default for +the current iteration. +""" + +from boltons.iterutils import remap, get_path, default_enter, default_visit + + +def remerge(target_list, sourced=False): + """ + Takes a list of containers (e.g., dicts) and merges them using + boltons.iterutils.remap. Containers later in the list take + precedence (last-wins). + By default, returns a new, merged top-level container. With the + *sourced* option, `remerge` expects a list of (*name*, container*) + pairs, and will return a source map: a dictionary mapping between + path and the name of the container it came from. + """ + + if not sourced: + target_list = [(id(t), t) for t in target_list] + + ret = None + source_map = {} + + def remerge_enter(path, key, value): + new_parent, new_items = default_enter(path, key, value) + if ret and not path and key is None: + new_parent = ret + try: + cur_val = get_path(ret, path + (key,)) + except KeyError: + pass + else: + # TODO: type check? + new_parent = cur_val + + if isinstance(value, list): + # lists are purely additive. + # See https://github.com/mahmoud/boltons/issues/81 + new_parent.extend(value) + new_items = [] + + return new_parent, new_items + + for t_name, target in target_list: + if sourced: + def remerge_visit(path, key, value): + source_map[path + (key,)] = t_name + return True + else: + remerge_visit = default_visit + + ret = remap(target, enter=remerge_enter, visit=remerge_visit) + + if not sourced: + return ret + return ret, source_map diff --git a/app/utils/dict_naming_converter.py b/app/utils/dict_naming_converter.py new file mode 100644 index 0000000..91fea2e --- /dev/null +++ b/app/utils/dict_naming_converter.py @@ -0,0 +1,40 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from bson.objectid import ObjectId + + +class DictNamingConverter: + + # Convert a nested dictionary from one convention to another. + # Args: + # d (dict): dictionary (nested or not) to be converted. + # cf (func): convert function - takes the string in one convention, + # returns it in the other one. + # Returns: + # Dictionary with the new keys. + @staticmethod + def change_dict_naming_convention(d, cf): + new = {} + if not d: + return d + if isinstance(d, str): + return d + if isinstance(d, ObjectId): + return d + for k, v in d.items(): + new_v = v + if isinstance(v, dict): + new_v = DictNamingConverter.change_dict_naming_convention(v, cf) + elif isinstance(v, list): + new_v = list() + for x in v: + new_v.append(DictNamingConverter.change_dict_naming_convention(x, cf)) + new[cf(k)] = new_v + return new diff --git a/app/utils/exceptions.py b/app/utils/exceptions.py new file mode 100644 index 0000000..07e46dc --- /dev/null +++ b/app/utils/exceptions.py @@ -0,0 +1,13 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + + +class ScanArgumentsError(ValueError): + pass diff --git a/app/utils/inventory_mgr.py b/app/utils/inventory_mgr.py new file mode 100644 index 0000000..2fe2894 --- /dev/null +++ b/app/utils/inventory_mgr.py @@ -0,0 +1,445 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from datetime import datetime + +import bson + +from utils.constants import EnvironmentFeatures +from utils.logging.console_logger import ConsoleLogger +from utils.mongo_access import MongoAccess +from utils.singleton import Singleton + + +def inv_initialization_required(func): + def decorated(self, *args, **kwargs): + if self.inventory_collection is None: + raise TypeError("Inventory collection is not set.") + return func(self, *args, **kwargs) + return decorated + + +class InventoryMgr(MongoAccess, metaclass=Singleton): + + def __init__(self): + super().__init__() + self.log = ConsoleLogger() + self.inventory_collection = None + self.inventory_collection_name = None + self.collections = {} + self.monitoring_setup_manager = None + + def set_collection(self, collection_type: str = None, + use_default_name: bool = False): + # do not allow setting the collection more than once + if not self.collections.get(collection_type): + collection_name = collection_type \ + if use_default_name \ + else self.get_coll_name(collection_type) + + self.log.info("Using {} collection: {}" + .format(collection_type, collection_name)) + + self.collections[collection_type] = MongoAccess.db[collection_name] + + def set_inventory_collection(self, collection_name: str = None): + if not self.inventory_collection: + if not collection_name: + collection_name = "inventory" + + self.log.info("Using inventory collection: {}" + .format(collection_name)) + + collection = MongoAccess.db[collection_name] + self.collections["inventory"] = collection + self.inventory_collection = collection + self.inventory_collection_name = collection_name + + def get_coll_name(self, coll_name): + if not self.inventory_collection_name: + raise TypeError("inventory_collection_name is not set") + + return self.inventory_collection_name.replace("inventory", coll_name) \ + if self.inventory_collection_name.startswith("inventory") \ + else self.inventory_collection_name + "_" + coll_name + + def set_collections(self, inventory_collection: str = None): + self.set_inventory_collection(inventory_collection) + self.set_collection("links") + self.set_collection("link_types") + self.set_collection("clique_types") + self.set_collection("clique_constraints") + self.set_collection("cliques") + self.set_collection("monitoring_config") + self.set_collection("constants", use_default_name=True) + self.set_collection("scans") + self.set_collection("messages") + self.set_collection("monitoring_config_templates", + use_default_name=True) + self.set_collection("environments_config") + self.set_collection("supported_environments") + + def clear(self, scan_plan): + if scan_plan.inventory_only: + collections = {"inventory"} + elif scan_plan.links_only: + collections = {"links"} + elif scan_plan.cliques_only: + collections = {"cliques"} + else: + collections = {"inventory", "links", "cliques", "monitoring_config"} + + env_cond = {} if scan_plan.clear_all else {"environment": scan_plan.env} + + for collection_name in collections: + collection = self.collections[collection_name] + self.log.info("clearing collection: " + collection.full_name) + # delete docs from the collection, + # either all or just for the specified environment + collection.delete_many(env_cond) + + # return single match + def get_by_id(self, environment, item_id): + return self.find({ + "environment": environment, + "id": item_id + }, get_single=True) + + # return matches for ID in list of values + def get_by_ids(self, environment, ids_list): + return self.find({ + "environment": environment, + "id": {"$in": ids_list} + }) + + def get_by_field(self, environment, item_type, field_name, field_value, + get_single=False): + if field_value: + return self.find({"environment": environment, + "type": item_type, + field_name: field_value}, + get_single=get_single) + else: + return self.find({"environment": environment, + "type": item_type}, + get_single=get_single) + + def get(self, environment, item_type, item_id, get_single=False): + return self.get_by_field(environment, item_type, "id", item_id, + get_single=get_single) + + def get_children(self, environment, item_type, parent_id): + if parent_id: + if not item_type: + return self.find({"environment": environment, + "parent_id": parent_id}) + else: + return self.find({"environment": environment, + "type": item_type, + "parent_id": parent_id}) + else: + return self.find({"environment": environment, + "type": item_type}) + + def get_single(self, environment, item_type, item_id): + matches = self.find({"environment": environment, + "type": item_type, + "id": item_id}) + if len(matches) > 1: + raise ValueError("Found multiple matches for item: " + + "type=" + item_type + ", id=" + item_id) + if len(matches) == 0: + raise ValueError("No matches for item: " + + "type=" + item_type + ", id=" + item_id) + return matches[0] + + # item must contain properties 'environment', 'type' and 'id' + def set(self, item, collection=None): + col = collection + mongo_id = None + projects = None + if "_id" in item: + mongo_id = item.pop("_id", None) + + if not collection or collection == self.collections['inventory']: + # make sure we have environment, type & id + self.check(item, "environment") + self.check(item, "type") + self.check(item, "id") + + item["last_scanned"] = datetime.now() + item.pop("projects", []) + + obj_name = item["name_path"] + obj_name = obj_name[obj_name.rindex('/') + 1:] + + if 'object_name' not in item: + item['object_name'] = obj_name + + self.set_collections() # make sure we have all collections set + if not col: + col = self.collections['inventory'] + + find_tuple = {"environment": item["environment"], + "type": item["type"], "id": item["id"]} + else: + find_tuple = {'_id': bson.ObjectId(mongo_id)} + doc = col.find_one(find_tuple) + if not doc: + raise ValueError('set(): could not find document with _id=' + + mongo_id) + + col.update_one(find_tuple, + {'$set': self.encode_mongo_keys(item)}, + upsert=True) + if mongo_id: + # restore original mongo ID of document, in case we need to use it + item['_id'] = mongo_id + if projects: + col.update_one(find_tuple, + {'$addToSet': {"projects": {'$each': projects}}}, + upsert=True) + + @staticmethod + def check(obj, field_name): + arg = obj[field_name] + if not arg or not str(arg).rstrip(): + raise ValueError("Inventory item - " + + "the following field is not defined: " + + field_name) + + # note: to use general find, call find_items(), + # which also does process_results + @inv_initialization_required + def find(self, search, projection=None, collection=None, get_single=False): + coll = self.inventory_collection if not collection \ + else self.collections[collection] + if get_single is True: + return self.decode_object_id( + self.decode_mongo_keys( + coll.find_one(search, projection=projection) + ) + ) + else: + return list( + map( + self.decode_object_id, + map( + self.decode_mongo_keys, + coll.find(search, projection=projection)) + ) + ) + + def find_one(self, search, projection=None, collection=None) -> dict: + return self.find(search, projection, collection, True) + + def find_items(self, search, + projection=None, + get_single=False, + collection=None): + return self.find(search, projection, collection, get_single) + + # record a link between objects in the inventory, to be used in graphs + # returns - the new link document + # parameters - + # environment: name of environment + # host: name of host + # source: node mongo _id + # source_id: node id value of source node + # target: node mongo _id + # target_id: node id value of target node + # link_type: string showing types of connected objects, e.g. "instance-vnic" + # link_name: label for the link itself + # state: up/down + # link_weight: integer, position/priority for graph placement + # source_label, target_label: labels for the ends of the link (optional) + def create_link(self, env, host, src, source_id, target, target_id, + link_type, link_name, state, link_weight, + source_label="", target_label="", + extra_attributes=None): + s = bson.ObjectId(src) + t = bson.ObjectId(target) + link = { + "environment": env, + "host": host, + "source": s, + "source_id": source_id, + "target": t, + "target_id": target_id, + "link_type": link_type, + "link_name": link_name, + "state": state, + "link_weight": link_weight, + "source_label": source_label, + "target_label": target_label, + "attributes": extra_attributes if extra_attributes else {} + } + return self.write_link(link) + + def write_link(self, link): + find_tuple = { + 'environment': link['environment'], + 'source_id': link['source_id'], + 'target_id': link['target_id'] + } + if "_id" in link: + link.pop("_id", None) + link_encoded = self.encode_mongo_keys(link) + links_col = self.collections["links"] + result = links_col.update_one(find_tuple, {'$set': link_encoded}, + upsert=True) + link['_id'] = result.upserted_id + return link + + def values_replace_in_object(self, o, values_replacement): + for k in values_replacement.keys(): + if k not in o: + continue + repl = values_replacement[k] + if 'from' not in repl or 'to' not in repl: + continue + o[k] = o[k].replace(repl['from'], repl['to']) + self.set(o) + + # perform replacement of substring in values of objects in the inventory + # input: + # - search: dict with search parameters + # - values_replacement: dict, + # - keys: names of keys for which to replace the values + # - values: dict with "from" (value to be replaced) and "to" (new value) + @inv_initialization_required + def values_replace(self, search, values_replacement): + for doc in self.inventory_collection.find(search): + self.values_replace_in_object(doc, values_replacement) + + def delete(self, coll, query_filter): + collection = self.collections[coll] + if not collection: + self.log.warn('delete(): collection not found - ' + coll) + return + result = collection.delete_many(query_filter) + count = result.deleted_count + self.log.info('delete(): ' + ('deleted ' + str(count) + ' documents' + if count else 'no matching documents')) + return count + + def get_env_config(self, env: str): + return self.find_one(search={'name': env}, + collection='environments_config') + + def is_feature_supported(self, env: str, feature: EnvironmentFeatures)\ + -> bool: + env_config = self.get_env_config(env) + if not env_config: + return False + + # Workaround for mechanism_drivers field type + mechanism_driver = env_config['mechanism_drivers'][0] \ + if isinstance(env_config['mechanism_drivers'], list) \ + else env_config['mechanism_drivers'] + + full_env = {'environment.distribution': env_config['distribution'], + 'environment.type_drivers': env_config['type_drivers'], + 'environment.mechanism_drivers': mechanism_driver} + return self.is_feature_supported_in_env(full_env, feature) + + def is_feature_supported_in_env(self, env_def: dict, + feature: EnvironmentFeatures) -> bool: + + result = self.collections['supported_environments'].find_one(env_def) + if not result: + return False + features_in_env = result.get('features', {}) + return features_in_env.get(feature.value) is True + + def save_inventory_object(self, o: dict, parent: dict, + environment: str, type_to_fetch: dict = None) -> bool: + if not type_to_fetch: + type_to_fetch = {} + + o["id"] = str(o["id"]) + o["environment"] = environment + if type_to_fetch.get("type"): + o["type"] = type_to_fetch["type"] + o["show_in_tree"] = type_to_fetch.get("show_in_tree", True) + + parent_id_path = parent.get("id_path", "/{}".format(environment)) + parent_name_path = parent.get("name_path", "/{}".format(environment)) + + try: + # case of dynamic folder added by need + master_parent_type = o["master_parent_type"] + master_parent_id = o["master_parent_id"] + master_parent = self.get_by_id(environment, master_parent_id) + if not master_parent: + self.log.error("failed to find master parent " + + master_parent_id) + return False + folder_id_path = "/".join((master_parent["id_path"], o["parent_id"])) + folder_name_path = "/".join((master_parent["name_path"], o["parent_text"])) + folder = { + "environment": parent["environment"], + "parent_id": master_parent_id, + "parent_type": master_parent_type, + "id": o["parent_id"], + "id_path": folder_id_path, + "show_in_tree": True, + "name_path": folder_name_path, + "name": o["parent_id"], + "type": o["parent_type"], + "text": o["parent_text"] + } + # remove master_parent_type & master_parent_id after use, + # as they're there just ro help create the dynamic folder + o.pop("master_parent_type", True) + o.pop("master_parent_id", True) + self.set(folder) + except KeyError: + pass + + if o.get("text"): + o["name"] = o["text"] + elif not o.get("name"): + o["name"] = o["id"] + + if "parent_id" not in o and parent: + parent_id = parent["id"] + o["parent_id"] = parent_id + o["parent_type"] = parent["type"] + elif "parent_id" in o and o["parent_id"] != parent["id"]: + # using alternate parent - fetch parent path from inventory + parent_obj = self.get_by_id(environment, o["parent_id"]) + if parent_obj: + parent_id_path = parent_obj["id_path"] + parent_name_path = parent_obj["name_path"] + o["id_path"] = "/".join((parent_id_path, o["id"].strip())) + o["name_path"] = "/".join((parent_name_path, o["name"])) + + # keep list of projects that an object is in + associated_projects = [] + keys_to_remove = [] + for k in o: + if k.startswith("in_project-"): + proj_name = k[k.index('-') + 1:] + associated_projects.append(proj_name) + keys_to_remove.append(k) + for k in keys_to_remove: + o.pop(k) + if len(associated_projects) > 0: + projects = o["projects"] if "projects" in o.keys() else [] + projects.extend(associated_projects) + if projects: + o["projects"] = projects + + if "create_object" not in o or o["create_object"]: + # add/update object in DB + self.set(o) + if self.is_feature_supported(environment, EnvironmentFeatures.MONITORING): + self.monitoring_setup_manager.create_setup(o) + return True diff --git a/app/utils/logging/__init__.py b/app/utils/logging/__init__.py new file mode 100644 index 0000000..1e85a2a --- /dev/null +++ b/app/utils/logging/__init__.py @@ -0,0 +1,10 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### + diff --git a/app/utils/logging/console_logger.py b/app/utils/logging/console_logger.py new file mode 100644 index 0000000..bb8b2ed --- /dev/null +++ b/app/utils/logging/console_logger.py @@ -0,0 +1,21 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import logging + +from utils.logging.logger import Logger + + +class ConsoleLogger(Logger): + + def __init__(self, level: str = Logger.default_level): + super().__init__(logger_name="{}-Console".format(self.PROJECT_NAME), + level=level) + self.add_handler(logging.StreamHandler()) + diff --git a/app/utils/logging/file_logger.py b/app/utils/logging/file_logger.py new file mode 100644 index 0000000..e205bc3 --- /dev/null +++ b/app/utils/logging/file_logger.py @@ -0,0 +1,23 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import logging.handlers + +from utils.logging.logger import Logger + + +class FileLogger(Logger): + + LOG_DIRECTORY = "/local_dir/log/calipso/" + + def __init__(self, log_file: str, level: str = Logger.default_level): + super().__init__(logger_name="{}-File".format(self.PROJECT_NAME), + level=level) + self.add_handler(logging.handlers.WatchedFileHandler(log_file)) + diff --git a/app/utils/logging/full_logger.py b/app/utils/logging/full_logger.py new file mode 100644 index 0000000..a88f00e --- /dev/null +++ b/app/utils/logging/full_logger.py @@ -0,0 +1,47 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import logging +import logging.handlers + +from utils.logging.logger import Logger +from utils.logging.mongo_logging_handler import MongoLoggingHandler + + +class FullLogger(Logger): + + def __init__(self, env: str = None, log_file: str = None, + level: str = Logger.default_level): + super().__init__(logger_name="{}-Full".format(self.PROJECT_NAME), + level=level) + + # Console handler + self.add_handler(logging.StreamHandler()) + + # Message handler + self.add_handler(MongoLoggingHandler(env, self.level)) + + # File handler + if log_file: + self.add_handler(logging.handlers.WatchedFileHandler(log_file)) + + # Make sure we update MessageHandler with new env + def set_env(self, env): + super().set_env(env) + + defined_handler = next( + filter( + lambda handler: handler.__class__ == MongoLoggingHandler.__class__, + self.log.handlers + ), None) + + if defined_handler: + defined_handler.env = env + else: + self.add_handler(MongoLoggingHandler(env, self.level)) diff --git a/app/utils/logging/logger.py b/app/utils/logging/logger.py new file mode 100644 index 0000000..bcf8287 --- /dev/null +++ b/app/utils/logging/logger.py @@ -0,0 +1,99 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import logging +from abc import ABC + + +class Logger(ABC): + DEBUG = 'DEBUG' + INFO = 'INFO' + WARNING = 'WARNING' + ERROR = 'ERROR' + CRITICAL = 'CRITICAL' + + PROJECT_NAME = 'CALIPSO' + + levels = [DEBUG, INFO, WARNING, ERROR, CRITICAL] + log_format = '%(asctime)s %(levelname)s: %(message)s' + formatter = logging.Formatter(log_format) + default_level = INFO + + def __init__(self, logger_name: str = PROJECT_NAME, + level: str = default_level): + super().__init__() + self.check_level(level) + self.log = logging.getLogger(logger_name) + logging.basicConfig(format=self.log_format, + level=level) + self.log.propagate = False + self.set_loglevel(level) + self.env = None + self.level = level + + def set_env(self, env): + self.env = env + + @staticmethod + def check_level(level): + if level.upper() not in Logger.levels: + raise ValueError('Invalid log level: {}. Supported levels: ({})' + .format(level, ", ".join(Logger.levels))) + + @staticmethod + def get_numeric_level(loglevel): + Logger.check_level(loglevel) + numeric_level = getattr(logging, loglevel.upper(), Logger.default_level) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: {}'.format(loglevel)) + return numeric_level + + def set_loglevel(self, loglevel): + # assuming loglevel is bound to the string value obtained from the + # command line argument. Convert to upper case to allow the user to + # specify --log=DEBUG or --log=debug + numeric_level = self.get_numeric_level(loglevel) + + for handler in self.log.handlers: + handler.setLevel(numeric_level) + self.log.setLevel(numeric_level) + self.level = loglevel + + def _log(self, level, message, *args, exc_info=False, **kwargs): + self.log.log(level, message, *args, exc_info=exc_info, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log(logging.DEBUG, message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log(logging.INFO, message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log(logging.WARNING, message, *args, **kwargs) + + def warn(self, message, *args, **kwargs): + self.warning(message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log(logging.ERROR, message, *args, **kwargs) + + def exception(self, message, *args, **kwargs): + self._log(logging.ERROR, message, exc_info=True, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log(logging.CRITICAL, message, *args, **kwargs) + + def add_handler(self, handler): + handler_defined = handler.__class__ in map(lambda h: h.__class__, + self.log.handlers) + + if not handler_defined: + handler.setLevel(self.level) + handler.setFormatter(self.formatter) + self.log.addHandler(handler) diff --git a/app/utils/logging/message_logger.py b/app/utils/logging/message_logger.py new file mode 100644 index 0000000..02e098f --- /dev/null +++ b/app/utils/logging/message_logger.py @@ -0,0 +1,21 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import logging + +from utils.logging.logger import Logger +from utils.logging.mongo_logging_handler import MongoLoggingHandler + + +class MessageLogger(Logger): + + def __init__(self, env: str = None, level: str = None): + super().__init__(logger_name="{}-Message".format(self.PROJECT_NAME), + level=level) + self.add_handler(MongoLoggingHandler(env, self.level)) diff --git a/app/utils/logging/mongo_logging_handler.py b/app/utils/logging/mongo_logging_handler.py new file mode 100644 index 0000000..b69270e --- /dev/null +++ b/app/utils/logging/mongo_logging_handler.py @@ -0,0 +1,53 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import datetime +import logging + +from messages.message import Message +from utils.inventory_mgr import InventoryMgr +from utils.logging.logger import Logger +from utils.string_utils import stringify_datetime + + +class MongoLoggingHandler(logging.Handler): + """ + Logging handler for MongoDB + """ + SOURCE_SYSTEM = 'Calipso' + + def __init__(self, env: str, level: str): + super().__init__(Logger.get_numeric_level(level)) + self.str_level = level + self.env = env + self.inv = None + + def emit(self, record): + # Try to invoke InventoryMgr for logging + if not self.inv: + try: + self.inv = InventoryMgr() + except: + return + + # make sure we do not try to log to DB when DB is not ready + if not (self.inv.is_db_ready() + and 'messages' in self.inv.collections): + return + + # make ID from current timestamp + now = datetime.datetime.utcnow() + d = now - datetime.datetime(1970, 1, 1) + ts = stringify_datetime(now) + timestamp_id = '{}.{}.{}'.format(d.days, d.seconds, d.microseconds) + source = self.SOURCE_SYSTEM + message = Message(msg_id=timestamp_id, env=self.env, source=source, + msg=Logger.formatter.format(record), ts=ts, + level=record.levelname) + self.inv.collections['messages'].insert_one(message.get()) \ No newline at end of file diff --git a/app/utils/metadata_parser.py b/app/utils/metadata_parser.py new file mode 100644 index 0000000..1ed49ab --- /dev/null +++ b/app/utils/metadata_parser.py @@ -0,0 +1,83 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json +import os +from abc import abstractmethod, ABCMeta + +from utils.util import get_extension + + +class MetadataParser(metaclass=ABCMeta): + + def __init__(self): + super().__init__() + self.errors = [] + + @abstractmethod + def get_required_fields(self) -> list: + pass + + def validate_metadata(self, metadata: dict) -> bool: + if not isinstance(metadata, dict): + raise ValueError('metadata needs to be a valid dict') + + # make sure metadata json contains all fields we need + required_fields = self.get_required_fields() + if not all([field in metadata for field in required_fields]): + raise ValueError("Metadata json should contain " + "all the following fields: {}" + .format(', '.join(required_fields))) + return True + + @staticmethod + def _load_json_file(file_path: str): + with open(file_path) as data_file: + return json.load(data_file) + + def _parse_json_file(self, file_path: str): + metadata = self._load_json_file(file_path) + + # validate metadata correctness + if not self.validate_metadata(metadata): + return None + + return metadata + + @staticmethod + def check_metadata_file_ok(file_path: str): + extension = get_extension(file_path) + if extension != 'json': + raise ValueError("Extension '{}' is not supported. " + "Please provide a .json metadata file." + .format(extension)) + + if not os.path.isfile(file_path): + raise ValueError("Couldn't load metadata file. " + "Path '{}' doesn't exist or is not a file" + .format(file_path)) + + def parse_metadata_file(self, file_path: str) -> dict: + # reset errors in case same parser is used to read multiple inputs + self.errors = [] + self.check_metadata_file_ok(file_path) + + # Try to parse metadata file if it has one of the supported extensions + metadata = self._parse_json_file(file_path) + self.check_errors() + return metadata + + def check_errors(self): + if self.errors: + raise ValueError("Errors encountered during " + "metadata file parsing:\n{}" + .format("\n".join(self.errors))) + + def add_error(self, msg): + self.errors.append(msg) diff --git a/app/utils/mongo_access.py b/app/utils/mongo_access.py new file mode 100644 index 0000000..1425017 --- /dev/null +++ b/app/utils/mongo_access.py @@ -0,0 +1,137 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os + +from pymongo import MongoClient + +from utils.config_file import ConfigFile +from utils.dict_naming_converter import DictNamingConverter +from utils.logging.console_logger import ConsoleLogger +from utils.logging.file_logger import FileLogger + + +# Provides access to MongoDB using PyMongo library +# +# Notes on authentication: +# default config file is calipso_mongo_access.conf +# you can also specify name of file from CLI with --mongo_config + + +class MongoAccess(DictNamingConverter): + client = None + db = None + default_conf_file = '/local_dir/calipso_mongo_access.conf' + config_file = None + + DB_NAME = 'calipso' + LOG_FILENAME = 'mongo_access.log' + DEFAULT_LOG_FILE = os.path.join(os.path.abspath("."), LOG_FILENAME) + + def __init__(self): + super().__init__() + self.log_file = os.path.join(FileLogger.LOG_DIRECTORY, + MongoAccess.LOG_FILENAME) + + try: + self.log = FileLogger(self.log_file) + except OSError as e: + ConsoleLogger().warning("Couldn't use file {} for logging. " + "Using default location: {}.\n" + "Error: {}" + .format(self.log_file, + self.DEFAULT_LOG_FILE, + e)) + + self.log_file = self.DEFAULT_LOG_FILE + self.log = FileLogger(self.log_file) + + self.connect_params = {} + self.mongo_connect(self.config_file) + + def is_db_ready(self) -> bool: + return MongoAccess.client is not None + + @staticmethod + def set_config_file(_conf_file): + MongoAccess.config_file = _conf_file + + def mongo_connect(self, config_file_path=""): + if MongoAccess.client: + return + + self.connect_params = { + "server": "localhost", + "port": 27017 + } + + if not config_file_path: + config_file_path = self.default_conf_file + + try: + config_file = ConfigFile(config_file_path) + # read connection parameters from config file + config_params = config_file.read_config() + self.connect_params.update(config_params) + except Exception as e: + self.log.exception(e) + raise + + self.prepare_connect_uri() + MongoAccess.client = MongoClient( + self.connect_params["server"], + self.connect_params["port"] + ) + MongoAccess.db = getattr(MongoAccess.client, + config_params.get('auth_db', self.DB_NAME)) + self.log.info('Connected to MongoDB') + + def prepare_connect_uri(self): + params = self.connect_params + self.log.debug('connecting to MongoDb server: {}' + .format(params['server'])) + uri = 'mongodb://' + if 'password' in params: + uri = uri + params['user'] + ':' + params['password'] + '@' + uri = uri + params['server'] + if 'auth_db' in params: + uri = uri + '/' + params['auth_db'] + self.connect_params['server'] = uri + + @staticmethod + def update_document(collection, document, upsert=False): + if isinstance(collection, str): + collection = MongoAccess.db[collection] + doc_id = document.pop('_id') + collection.update_one({'_id': doc_id}, {'$set': document}, + upsert=upsert) + document['_id'] = doc_id + + @staticmethod + def encode_dots(s): + return s.replace(".", "[dot]") + + @staticmethod + def decode_dots(s): + return s.replace("[dot]", ".") + + # Mongo will not accept dot (".") in keys, or $ in start of keys + # $ in beginning of key does not happen in OpenStack, + # so need to translate only "." --> "[dot]" + @staticmethod + def encode_mongo_keys(item): + return MongoAccess.change_dict_naming_convention(item, MongoAccess.encode_dots) + + @staticmethod + def decode_mongo_keys(item): + return MongoAccess.change_dict_naming_convention(item, MongoAccess.decode_dots) + + @staticmethod + def decode_object_id(item: dict): + return dict(item, **{"_id": str(item["_id"])}) if item and "_id" in item else item diff --git a/app/utils/singleton.py b/app/utils/singleton.py new file mode 100644 index 0000000..fc1147f --- /dev/null +++ b/app/utils/singleton.py @@ -0,0 +1,16 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] diff --git a/app/utils/special_char_converter.py b/app/utils/special_char_converter.py new file mode 100644 index 0000000..fb469bb --- /dev/null +++ b/app/utils/special_char_converter.py @@ -0,0 +1,32 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import re + + +class SpecialCharConverter: + + translated_re = re.compile(r'---[.][.][0-9]+[.][.]---') + + def encode_special_characters(self, s): + SPECIAL_CHARS = [':', '/'] + for c in SPECIAL_CHARS: + if c in s: + s = s.replace(c, '---..' + str(ord(c)) + '..---') + return s + + def decode_special_characters(self, s): + replaced = [] + for m in re.finditer(self.translated_re, s): + match = m.group(0) + char_code = match[5:len(match)-5] + if char_code not in replaced: + replaced.append(char_code) + s = s.replace(match, chr(int(char_code))) + return s diff --git a/app/utils/ssh_conn.py b/app/utils/ssh_conn.py new file mode 100644 index 0000000..d4b7954 --- /dev/null +++ b/app/utils/ssh_conn.py @@ -0,0 +1,94 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os + +from discover.configuration import Configuration +from utils.inventory_mgr import InventoryMgr +from utils.ssh_connection import SshConnection + + +class SshConn(SshConnection): + config = None + ssh = None + connections = {} + + max_call_count_per_con = 100 + timeout = 15 # timeout for exec in seconds + + def __init__(self, host_name, for_sftp=False): + self.config = Configuration() + self.env_config = self.config.get_env_config() + self.env = self.env_config['name'] + self.conf = self.config.get('CLI') + self.gateway = None + self.host = None + self.host_conf = self.get_host_conf(host_name) + self.ssh = None + self.ftp = None + self.for_sftp = for_sftp + self.key = None + self.port = None + self.user = None + self.pwd = None + self.check_definitions() + super().__init__(self.host, self.user, _pwd=self.pwd, _key=self.key, + _port=self.port, for_sftp=for_sftp) + self.inv = InventoryMgr() + if host_name in self.connections and not self.ssh: + self.ssh = self.connections[host_name] + + def get_host_conf(self, host_name): + if 'hosts' in self.conf: + if not host_name: + raise ValueError('SshConn(): host must be specified ' + + 'if multi-host CLI config is used') + if host_name not in self.conf['hosts']: + raise ValueError('host details missing: ' + host_name) + return self.conf['hosts'][host_name] + else: + return self.conf + + def check_definitions(self): + try: + self.host = self.host_conf['host'] + if self.host in self.connections: + self.ssh = self.connections[self.host] + except KeyError: + raise ValueError('Missing definition of host for CLI access') + try: + self.user = self.host_conf['user'] + except KeyError: + raise ValueError('Missing definition of user for CLI access') + try: + self.key = self.host_conf['key'] + if self.key and not os.path.exists(self.key): + raise ValueError('Key file not found: ' + self.key) + except KeyError: + pass + try: + self.pwd = self.host_conf['pwd'] + except KeyError: + self.pwd = None + if not self.key and not self.pwd: + raise ValueError('Must specify key or password for CLI access') + + gateway_hosts = {} + + @staticmethod + def get_gateway_host(host): + if not SshConn.gateway_hosts.get(host): + ssh = SshConn(host) + gateway = ssh.exec('uname -n') + SshConn.gateway_hosts[host] = gateway.strip() + return SshConn.gateway_hosts[host] + + def is_gateway_host(self, host): + gateway_host = self.get_gateway_host(host) + return host == gateway_host diff --git a/app/utils/ssh_connection.py b/app/utils/ssh_connection.py new file mode 100644 index 0000000..0fa197a --- /dev/null +++ b/app/utils/ssh_connection.py @@ -0,0 +1,217 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import os + +import paramiko + +from utils.binary_converter import BinaryConverter + + +class SshConnection(BinaryConverter): + config = None + ssh = None + connections = {} + cli_connections = {} + sftp_connections = {} + + max_call_count_per_con = 100 + timeout = 15 # timeout for exec in seconds + + DEFAULT_PORT = 22 + + def __init__(self, _host: str, _user: str, _pwd: str=None, _key: str = None, + _port: int = None, _call_count_limit: int=None, + for_sftp: bool = False): + super().__init__() + self.host = _host + self.ssh = None + self.ftp = None + self.for_sftp = for_sftp + self.key = _key + self.port = _port + self.user = _user + self.pwd = _pwd + self.check_definitions() + self.fetched_host_details = False + self.call_count = 0 + self.call_count_limit = 0 if for_sftp \ + else (SshConnection.max_call_count_per_con + if _call_count_limit is None else _call_count_limit) + if for_sftp: + self.sftp_connections[_host] = self + else: + self.cli_connections[_host] = self + + def check_definitions(self): + if not self.host: + raise ValueError('Missing definition of host for CLI access') + if not self.user: + raise ValueError('Missing definition of user ' + + 'for CLI access to host {}'.format(self.host)) + if self.key and not os.path.exists(self.key): + raise ValueError('Key file not found: ' + self.key) + if not self.key and not self.pwd: + raise ValueError('Must specify key or password ' + + 'for CLI access to host {}'.format(self.host)) + + @staticmethod + def get_ssh(host, for_sftp=False): + if for_sftp: + return SshConnection.cli_connections.get(host) + return SshConnection.sftp_connections.get(host) + + @staticmethod + def get_connection(host, for_sftp=False): + key = ('sftp-' if for_sftp else '') + host + return SshConnection.connections.get(key) + + def disconnect(self): + if self.ssh: + self.ssh.close() + + @staticmethod + def disconnect_all(): + for ssh in SshConnection.cli_connections.values(): + ssh.disconnect() + SshConnection.cli_connections = {} + for ssh in SshConnection.sftp_connections.values(): + ssh.disconnect() + SshConnection.sftp_connections = {} + + def get_host(self): + return self.host + + def get_user(self): + return self.user + + def set_call_limit(self, _limit: int): + self.call_count_limit = _limit + + def connect(self, reconnect=False) -> bool: + connection = self.get_connection(self.host, self.for_sftp) + if connection: + self.ssh = connection + if reconnect: + self.log.info("SshConnection: " + + "****** forcing reconnect: %s ******", + self.host) + elif self.call_count >= self.call_count_limit > 0: + self.log.info("SshConnection: ****** reconnecting: %s, " + + "due to call count: %s ******", + self.host, self.call_count) + else: + return True + connection.close() + self.ssh = None + self.ssh = paramiko.SSHClient() + connection_key = ('sftp-' if self.for_sftp else '') + self.host + SshConnection.connections[connection_key] = self.ssh + self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + if self.key: + k = paramiko.RSAKey.from_private_key_file(self.key) + self.ssh.connect(hostname=self.host, username=self.user, pkey=k, + port=self.port if self.port is not None + else self.DEFAULT_PORT, + password=self.pwd, timeout=30) + else: + try: + port = self.port if self.port is not None else self.DEFAULT_PORT + self.ssh.connect(self.host, + username=self.user, + password=self.pwd, + port=port, + timeout=30) + except paramiko.ssh_exception.AuthenticationException: + self.log.error('Failed SSH connect to host {}, port={}' + .format(self.host, port)) + self.ssh = None + self.call_count = 0 + return self.ssh is not None + + def exec(self, cmd): + if not self.connect(): + return '' + self.call_count += 1 + self.log.debug("call count: %s, running call:\n%s\n", + str(self.call_count), cmd) + stdin, stdout, stderr = self.ssh.exec_command(cmd, timeout=self.timeout) + stdin.close() + err = self.binary2str(stderr.read()) + if err: + # ignore messages about loading plugin + err_lines = [l for l in err.splitlines() + if 'Loaded plugin: ' not in l] + if err_lines: + self.log.error("CLI access: \n" + + "Host: {}\nCommand: {}\nError: {}\n". + format(self.host, cmd, err)) + stderr.close() + stdout.close() + return "" + ret = self.binary2str(stdout.read()) + stderr.close() + stdout.close() + return ret + + def copy_file(self, local_path, remote_path, mode=None): + if not self.connect(): + return + if not self.ftp: + self.ftp = self.ssh.open_sftp() + try: + self.ftp.put(local_path, remote_path) + except IOError as e: + self.log.error('SFTP copy_file failed to copy file: ' + + 'local: ' + local_path + + ', remote host: ' + self.host + + ', error: ' + str(e)) + return str(e) + try: + remote_file = self.ftp.file(remote_path, 'a+') + except IOError as e: + self.log.error('SFTP copy_file failed to open file after put(): ' + + 'local: ' + local_path + + ', remote host: ' + self.host + + ', error: ' + str(e)) + return str(e) + try: + if mode: + remote_file.chmod(mode) + except IOError as e: + self.log.error('SFTP copy_file failed to chmod file: ' + + 'local: ' + local_path + + ', remote host: ' + self.host + + ', port: ' + self.port + + ', error: ' + str(e)) + return str(e) + self.log.info('SFTP copy_file success: ' + 'host={},port={},{} -> {}'.format( + str(self.host), str(self.port), str(local_path), str(remote_path))) + return '' + + def copy_file_from_remote(self, remote_path, local_path): + if not self.connect(): + return + if not self.ftp: + self.ftp = self.ssh.open_sftp() + try: + self.ftp.get(remote_path, local_path) + except IOError as e: + self.log.error('SFTP copy_file_from_remote failed to copy file: ' + 'remote host: {}, ' + 'remote_path: {}, local: {}, error: {}' + .format(self.host, remote_path, local_path, str(e))) + return str(e) + self.log.info('SFTP copy_file_from_remote success: host={},{} -> {}'. + format(self.host, remote_path, local_path)) + return '' + + def is_gateway_host(self, host): + return True diff --git a/app/utils/string_utils.py b/app/utils/string_utils.py new file mode 100644 index 0000000..1f51992 --- /dev/null +++ b/app/utils/string_utils.py @@ -0,0 +1,59 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import json +from datetime import datetime + +from bson import ObjectId + + +def jsonify(obj, prettify=False): + if prettify: + return json.dumps(obj, sort_keys=True, indent=4, separators=(',', ': ')) + else: + return json.dumps(obj) + + +# stringify datetime object +def stringify_datetime(dt): + return dt.strftime("%Y-%m-%dT%H:%M:%S.%f%z") + + +# stringify ObjectId +def stringify_object_id(object_id): + return str(object_id) + + +stringify_map = { + ObjectId: stringify_object_id, + datetime: stringify_datetime +} + + +def stringify_object_values_by_type(obj, object_type): + if isinstance(obj, dict): + for key, value in obj.items(): + if isinstance(value, object_type): + obj[key] = stringify_map[object_type](value) + else: + stringify_object_values_by_type(value, object_type) + elif isinstance(obj, list): + for index, value in enumerate(obj): + if isinstance(value, object_type): + obj[index] = stringify_map[object_type](value) + else: + stringify_object_values_by_type(value, object_type) + + +# convert some values of the specific types of the object into string +# e.g convert all the ObjectId to string +# convert all the datetime object to string +def stringify_object_values_by_types(obj, object_types): + for object_type in object_types: + stringify_object_values_by_type(obj, object_type) diff --git a/app/utils/util.py b/app/utils/util.py new file mode 100644 index 0000000..4695879 --- /dev/null +++ b/app/utils/util.py @@ -0,0 +1,172 @@ +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +import importlib +import signal +from argparse import Namespace +from typing import Dict, Callable + +import os +import re + +from bson.objectid import ObjectId + + +class SignalHandler: + + def __init__(self, signals=(signal.SIGTERM, signal.SIGINT)): + super().__init__() + self.terminated = False + for sig in signals: + signal.signal(sig, self.handle) + + def handle(self, signum, frame): + self.terminated = True + + +class ClassResolver: + instances = {} + + # convert class name in camel case to module file name in underscores + @staticmethod + def get_module_file_by_class_name(class_name): + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', class_name) + module_file = re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + return module_file + + # convert module file name in underscores to class name in camel case + @staticmethod + def get_class_name_by_module(module_name): + name_parts = [word.capitalize() for word in module_name.split('_')] + class_name = ''.join(name_parts) + return class_name + + + @staticmethod + def get_fully_qualified_class(class_name: str = None, + package_name: str = "discover", + module_name: str = None): + module_file = module_name if module_name \ + else ClassResolver.get_module_file_by_class_name(class_name) + module_parts = [package_name, module_file] + module_name = ".".join(module_parts) + try: + class_module = importlib.import_module(module_name) + except ImportError: + raise ValueError('could not import module {}'.format(module_name)) + + clazz = getattr(class_module, class_name) + return clazz + + @staticmethod + def prepare_class(class_name: str = None, + package_name: str = "discover", + module_name: str = None): + if not class_name and not module_name: + raise ValueError('class_name or module_name must be provided') + if not class_name: + class_name = ClassResolver.get_class_name_by_module(module_name) + if class_name in ClassResolver.instances: + return 'instance', ClassResolver.instances[class_name] + clazz = ClassResolver.get_fully_qualified_class(class_name, package_name, + module_name) + return 'class', clazz + + @staticmethod + def get_instance_of_class(class_name: str = None, + package_name: str = "discover", + module_name: str = None): + val_type, clazz = \ + ClassResolver.prepare_class(class_name=class_name, + package_name=package_name, + module_name=module_name) + if val_type == 'instance': + return clazz + instance = clazz() + ClassResolver.instances[class_name] = instance + return instance + + @staticmethod + def get_instance_single_arg(arg: object, + class_name: str = None, + package_name: str = "discover", + module_name: str = None): + val_type, clazz = \ + ClassResolver.prepare_class(class_name=class_name, + package_name=package_name, + module_name=module_name) + if val_type == 'instance': + return clazz + instance = clazz(arg) + ClassResolver.instances[class_name] = instance + return instance + + +# TODO: translate the following comment +# when search in the mongo db, need to +# generate the ObjectId with the string +def generate_object_ids(keys, obj): + for key in keys: + if key in obj: + o = obj.pop(key) + if o: + try: + o = ObjectId(o) + except Exception: + raise Exception("{0} is not a valid object id". + format(o)) + obj[key] = o + + +# Get arguments from CLI or another source +# and convert them to dict to enforce uniformity. +# Throws a TypeError if arguments can't be converted to dict. +def setup_args(args: dict, + defaults: Dict[str, object], + get_cmd_args: Callable[[], Namespace] = None): + if defaults is None: + defaults = {} + + if args is None and get_cmd_args is not None: + args = vars(get_cmd_args()) + elif not isinstance(args, dict): + try: + args = dict(args) + except TypeError: + try: + args = vars(args) + except TypeError: + raise TypeError("Wrong arguments format") + + return dict(defaults, **args) + + +def encode_router_id(host_id: str, uuid: str): + return '-'.join([host_id, 'qrouter', uuid]) + + +def decode_router_id(router_id: str): + return router_id.split('qrouter-')[-1] + + +def get_extension(file_path: str) -> str: + return os.path.splitext(file_path)[1][1:] + + +def encode_aci_dn(object_id): + return object_id.replace("topology/", "").replace("/", "___").replace("-", "__") + + +def decode_aci_dn(object_id): + return object_id.replace("___", "/").replace("__", "-") + + +def get_object_path_part(path: str, part_name: str): + match = re.match(".*/{}/(.+?)/.*".format(part_name), path) + return match.group(1) if match else None -- cgit 1.2.3-korg