summaryrefslogtreecommitdiffstats
path: root/app/utils/inventory_mgr.py
diff options
context:
space:
mode:
authorKoren Lev <korenlev@gmail.com>2017-07-27 15:04:07 +0000
committerGerrit Code Review <gerrit@opnfv.org>2017-07-27 15:04:07 +0000
commit162c03ef301396072c1934e7e7e0c40a841b4fe2 (patch)
tree7a2a2781949252436450ff5832962785061a774a /app/utils/inventory_mgr.py
parentb88c78e3cf2bef22aa2f1c4d0bf305e303bc15f0 (diff)
parent7e83d0876ddb84a45e130eeba28bc40ef53c074b (diff)
Merge "Calipso initial release for OPNFV"
Diffstat (limited to 'app/utils/inventory_mgr.py')
-rw-r--r--app/utils/inventory_mgr.py445
1 files changed, 445 insertions, 0 deletions
diff --git a/app/utils/inventory_mgr.py b/app/utils/inventory_mgr.py
new file mode 100644
index 0000000..2fe2894
--- /dev/null
+++ b/app/utils/inventory_mgr.py
@@ -0,0 +1,445 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) #
+# and others #
+# #
+# All rights reserved. This program and the accompanying materials #
+# are made available under the terms of the Apache License, Version 2.0 #
+# which accompanies this distribution, and is available at #
+# http://www.apache.org/licenses/LICENSE-2.0 #
+###############################################################################
+from datetime import datetime
+
+import bson
+
+from utils.constants import EnvironmentFeatures
+from utils.logging.console_logger import ConsoleLogger
+from utils.mongo_access import MongoAccess
+from utils.singleton import Singleton
+
+
+def inv_initialization_required(func):
+ def decorated(self, *args, **kwargs):
+ if self.inventory_collection is None:
+ raise TypeError("Inventory collection is not set.")
+ return func(self, *args, **kwargs)
+ return decorated
+
+
+class InventoryMgr(MongoAccess, metaclass=Singleton):
+
+ def __init__(self):
+ super().__init__()
+ self.log = ConsoleLogger()
+ self.inventory_collection = None
+ self.inventory_collection_name = None
+ self.collections = {}
+ self.monitoring_setup_manager = None
+
+ def set_collection(self, collection_type: str = None,
+ use_default_name: bool = False):
+ # do not allow setting the collection more than once
+ if not self.collections.get(collection_type):
+ collection_name = collection_type \
+ if use_default_name \
+ else self.get_coll_name(collection_type)
+
+ self.log.info("Using {} collection: {}"
+ .format(collection_type, collection_name))
+
+ self.collections[collection_type] = MongoAccess.db[collection_name]
+
+ def set_inventory_collection(self, collection_name: str = None):
+ if not self.inventory_collection:
+ if not collection_name:
+ collection_name = "inventory"
+
+ self.log.info("Using inventory collection: {}"
+ .format(collection_name))
+
+ collection = MongoAccess.db[collection_name]
+ self.collections["inventory"] = collection
+ self.inventory_collection = collection
+ self.inventory_collection_name = collection_name
+
+ def get_coll_name(self, coll_name):
+ if not self.inventory_collection_name:
+ raise TypeError("inventory_collection_name is not set")
+
+ return self.inventory_collection_name.replace("inventory", coll_name) \
+ if self.inventory_collection_name.startswith("inventory") \
+ else self.inventory_collection_name + "_" + coll_name
+
+ def set_collections(self, inventory_collection: str = None):
+ self.set_inventory_collection(inventory_collection)
+ self.set_collection("links")
+ self.set_collection("link_types")
+ self.set_collection("clique_types")
+ self.set_collection("clique_constraints")
+ self.set_collection("cliques")
+ self.set_collection("monitoring_config")
+ self.set_collection("constants", use_default_name=True)
+ self.set_collection("scans")
+ self.set_collection("messages")
+ self.set_collection("monitoring_config_templates",
+ use_default_name=True)
+ self.set_collection("environments_config")
+ self.set_collection("supported_environments")
+
+ def clear(self, scan_plan):
+ if scan_plan.inventory_only:
+ collections = {"inventory"}
+ elif scan_plan.links_only:
+ collections = {"links"}
+ elif scan_plan.cliques_only:
+ collections = {"cliques"}
+ else:
+ collections = {"inventory", "links", "cliques", "monitoring_config"}
+
+ env_cond = {} if scan_plan.clear_all else {"environment": scan_plan.env}
+
+ for collection_name in collections:
+ collection = self.collections[collection_name]
+ self.log.info("clearing collection: " + collection.full_name)
+ # delete docs from the collection,
+ # either all or just for the specified environment
+ collection.delete_many(env_cond)
+
+ # return single match
+ def get_by_id(self, environment, item_id):
+ return self.find({
+ "environment": environment,
+ "id": item_id
+ }, get_single=True)
+
+ # return matches for ID in list of values
+ def get_by_ids(self, environment, ids_list):
+ return self.find({
+ "environment": environment,
+ "id": {"$in": ids_list}
+ })
+
+ def get_by_field(self, environment, item_type, field_name, field_value,
+ get_single=False):
+ if field_value:
+ return self.find({"environment": environment,
+ "type": item_type,
+ field_name: field_value},
+ get_single=get_single)
+ else:
+ return self.find({"environment": environment,
+ "type": item_type},
+ get_single=get_single)
+
+ def get(self, environment, item_type, item_id, get_single=False):
+ return self.get_by_field(environment, item_type, "id", item_id,
+ get_single=get_single)
+
+ def get_children(self, environment, item_type, parent_id):
+ if parent_id:
+ if not item_type:
+ return self.find({"environment": environment,
+ "parent_id": parent_id})
+ else:
+ return self.find({"environment": environment,
+ "type": item_type,
+ "parent_id": parent_id})
+ else:
+ return self.find({"environment": environment,
+ "type": item_type})
+
+ def get_single(self, environment, item_type, item_id):
+ matches = self.find({"environment": environment,
+ "type": item_type,
+ "id": item_id})
+ if len(matches) > 1:
+ raise ValueError("Found multiple matches for item: " +
+ "type=" + item_type + ", id=" + item_id)
+ if len(matches) == 0:
+ raise ValueError("No matches for item: " +
+ "type=" + item_type + ", id=" + item_id)
+ return matches[0]
+
+ # item must contain properties 'environment', 'type' and 'id'
+ def set(self, item, collection=None):
+ col = collection
+ mongo_id = None
+ projects = None
+ if "_id" in item:
+ mongo_id = item.pop("_id", None)
+
+ if not collection or collection == self.collections['inventory']:
+ # make sure we have environment, type & id
+ self.check(item, "environment")
+ self.check(item, "type")
+ self.check(item, "id")
+
+ item["last_scanned"] = datetime.now()
+ item.pop("projects", [])
+
+ obj_name = item["name_path"]
+ obj_name = obj_name[obj_name.rindex('/') + 1:]
+
+ if 'object_name' not in item:
+ item['object_name'] = obj_name
+
+ self.set_collections() # make sure we have all collections set
+ if not col:
+ col = self.collections['inventory']
+
+ find_tuple = {"environment": item["environment"],
+ "type": item["type"], "id": item["id"]}
+ else:
+ find_tuple = {'_id': bson.ObjectId(mongo_id)}
+ doc = col.find_one(find_tuple)
+ if not doc:
+ raise ValueError('set(): could not find document with _id=' +
+ mongo_id)
+
+ col.update_one(find_tuple,
+ {'$set': self.encode_mongo_keys(item)},
+ upsert=True)
+ if mongo_id:
+ # restore original mongo ID of document, in case we need to use it
+ item['_id'] = mongo_id
+ if projects:
+ col.update_one(find_tuple,
+ {'$addToSet': {"projects": {'$each': projects}}},
+ upsert=True)
+
+ @staticmethod
+ def check(obj, field_name):
+ arg = obj[field_name]
+ if not arg or not str(arg).rstrip():
+ raise ValueError("Inventory item - " +
+ "the following field is not defined: " +
+ field_name)
+
+ # note: to use general find, call find_items(),
+ # which also does process_results
+ @inv_initialization_required
+ def find(self, search, projection=None, collection=None, get_single=False):
+ coll = self.inventory_collection if not collection \
+ else self.collections[collection]
+ if get_single is True:
+ return self.decode_object_id(
+ self.decode_mongo_keys(
+ coll.find_one(search, projection=projection)
+ )
+ )
+ else:
+ return list(
+ map(
+ self.decode_object_id,
+ map(
+ self.decode_mongo_keys,
+ coll.find(search, projection=projection))
+ )
+ )
+
+ def find_one(self, search, projection=None, collection=None) -> dict:
+ return self.find(search, projection, collection, True)
+
+ def find_items(self, search,
+ projection=None,
+ get_single=False,
+ collection=None):
+ return self.find(search, projection, collection, get_single)
+
+ # record a link between objects in the inventory, to be used in graphs
+ # returns - the new link document
+ # parameters -
+ # environment: name of environment
+ # host: name of host
+ # source: node mongo _id
+ # source_id: node id value of source node
+ # target: node mongo _id
+ # target_id: node id value of target node
+ # link_type: string showing types of connected objects, e.g. "instance-vnic"
+ # link_name: label for the link itself
+ # state: up/down
+ # link_weight: integer, position/priority for graph placement
+ # source_label, target_label: labels for the ends of the link (optional)
+ def create_link(self, env, host, src, source_id, target, target_id,
+ link_type, link_name, state, link_weight,
+ source_label="", target_label="",
+ extra_attributes=None):
+ s = bson.ObjectId(src)
+ t = bson.ObjectId(target)
+ link = {
+ "environment": env,
+ "host": host,
+ "source": s,
+ "source_id": source_id,
+ "target": t,
+ "target_id": target_id,
+ "link_type": link_type,
+ "link_name": link_name,
+ "state": state,
+ "link_weight": link_weight,
+ "source_label": source_label,
+ "target_label": target_label,
+ "attributes": extra_attributes if extra_attributes else {}
+ }
+ return self.write_link(link)
+
+ def write_link(self, link):
+ find_tuple = {
+ 'environment': link['environment'],
+ 'source_id': link['source_id'],
+ 'target_id': link['target_id']
+ }
+ if "_id" in link:
+ link.pop("_id", None)
+ link_encoded = self.encode_mongo_keys(link)
+ links_col = self.collections["links"]
+ result = links_col.update_one(find_tuple, {'$set': link_encoded},
+ upsert=True)
+ link['_id'] = result.upserted_id
+ return link
+
+ def values_replace_in_object(self, o, values_replacement):
+ for k in values_replacement.keys():
+ if k not in o:
+ continue
+ repl = values_replacement[k]
+ if 'from' not in repl or 'to' not in repl:
+ continue
+ o[k] = o[k].replace(repl['from'], repl['to'])
+ self.set(o)
+
+ # perform replacement of substring in values of objects in the inventory
+ # input:
+ # - search: dict with search parameters
+ # - values_replacement: dict,
+ # - keys: names of keys for which to replace the values
+ # - values: dict with "from" (value to be replaced) and "to" (new value)
+ @inv_initialization_required
+ def values_replace(self, search, values_replacement):
+ for doc in self.inventory_collection.find(search):
+ self.values_replace_in_object(doc, values_replacement)
+
+ def delete(self, coll, query_filter):
+ collection = self.collections[coll]
+ if not collection:
+ self.log.warn('delete(): collection not found - ' + coll)
+ return
+ result = collection.delete_many(query_filter)
+ count = result.deleted_count
+ self.log.info('delete(): ' + ('deleted ' + str(count) + ' documents'
+ if count else 'no matching documents'))
+ return count
+
+ def get_env_config(self, env: str):
+ return self.find_one(search={'name': env},
+ collection='environments_config')
+
+ def is_feature_supported(self, env: str, feature: EnvironmentFeatures)\
+ -> bool:
+ env_config = self.get_env_config(env)
+ if not env_config:
+ return False
+
+ # Workaround for mechanism_drivers field type
+ mechanism_driver = env_config['mechanism_drivers'][0] \
+ if isinstance(env_config['mechanism_drivers'], list) \
+ else env_config['mechanism_drivers']
+
+ full_env = {'environment.distribution': env_config['distribution'],
+ 'environment.type_drivers': env_config['type_drivers'],
+ 'environment.mechanism_drivers': mechanism_driver}
+ return self.is_feature_supported_in_env(full_env, feature)
+
+ def is_feature_supported_in_env(self, env_def: dict,
+ feature: EnvironmentFeatures) -> bool:
+
+ result = self.collections['supported_environments'].find_one(env_def)
+ if not result:
+ return False
+ features_in_env = result.get('features', {})
+ return features_in_env.get(feature.value) is True
+
+ def save_inventory_object(self, o: dict, parent: dict,
+ environment: str, type_to_fetch: dict = None) -> bool:
+ if not type_to_fetch:
+ type_to_fetch = {}
+
+ o["id"] = str(o["id"])
+ o["environment"] = environment
+ if type_to_fetch.get("type"):
+ o["type"] = type_to_fetch["type"]
+ o["show_in_tree"] = type_to_fetch.get("show_in_tree", True)
+
+ parent_id_path = parent.get("id_path", "/{}".format(environment))
+ parent_name_path = parent.get("name_path", "/{}".format(environment))
+
+ try:
+ # case of dynamic folder added by need
+ master_parent_type = o["master_parent_type"]
+ master_parent_id = o["master_parent_id"]
+ master_parent = self.get_by_id(environment, master_parent_id)
+ if not master_parent:
+ self.log.error("failed to find master parent " +
+ master_parent_id)
+ return False
+ folder_id_path = "/".join((master_parent["id_path"], o["parent_id"]))
+ folder_name_path = "/".join((master_parent["name_path"], o["parent_text"]))
+ folder = {
+ "environment": parent["environment"],
+ "parent_id": master_parent_id,
+ "parent_type": master_parent_type,
+ "id": o["parent_id"],
+ "id_path": folder_id_path,
+ "show_in_tree": True,
+ "name_path": folder_name_path,
+ "name": o["parent_id"],
+ "type": o["parent_type"],
+ "text": o["parent_text"]
+ }
+ # remove master_parent_type & master_parent_id after use,
+ # as they're there just ro help create the dynamic folder
+ o.pop("master_parent_type", True)
+ o.pop("master_parent_id", True)
+ self.set(folder)
+ except KeyError:
+ pass
+
+ if o.get("text"):
+ o["name"] = o["text"]
+ elif not o.get("name"):
+ o["name"] = o["id"]
+
+ if "parent_id" not in o and parent:
+ parent_id = parent["id"]
+ o["parent_id"] = parent_id
+ o["parent_type"] = parent["type"]
+ elif "parent_id" in o and o["parent_id"] != parent["id"]:
+ # using alternate parent - fetch parent path from inventory
+ parent_obj = self.get_by_id(environment, o["parent_id"])
+ if parent_obj:
+ parent_id_path = parent_obj["id_path"]
+ parent_name_path = parent_obj["name_path"]
+ o["id_path"] = "/".join((parent_id_path, o["id"].strip()))
+ o["name_path"] = "/".join((parent_name_path, o["name"]))
+
+ # keep list of projects that an object is in
+ associated_projects = []
+ keys_to_remove = []
+ for k in o:
+ if k.startswith("in_project-"):
+ proj_name = k[k.index('-') + 1:]
+ associated_projects.append(proj_name)
+ keys_to_remove.append(k)
+ for k in keys_to_remove:
+ o.pop(k)
+ if len(associated_projects) > 0:
+ projects = o["projects"] if "projects" in o.keys() else []
+ projects.extend(associated_projects)
+ if projects:
+ o["projects"] = projects
+
+ if "create_object" not in o or o["create_object"]:
+ # add/update object in DB
+ self.set(o)
+ if self.is_feature_supported(environment, EnvironmentFeatures.MONITORING):
+ self.monitoring_setup_manager.create_setup(o)
+ return True