Calipso initial release for OPNFV

Change-Id: I7210c244b0c10fa80bfa8c77cb86c9d6ddf8bc88
Signed-off-by: Yaron Yogev <yaronyogev@gmail.com>

9 files changed, 390 insertions, 0 deletions

diff --git a/app/monitoring/handlers/__init__.py b/app/monitoring/handlers/__init__.py
new file mode 100644
index 0000000..1e85a2a
--- /dev/null
+++ b/app/monitoring/handlers/__init__.py
@@ -0,0 +1,10 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+
diff --git a/app/monitoring/handlers/basic_check_handler.py b/app/monitoring/handlers/basic_check_handler.py
new file mode 100644
index 0000000..7c945e8
--- /dev/null
+++ b/app/monitoring/handlers/basic_check_handler.py
@@ -0,0 +1,25 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for VPP vEdge objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+
+class BasicCheckHandler(MonitoringCheckHandler):
+
+    def __init__(self, args):
+        super().__init__(args)
+
+    def handle(self, id, check_result):
+        doc = self.doc_by_id(id)
+        if not doc:
+            return 1
+        self.keep_result(doc, check_result)
+        return check_result['status']
diff --git a/app/monitoring/handlers/handle_link.py b/app/monitoring/handlers/handle_link.py
new file mode 100644
index 0000000..26f4d12
--- /dev/null
+++ b/app/monitoring/handlers/handle_link.py
@@ -0,0 +1,36 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for links
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+
+class HandleLink(MonitoringCheckHandler):
+
+    def __init__(self, args):
+        super().__init__(args)
+
+    def handle(self, link_id_from_check, check_result):
+        # link ID from check is formatted like this:
+        # <link type>_<source_id>_<target_id>
+        link_type = link_id_from_check[:link_id_from_check.index('_')]
+        remainder = link_id_from_check[len(link_type)+1:]
+        source_id = remainder[:remainder.index('_')]
+        target_id = remainder[len(source_id)+1:]
+        search = {
+            'link_type': link_type,
+            'source_id': source_id,
+            'target_id': target_id
+        }
+        doc = self.inv.find_items(search, collection='links', get_single=True)
+        if not doc:
+            return 1
+        self.keep_result(doc, check_result)
+        return check_result['status']
diff --git a/app/monitoring/handlers/handle_otep.py b/app/monitoring/handlers/handle_otep.py
new file mode 100644
index 0000000..0189625
--- /dev/null
+++ b/app/monitoring/handlers/handle_otep.py
@@ -0,0 +1,48 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for OTEP objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+
+class HandleOtep(MonitoringCheckHandler):
+
+    def __init__(self, args):
+        super().__init__(args)
+
+    def handle(self, id, check_result):
+        object_id = id[:id.index('_')]
+        port_id = id[id.index('_')+1:]
+        doc = self.doc_by_id(object_id)
+        if not doc:
+            return 1
+        ports = doc['ports']
+        port = ports[port_id]
+        if not port:
+            self.log.error('Port not found: ' + port_id)
+            return 1
+        status = check_result['status']
+        port['status'] = self.STATUS_LABEL[status]
+        port['status_value'] = status
+        port['status_text'] = check_result['output']
+
+        # set object status based on overall state of ports
+        status_list = [p['status'] for p in ports.values() if 'status' in p]
+        # OTEP overall status:
+        # - Critical if no port is OK
+        # - Warning if some ports not OK
+        # - otherwise OK
+        status = \
+            2 if 'OK' not in status_list \
+            else 1 if 'Critical' in status_list or 'Warning' in status_list \
+            else 0
+        self.set_doc_status(doc, status, None, self.check_ts(check_result))
+        self.keep_message(doc, check_result)
+        return status
diff --git a/app/monitoring/handlers/handle_pnic.py b/app/monitoring/handlers/handle_pnic.py
new file mode 100644
index 0000000..934bb16
--- /dev/null
+++ b/app/monitoring/handlers/handle_pnic.py
@@ -0,0 +1,29 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for pNIC objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+class HandlePnic(MonitoringCheckHandler):
+
+  def __init__(self, args):
+    super().__init__(args)
+
+  def handle(self, id, check_result):
+    object_id = id[:id.index('-')]
+    mac = id[id.index('-')+1:]
+    mac_address = '%s:%s:%s:%s:%s:%s' % \
+      (mac[0:2], mac[2:4], mac[4:6], mac[6:8], mac[8:10], mac[10:12])
+    object_id += '-' + mac_address
+    doc = self.doc_by_id(object_id)
+    if not doc:
+      return 1
+    self.keep_result(doc, check_result)
+    return check_result['status']
diff --git a/app/monitoring/handlers/handle_pnic_vpp.py b/app/monitoring/handlers/handle_pnic_vpp.py
new file mode 100644
index 0000000..47a76e5
--- /dev/null
+++ b/app/monitoring/handlers/handle_pnic_vpp.py
@@ -0,0 +1,28 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for VPP vEdge objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+
+class HandlePnicVpp(MonitoringCheckHandler):
+
+    def __init__(self, args):
+        super().__init__(args)
+
+    def handle(self, id, check_result):
+        id = self.decode_special_characters(id)
+        pnic = self.doc_by_id(id)
+        if not pnic:
+            return 1
+        self.keep_result(pnic, check_result)
+        # in vEdge object in corresponding port name, set attributes:
+        # "status", "status_timestamp", "status_text"
+        return check_result['status']
diff --git a/app/monitoring/handlers/handle_vnic_vpp.py b/app/monitoring/handlers/handle_vnic_vpp.py
new file mode 100644
index 0000000..c7d234d
--- /dev/null
+++ b/app/monitoring/handlers/handle_vnic_vpp.py
@@ -0,0 +1,28 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event for VPP vEdge objects
+
+from monitoring.handlers.monitoring_check_handler import MonitoringCheckHandler
+
+
+class HandleVnicVpp(MonitoringCheckHandler):
+
+    def __init__(self, args):
+        super().__init__(args)
+
+    def handle(self, id, check_result):
+        is_instance_vnic = id.startswith('instance_vnic')
+        vnic_type = 'instance_vnic' if is_instance_vnic else 'vservice_vnic'
+        id = self.decode_special_characters(id[len(vnic_type)+1:])
+        doc = self.doc_by_id(id)
+        if not doc:
+            return 1
+        self.keep_result(doc, check_result)
+        return check_result['status']
diff --git a/app/monitoring/handlers/monitor.py b/app/monitoring/handlers/monitor.py
new file mode 100755
index 0000000..e147a7d
--- /dev/null
+++ b/app/monitoring/handlers/monitor.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+
+# handle monitoring events
+
+import argparse
+import json
+import sys
+
+from utils.mongo_access import MongoAccess
+from utils.util import ClassResolver
+
+DEFAULTS = {
+    'env': 'WebEX-Mirantis@Cisco',
+    'inventory': 'inventory',
+    'loglevel': 'WARNING'
+}
+
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-m", "--mongo_config", nargs="?", type=str,
+                        default="",
+                        help="name of config file with MongoDB server " +
+                        "access details")
+    parser.add_argument("-e", "--env", nargs="?", type=str,
+                        default=DEFAULTS['env'],
+                        help="name of environment to scan \n" +
+                        "(default: {})".format(DEFAULTS['env']))
+    parser.add_argument("-y", "--inventory", nargs="?", type=str,
+                        default=DEFAULTS['inventory'],
+                        help="name of inventory collection \n" +
+                        "(default: {}".format(DEFAULTS['inventory']))
+    parser.add_argument('-i', '--inputfile', nargs='?', type=str,
+                        default='',
+                        help="read input from the specifed file \n" +
+                        "(default: from stdin)")
+    parser.add_argument("-l", "--loglevel", nargs="?", type=str,
+                        default=DEFAULTS["loglevel"],
+                        help="logging level \n(default: '{}')"
+                        .format(DEFAULTS["loglevel"]))
+    args = parser.parse_args()
+    return args
+
+input = None
+args = get_args()
+MongoAccess.set_config_file(args.mongo_config)
+if args.inputfile:
+    try:
+        with open(args.inputfile, 'r') as input_file:
+            input = input_file.read()
+    except Exception as e:
+        raise FileNotFoundError("failed to open input file: " + args.inputfile)
+        exit(1)
+else:
+    input = sys.stdin.read()
+    if not input:
+        raise ValueError("No input provided on stdin")
+        exit(1)
+
+check_result_full = json.loads(input)
+check_client = check_result_full['client']
+check_result = check_result_full['check']
+check_result['id'] = check_result_full['id']
+name = check_result['name']
+status = check_result['status']
+object_type = name[:name.index('_')]
+object_id = name[name.index('_')+1:]
+if 'environment' in check_client:
+    args.env = check_client['environment']
+
+handler = None
+basic_handling_types = ['vedge', 'vservice']
+if object_type in basic_handling_types:
+    from monitoring.handlers.basic_check_handler import BasicCheckHandler
+    handler = BasicCheckHandler(args)
+else:
+    module_name = 'handle_' + object_type
+    handler = ClassResolver.get_instance_single_arg(args,
+                                                    module_name=module_name,
+                                                    package_name='monitoring.handlers')
+if handler:
+    handler.handle(object_id, check_result)
diff --git a/app/monitoring/handlers/monitoring_check_handler.py b/app/monitoring/handlers/monitoring_check_handler.py
new file mode 100644
index 0000000..51769ab
--- /dev/null
+++ b/app/monitoring/handlers/monitoring_check_handler.py
@@ -0,0 +1,94 @@
+###############################################################################
+# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems)   #
+# and others                                                                  #
+#                                                                             #
+# All rights reserved. This program and the accompanying materials            #
+# are made available under the terms of the Apache License, Version 2.0       #
+# which accompanies this distribution, and is available at                    #
+# http://www.apache.org/licenses/LICENSE-2.0                                  #
+###############################################################################
+# handle monitoring event
+import datetime
+import sys
+from time import gmtime, strftime
+
+from bson import ObjectId
+
+from discover.configuration import Configuration
+from messages.message import Message
+from utils.inventory_mgr import InventoryMgr
+from utils.logging.full_logger import FullLogger
+from utils.special_char_converter import SpecialCharConverter
+from utils.string_utils import stringify_datetime
+
+TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
+SOURCE_SYSTEM = 'Sensu'
+ERROR_LEVEL = ['info', 'warn', 'error']
+
+
+class MonitoringCheckHandler(SpecialCharConverter):
+    STATUS_LABEL = ['OK', 'Warning', 'Critical']
+
+    def __init__(self, args):
+        super().__init__()
+        self.log = FullLogger()
+        self.log.set_loglevel(args.loglevel)
+        self.env = args.env
+        try:
+            self.conf = Configuration(args.mongo_config)
+            self.inv = InventoryMgr()
+            self.inv.log.set_loglevel(args.loglevel)
+            self.inv.set_collections(args.inventory)
+        except FileNotFoundError:
+            sys.exit(1)
+
+    def doc_by_id(self, object_id):
+        doc = self.inv.get_by_id(self.env, object_id)
+        if not doc:
+            self.log.warn('No matching object found with ID: ' + object_id)
+        return doc
+
+    def doc_by_db_id(self, db_id, coll_name=None):
+        coll = self.inv.collections[coll_name] if coll_name else None
+        doc = self.inv.find({'_id': ObjectId(db_id)},
+                            get_single=True, collection=coll)
+        if not doc:
+            self.log.warn('No matching object found with DB ID: ' + db_id)
+        return doc
+
+    def set_doc_status(self, doc, status, status_text, timestamp):
+        doc['status'] = self.STATUS_LABEL[status] if isinstance(status, int) \
+            else status
+        if status_text:
+            doc['status_text'] = status_text
+        doc['status_timestamp'] = strftime(TIME_FORMAT, timestamp)
+        if 'link_type' in doc:
+            self.inv.write_link(doc)
+        else:
+            self.inv.set(doc)
+
+    @staticmethod
+    def check_ts(check_result):
+        return gmtime(check_result['executed'])
+
+    def keep_result(self, doc, check_result):
+        status = check_result['status']
+        ts = self.check_ts(check_result)
+        self.set_doc_status(doc, status, check_result['output'], ts)
+        self.keep_message(doc, check_result)
+
+    def keep_message(self, doc, check_result, error_level=None):
+        msg_id = check_result['id']
+        obj_id = doc['id']
+        display_context = doc['network_id'] if doc['type'] == 'port'\
+            else doc['id']
+        level = error_level if error_level\
+            else ERROR_LEVEL[check_result['status']]
+        dt = datetime.datetime.utcfromtimestamp(check_result['executed'])
+        ts = stringify_datetime(dt)
+        message = Message(msg_id=msg_id, env=self.env, source=SOURCE_SYSTEM,
+                          object_id=obj_id, object_type=doc['type'],
+                          display_context=display_context, level=level,
+                          msg=check_result, ts=ts)
+        collection = self.inv.collections['messages']
+        collection.insert_one(message.get())