From 9fa1356bf9aabd50c4adcec082eedf1410f9a7a7 Mon Sep 17 00:00:00 2001 From: Parth Yadav Date: Wed, 26 Aug 2020 00:08:29 +0530 Subject: Add support for TestAPI, Reporting and new checks Adds following new checks: * ceph_health_check * prometheus_check * grafana_check * elasticsearch_check * kibana_check * nagios_check * elasticsearch_exporter_check * fluentd_exporter_check * physical_network_check * reserved_vnf_cores_check * isolated_cores_check * vswitch_pmd_cores_check * vswitch_dpdk_lcore_check * os_reserved_cores_check * nova_scheduler_filters_check * cpu_allocation_ratio_check Updated result reporting format to match with TestAPI result formating Signed-off-by: Parth Yadav Change-Id: I421fd20067d289e8735a2ed3f402c68f45ae69e9 --- .../airship/monitoring_logging_agent_check.py | 243 +++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py (limited to 'sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py') diff --git a/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py b/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py new file mode 100644 index 0000000..3754299 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py @@ -0,0 +1,243 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Monitoring & Logging Agents Related Checks +""" + +import ast + +from tools.kube_utils import kube_curl +from tools.result_api import rfile +from .store_result import store_result + + +def prometheus_check(): + """ + Check health of Prometheus + """ + username = "prometheus" + password = "password123" + service = "prom-metrics" + namespace = "osh-infra" + + health = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/healthy') + if 'Prometheus is Healthy' in res: + health = "pass" + + readiness = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/ready') + if 'Prometheus is Ready' in res: + readiness = "pass" + + if health == "pass" and readiness == "pass": + state = "pass" + else: + state = "fail" + + result = {'category': 'platform', + 'case_name': 'prometheus_check', + 'criteria': state, + 'details': {'health': health, 'readiness': readiness} + } + + store_result(result) + return result + + + +def grafana_check(): + """ + Check health of Grafana + """ + username = "grafana" + password = "password123" + service = "grafana-dashboard" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ + "-o", "/dev/null", "-u", \ + f'{username}:{password}', \ + f'{service}.{namespace}:3000/api/health') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'grafana_check', + 'criteria': state + } + + store_result(result) + return result + + +def prometheus_alert_manager_check(): + """ + Check health of Alert Manager + """ + service = "alerts-engine" + namespace = "osh-infra" + + health = "fail" #default + res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/healthy') + if 'Prometheus is Healthy' in res: + health = "pass" + + readiness = "fail" #default + res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/ready') + if 'Prometheus is Ready' in res: + readiness = "pass" + + if health == "pass" and readiness == "pass": + state = "pass" + else: + state = "fail" + + result = {'category': 'platform', + 'case_name': 'prometheus_alert_manager_check', + 'criteria': state, + 'details': {'health': health, 'readiness': readiness} + } + + + store_result(result) + return result + + +def elasticsearch_check(): + """ + Check health of Elasticsearch cluster + """ + username = "elasticsearch" + password = "password123" + service = "elasticsearch" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/_cluster/health') + + if res == '': + res = 'Elasticsearch not reachable' + else: + res = ast.literal_eval(res) + if res['status'] == 'green': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'elasticsearch_check', + 'criteria': state, + 'details': res + } + + store_result(result) + return result + + +def kibana_check(): + """ + Check health of Kibana + """ + username = "elasticsearch" + password = "password123" + service = "kibana-dash" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/api/status') + + if res == '': + res = 'kibana not reachable' + else: + res = ast.literal_eval(res) + if res['status']['overall']['state'] == 'green': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'kibana_check', + 'criteria': state, + 'details': rfile(str(res)) + } + + store_result(result) + return result + + +def nagios_check(): + """ + Check health of Nagios + """ + username = "nagios" + password = "password123" + service = "nagios-metrics" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ + "-o", "/dev/null", "-u", \ + f'{username}:{password}', \ + f'{service}.{namespace}') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'nagios_check', + 'criteria': state + } + + store_result(result) + return result + + +def elasticsearch_exporter_check(): + """ + Check health of Elasticsearch Exporter + """ + service = "elasticsearch-exporter" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9108/metrics') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'elasticsearch_exporter_check', + 'criteria': state + } + + store_result(result) + return result + + +def fluentd_exporter_check(): + """ + Check health of Fluentd Exporter + """ + service = "fluentd-exporter" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9309/metrics') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'fluentd_exporter_check', + 'criteria': state + } + + store_result(result) + return result -- cgit 1.2.3-korg