diff options
Diffstat (limited to 'sdv/docker/sdvstate/internal/validator/airship')
8 files changed, 50 insertions, 163 deletions
diff --git a/sdv/docker/sdvstate/internal/validator/airship/__init__.py b/sdv/docker/sdvstate/internal/validator/airship/__init__.py index 01abeeb..4b58500 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/__init__.py +++ b/sdv/docker/sdvstate/internal/validator/airship/__init__.py @@ -50,5 +50,3 @@ from .compute_check import vswitch_dpdk_lcores_check from .compute_check import os_reserved_cores_check from .compute_check import nova_scheduler_filters_check from .compute_check import cpu_allocation_ratio_check - -from .store_result import store_result diff --git a/sdv/docker/sdvstate/internal/validator/airship/airship.py b/sdv/docker/sdvstate/internal/validator/airship/airship.py index 3d49f64..8e0c6a4 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/airship.py +++ b/sdv/docker/sdvstate/internal/validator/airship/airship.py @@ -23,6 +23,7 @@ from datetime import datetime as dt from tools.conf import settings from tools.kube_utils import load_kube_api, delete_kube_curl_pod from internal.validator.validator import Validator +from internal import store_result from . import * @@ -67,10 +68,6 @@ class AirshipValidator(Validator): self._report['case_name'] = 'ook_airship' self.default_suite() - if test_suite == "k8s": - self._report['case_name'] = 'k8s_airship' - self.k8s_suite() - delete_kube_curl_pod() self._report['stop_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S') @@ -113,39 +110,11 @@ class AirshipValidator(Validator): self.update_report(nova_scheduler_filters_check()) self.update_report(cpu_allocation_ratio_check()) - def k8s_suite(self): - """ - Kubernetes Platform Test Suite - """ - - - def update_report(self, result): - """ - Updates report with new results - """ - case_name = result['case_name'] - criteria = result['criteria'] - - self._report['details']['total_checks'] += 1 - if criteria == 'pass': - self._report['details']['pass'].append(case_name) - elif criteria == 'fail': - self._report['details']['fail'].append(case_name) - self._report['criteria'] = 'fail' - - def get_report(self): """ Return final report as dict """ - self._report["project_name"] = settings.getValue("project_name") - self._report["version"] = settings.getValue("project_version") - self._report["build_tag"] = "none" - - pdf = settings.getValue('pdf_file') - self._report["pod_name"] = pdf['management_info']['resource_pool_name'] - - store_result(self._report) - + self._report = super(AirshipValidator, self).get_report() + store_result(self._logger, self._report) return self._report diff --git a/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py b/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py index b33e876..e8d254b 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py +++ b/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py @@ -17,9 +17,10 @@ Ceph Related Checks """ import ast +import logging from tools.kube_utils import get_pod_with_labels, kube_exec -from .store_result import store_result +from internal import store_result @@ -28,6 +29,7 @@ def ceph_health_check(): """ Check health of Ceph """ + logger = logging.getLogger(__name__) pod = get_pod_with_labels('application=ceph,component=mon') cmd = ['ceph', 'health', '-f', 'json'] @@ -47,5 +49,5 @@ def ceph_health_check(): result['criteria'] = 'fail' result['details'] = response - store_result(result) + store_result(logger, result) return result diff --git a/sdv/docker/sdvstate/internal/validator/airship/compute_check.py b/sdv/docker/sdvstate/internal/validator/airship/compute_check.py index a602471..4c0ea6f 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/compute_check.py +++ b/sdv/docker/sdvstate/internal/validator/airship/compute_check.py @@ -19,10 +19,11 @@ Compute Related Checks import configparser import json import re +import logging from tools.kube_utils import kube_exec, get_pod_with_labels from tools.conf import settings -from .store_result import store_result +from internal import store_result ########### @@ -33,6 +34,7 @@ def isolated_cores_check(): """ isolated_cores_check """ + logger = logging.getLogger(__name__) traced_value = trace_isolated_cores() required_value = required_isolated_cores() @@ -49,7 +51,7 @@ def isolated_cores_check(): result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -58,6 +60,7 @@ def reserved_vnf_cores_check(): """ reserved_vnf_cores_check """ + logger = logging.getLogger(__name__) traced_value = trace_reserved_vnf_cores() required_value = required_reserved_vnf_cores() @@ -74,7 +77,7 @@ def reserved_vnf_cores_check(): result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -83,6 +86,7 @@ def vswitch_pmd_cores_check(): """ vswitch_pmd_cores_check """ + logger = logging.getLogger(__name__) traced_value = trace_vswitch_pmd_cores() required_value = required_vswitch_pmd_cores() @@ -99,7 +103,7 @@ def vswitch_pmd_cores_check(): result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -108,6 +112,7 @@ def vswitch_dpdk_lcores_check(): """ vswitch_dpdk_lcores_check """ + logger = logging.getLogger(__name__) traced_value = trace_vswitch_dpdk_lcores() required_value = required_vswitch_dpdk_lcores() @@ -124,7 +129,7 @@ def vswitch_dpdk_lcores_check(): result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -133,6 +138,7 @@ def os_reserved_cores_check(): """ os_reserved_cores_check """ + logger = logging.getLogger(__name__) traced_value = trace_os_reserved_cores() required_value = required_os_reserved_cores() @@ -149,7 +155,7 @@ def os_reserved_cores_check(): result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -158,6 +164,7 @@ def nova_scheduler_filters_check(): """ nova_scheduler_filters_check """ + logger = logging.getLogger(__name__) traced_value = trace_nova_scheduler_filters() required_value = required_nova_scheduler_filters() @@ -173,7 +180,7 @@ def nova_scheduler_filters_check(): else: result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result @@ -182,6 +189,7 @@ def cpu_allocation_ratio_check(): """ cpu_allocation_ratio_check """ + logger = logging.getLogger(__name__) traced_value = trace_cpu_allocation_ratio() required_value = required_cpu_allocation_ratio() @@ -197,7 +205,7 @@ def cpu_allocation_ratio_check(): else: result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result diff --git a/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py b/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py index 3754299..e779a4b 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py +++ b/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py @@ -17,16 +17,18 @@ Monitoring & Logging Agents Related Checks """ import ast +import logging from tools.kube_utils import kube_curl from tools.result_api import rfile -from .store_result import store_result +from internal import store_result def prometheus_check(): """ Check health of Prometheus """ + logger = logging.getLogger(__name__) username = "prometheus" password = "password123" service = "prom-metrics" @@ -53,7 +55,7 @@ def prometheus_check(): 'details': {'health': health, 'readiness': readiness} } - store_result(result) + store_result(logger, result) return result @@ -62,6 +64,7 @@ def grafana_check(): """ Check health of Grafana """ + logger = logging.getLogger(__name__) username = "grafana" password = "password123" service = "grafana-dashboard" @@ -80,7 +83,7 @@ def grafana_check(): 'criteria': state } - store_result(result) + store_result(logger, result) return result @@ -88,6 +91,7 @@ def prometheus_alert_manager_check(): """ Check health of Alert Manager """ + logger = logging.getLogger(__name__) service = "alerts-engine" namespace = "osh-infra" @@ -113,7 +117,7 @@ def prometheus_alert_manager_check(): } - store_result(result) + store_result(logger, result) return result @@ -121,6 +125,7 @@ def elasticsearch_check(): """ Check health of Elasticsearch cluster """ + logger = logging.getLogger(__name__) username = "elasticsearch" password = "password123" service = "elasticsearch" @@ -142,7 +147,7 @@ def elasticsearch_check(): 'details': res } - store_result(result) + store_result(logger, result) return result @@ -150,6 +155,7 @@ def kibana_check(): """ Check health of Kibana """ + logger = logging.getLogger(__name__) username = "elasticsearch" password = "password123" service = "kibana-dash" @@ -171,7 +177,7 @@ def kibana_check(): 'details': rfile(str(res)) } - store_result(result) + store_result(logger, result) return result @@ -179,6 +185,7 @@ def nagios_check(): """ Check health of Nagios """ + logger = logging.getLogger(__name__) username = "nagios" password = "password123" service = "nagios-metrics" @@ -197,7 +204,7 @@ def nagios_check(): 'criteria': state } - store_result(result) + store_result(logger, result) return result @@ -205,6 +212,7 @@ def elasticsearch_exporter_check(): """ Check health of Elasticsearch Exporter """ + logger = logging.getLogger(__name__) service = "elasticsearch-exporter" namespace = "osh-infra" @@ -218,7 +226,7 @@ def elasticsearch_exporter_check(): 'criteria': state } - store_result(result) + store_result(logger, result) return result @@ -226,6 +234,7 @@ def fluentd_exporter_check(): """ Check health of Fluentd Exporter """ + logger = logging.getLogger(__name__) service = "fluentd-exporter" namespace = "osh-infra" @@ -239,5 +248,5 @@ def fluentd_exporter_check(): 'criteria': state } - store_result(result) + store_result(logger, result) return result diff --git a/sdv/docker/sdvstate/internal/validator/airship/network_check.py b/sdv/docker/sdvstate/internal/validator/airship/network_check.py index bddf579..83aa796 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/network_check.py +++ b/sdv/docker/sdvstate/internal/validator/airship/network_check.py @@ -16,19 +16,19 @@ Network Related Checks """ - +import logging import configparser +from internal import store_result from tools.conf import settings from tools.kube_utils import kube_exec, get_pod_with_labels -from .store_result import store_result - def physical_network_check(): """ physical_network_check """ + logger = logging.getLogger(__name__) ml2_config = neutron_ml2_config() physical_networks = settings.getValue('pdf_file')['physical_networks'] @@ -73,7 +73,7 @@ def physical_network_check(): if res['criteria'] == 'fail': result['criteria'] = 'fail' - store_result(result) + store_result(logger, result) return result diff --git a/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py b/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py index 0093ffc..ab38f84 100644 --- a/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py +++ b/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py @@ -17,15 +17,12 @@ Pod Health Checks """ - - import logging +from internal import checks +from internal import store_result from tools.kube_utils import kube_api from tools.conf import settings -from tools.result_api import rfile - -from .store_result import store_result @@ -33,79 +30,11 @@ def pod_health_check(): """ Check health of all pods and get logs of failed pods """ + logger = logging.getLogger(__name__) api = kube_api() namespace_list = settings.getValue('airship_namespace_list') - result = {'category': 'platform', - 'case_name': 'pod_health_check', - 'criteria': 'pass', - 'details': [] - } - - for namespace in namespace_list: - pod_list = api.list_namespaced_pod(namespace) - for pod in pod_list.items: - pod_stats = pod_status(pod) - if pod_stats['criteria'] == 'fail': - pod_stats['logs'] = get_logs(pod) - result['criteria'] = 'fail' - result['details'].append(pod_stats) - - - store_result(result) - return result - - - -def pod_status(pod): - """ - Check health of a pod and returns it's status as result - """ - result = {'criteria': 'pass', - 'name': pod.metadata.name, - 'namespace': pod.metadata.namespace, - 'node': pod.spec.node_name} - - if pod.status.container_statuses is None: - result['criteria'] = 'fail' - result['pod_details'] = rfile(str(pod)) - else: - for container in pod.status.container_statuses: - if container.state.running is not None: - status = 'Running' - if container.state.terminated is not None: - status = container.state.terminated.reason - if container.state.waiting is not None: - status = container.state.waiting.reason + result = checks.pod_health_check(logger, api, namespace_list) - if status not in ('Running', 'Completed'): - result['criteria'] = 'fail' - result['pod_details'] = rfile(str(pod)) - - info = f'[Health: {result["criteria"]}] Name: {result["name"]}, ' - info = info + f'Namespace: {result["namespace"]}, Node: {result["node"]}' - - logger = logging.getLogger(__name__) - logger.debug(info) + store_result(logger, result) return result - - -def get_logs(pod): - """ - Collects logs of all containers in ``pod`` - """ - api = kube_api() - logs = [] - if pod.status.container_statuses is not None: - for container in pod.status.container_statuses: - con = {'container': container.name} - if container.state.waiting is not None and \ - container.state.waiting.reason == 'PodInitializing': - log = 'Not found, status: waiting, reason: PodInitializing' - else: - log = api.read_namespaced_pod_log(name=pod.metadata.name, - namespace=pod.metadata.namespace, - container=container.name) - con['log'] = rfile(log) - logs.append(con) - return logs diff --git a/sdv/docker/sdvstate/internal/validator/airship/store_result.py b/sdv/docker/sdvstate/internal/validator/airship/store_result.py deleted file mode 100644 index 52f4e10..0000000 --- a/sdv/docker/sdvstate/internal/validator/airship/store_result.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2020 University Of Delhi. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -store_result function to log and store result -""" -import logging -from tools.result_api import result_api - -def store_result(result): - """ - Logs and stores result - """ - logger = logging.getLogger(__name__) - logger.info(f'[State: {result["criteria"]}] {result["case_name"]}') - - result_api.store(result) |