diff options
Diffstat (limited to 'sdv/docker/sdvstate/validator/airship')
8 files changed, 1244 insertions, 20 deletions
diff --git a/sdv/docker/sdvstate/validator/airship/__init__.py b/sdv/docker/sdvstate/validator/airship/__init__.py new file mode 100644 index 0000000..78e42c4 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/__init__.py @@ -0,0 +1,49 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Package for Airship +""" + + +### Pod Health Checks +from .pod_health_check import pod_health_check + +### Ceph Health Checks +from .ceph_check import ceph_health_check + +### Monitoring & Logging Agents Checks +from .monitoring_logging_agent_check import prometheus_check +from .monitoring_logging_agent_check import grafana_check +# from .monitoring_logging_agent_check import prometheus_alert_manager_check +from .monitoring_logging_agent_check import elasticsearch_check +from .monitoring_logging_agent_check import kibana_check +from .monitoring_logging_agent_check import nagios_check +from .monitoring_logging_agent_check import elasticsearch_exporter_check +from .monitoring_logging_agent_check import fluentd_exporter_check + +### Network Checks +from .network_check import physical_network_check + +### Compute Related Checks +from .compute_check import reserved_vnf_cores_check +from .compute_check import isolated_cores_check +from .compute_check import vswitch_pmd_cores_check +from .compute_check import vswitch_dpdk_lcores_check +from .compute_check import os_reserved_cores_check +from .compute_check import nova_scheduler_filters_check +from .compute_check import cpu_allocation_ratio_check + +from .store_result import store_result diff --git a/sdv/docker/sdvstate/validator/airship/airship.py b/sdv/docker/sdvstate/validator/airship/airship.py index e77f06f..18de66d 100644 --- a/sdv/docker/sdvstate/validator/airship/airship.py +++ b/sdv/docker/sdvstate/validator/airship/airship.py @@ -18,16 +18,15 @@ Airship Validator """ import logging -import ast -import json +from datetime import datetime as dt from tools.conf import settings -from tools.result_api import result_api, rfile -from tools.kube_utils import * +from tools.kube_utils import load_kube_api from validator.validator import Validator -## Checks -from .pod_health_check import pod_health_check +from . import * + + @@ -42,10 +41,87 @@ class AirshipValidator(Validator): super(AirshipValidator, self).__init__() self._logger = logging.getLogger(__name__) + self._report = {"installer": "Airship", + "criteria": "pass", + "details": {"total_checks": 0, + "pass": [], + "fail": [], + "metadata": {} + } + } + load_kube_api() - + def validate(self): """ + Validation method """ - pod_health_check() + + self._report['scenario'] = 'none' + self._report['case_name'] = 'ook_airship' + self._report['start_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S') + + + # PLATFORM CHECKS + self.update_report(pod_health_check()) + + # STORAGE CHECKS + self.update_report(ceph_health_check()) + + # MONITORING & LOGGING AGENTS CHECKS + self.update_report(prometheus_check()) + self.update_report(grafana_check()) + ## current version of AlertManager doesn't support this + # prometheus_alert_manager_check() + self.update_report(elasticsearch_check()) + self.update_report(kibana_check()) + self.update_report(nagios_check()) + self.update_report(elasticsearch_exporter_check()) + self.update_report(fluentd_exporter_check()) + + # NETWORK CHECKS + self.update_report(physical_network_check()) + + # COMPUTE CHECKS + self.update_report(reserved_vnf_cores_check()) + self.update_report(isolated_cores_check()) + self.update_report(vswitch_pmd_cores_check()) + self.update_report(vswitch_dpdk_lcores_check()) + self.update_report(os_reserved_cores_check()) + self.update_report(nova_scheduler_filters_check()) + self.update_report(cpu_allocation_ratio_check()) + + self._report['stop_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S') + + + def update_report(self, result): + """ + Updates report with new results + """ + case_name = result['case_name'] + criteria = result['criteria'] + + self._report['details']['total_checks'] += 1 + if criteria == 'pass': + self._report['details']['pass'].append(case_name) + elif criteria == 'fail': + self._report['details']['fail'].append(case_name) + self._report['criteria'] = 'fail' + + + + def get_report(self): + """ + Return final report as dict + """ + self._report["project_name"] = settings.getValue("project_name") + self._report["version"] = settings.getValue("project_version") + self._report["build_tag"] = "none" + + pdf = settings.getValue('pdf_file') + self._report["pod_name"] = pdf['management_info']['resource_pool_name'] + + store_result(self._report) + + return self._report diff --git a/sdv/docker/sdvstate/validator/airship/ceph_check.py b/sdv/docker/sdvstate/validator/airship/ceph_check.py new file mode 100644 index 0000000..b33e876 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/ceph_check.py @@ -0,0 +1,51 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Ceph Related Checks +""" + +import ast + +from tools.kube_utils import get_pod_with_labels, kube_exec +from .store_result import store_result + + + + +def ceph_health_check(): + """ + Check health of Ceph + """ + pod = get_pod_with_labels('application=ceph,component=mon') + + cmd = ['ceph', 'health', '-f', 'json'] + response = kube_exec(pod, cmd) + + response = ast.literal_eval(response) + + result = {'category': 'storage', + 'case_name': 'ceph_health_check', + 'details': [] + } + + if response['status'] == 'HEALTH_OK': + result['criteria'] = 'pass' + result['details'] = 'HEALTH_OK' + else: + result['criteria'] = 'fail' + result['details'] = response + + store_result(result) + return result diff --git a/sdv/docker/sdvstate/validator/airship/compute_check.py b/sdv/docker/sdvstate/validator/airship/compute_check.py new file mode 100644 index 0000000..ff6f6db --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/compute_check.py @@ -0,0 +1,646 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Compute Related Checks +""" + +import configparser +import json + +from tools.kube_utils import kube_exec, get_pod_with_labels +from tools.conf import settings +from .store_result import store_result + + +########### +# Checks +########### + +def isolated_cores_check(): + """ + isolated_cores_check + """ + traced_value = trace_isolated_cores() + required_value = required_isolated_cores() + + result = {'category': 'compute', + 'case_name': 'isolated_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def reserved_vnf_cores_check(): + """ + reserved_vnf_cores_check + """ + traced_value = trace_reserved_vnf_cores() + required_value = required_reserved_vnf_cores() + + result = {'category': 'compute', + 'case_name': 'reserved_vnf_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def vswitch_pmd_cores_check(): + """ + vswitch_pmd_cores_check + """ + traced_value = trace_vswitch_pmd_cores() + required_value = required_vswitch_pmd_cores() + + result = {'category': 'compute', + 'case_name': 'vswitch_pmd_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def vswitch_dpdk_lcores_check(): + """ + vswitch_dpdk_lcores_check + """ + traced_value = trace_vswitch_dpdk_lcores() + required_value = required_vswitch_dpdk_lcores() + + result = {'category': 'compute', + 'case_name': 'vswitch_dpdk_lcores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def os_reserved_cores_check(): + """ + os_reserved_cores_check + """ + traced_value = trace_os_reserved_cores() + required_value = required_os_reserved_cores() + + result = {'category': 'compute', + 'case_name': 'os_reserved_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def nova_scheduler_filters_check(): + """ + nova_scheduler_filters_check + """ + traced_value = trace_nova_scheduler_filters() + required_value = required_nova_scheduler_filters() + + result = {'category': 'compute', + 'case_name': 'nova_scheduler_filters_check', + 'details': {'traced_filters': traced_value, + 'required_filters': required_value + } + } + + if are_lists_equal(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + store_result(result) + return result + + + +def cpu_allocation_ratio_check(): + """ + cpu_allocation_ratio_check + """ + traced_value = trace_cpu_allocation_ratio() + required_value = required_cpu_allocation_ratio() + + result = {'category': 'compute', + 'case_name': 'cpu_allocation_ratio_check', + 'details': {'traced_ratio': traced_value, + 'required_ratio': required_value + } + } + + if traced_value == required_value: + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + store_result(result) + return result + + + + + + + + +############### +# helper functions +############### + + + +def trace_isolated_cores(): + """ + Trace isolated_cores from Airship deployment + + :return: value traced from `isolcpus` key in `/proc/cmdline` + """ + pod = get_pod_with_labels('application=nova,component=compute') + + cmd = ['cat', '/proc/cmdline'] + proc_cmd = kube_exec(pod, cmd) + + for option in proc_cmd.split(): + if 'isolcpus' in option: + _, isolcpus_value = split_key_value(option) + break + + return isolcpus_value + + +def required_isolated_cores(): + """ + Returns value of `isolated_cpus` from platform_profile used by + Role for worker nodes in PDF + + :return: isolated_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['isolated_cpus'] + + + + + + +def trace_reserved_vnf_cores(): + """ + Trace vnf_reserved_cores from Airship deployment + + :return: value traced from `vcpu_pin_set` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + vcpu_pin_set = config.get('DEFAULT', 'vcpu_pin_set') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + vcpu_pin_set = '' + + return vcpu_pin_set + + +def required_reserved_vnf_cores(): + """ + Returns value of vnf_cores from platform_profile used by + Role for worker nodes in PDF + + :return: vnf_reserverd_core value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vnf_cores'] + + + + + + +def trace_vswitch_pmd_cores(): + """ + Trace vswitch_pmd_cores from Airship deployment + + :return: value traced from `other_config:pmd-cpu-mask` in + openvswitchdb using ovs-vsctl + """ + ovs_pod = get_pod_with_labels('application=openvswitch,component=openvswitch-vswitchd') + + cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config'] + response = kube_exec(ovs_pod, cmd) + + response.replace('=', ':') + config = json.loads(response) + + if 'pmd-cpu-mask' in config: + pmd_cores = hex_to_comma_list(config['pmd-cpu-mask']) + else: + pmd_cores = '' + + return pmd_cores + + +def required_vswitch_pmd_cores(): + """ + Returns value of vswitch_pmd_cores from platform_profile used by + Role for worker nodes in PDF + + :return: vswitch_pmd_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vswitch_pmd_cores'] + + + + + + +def trace_vswitch_dpdk_lcores(): + """ + Trace vswitch_dpdk_lcores from Airship deployment + + :return: value traced from `other_config:dpdk-lcore-mask` in + openvswitchdb using ovs-vsctl + """ + ovs_pod = get_pod_with_labels('application=openvswitch,component=openvswitch-vswitchd') + + cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config'] + response = kube_exec(ovs_pod, cmd) + + response.replace('=', ':') + config = json.loads(response) + + if 'dpdk-lcore-mask' in config: + pmd_cores = hex_to_comma_list(config['dpdk-lcore-mask']) + else: + pmd_cores = '' + + return pmd_cores + + +def required_vswitch_dpdk_lcores(): + """ + Returns value of vswitch_dpdk_lcores from platform_profile used by + Role for worker nodes in PDF + + :return: vswitch_dpdk_lcores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vswitch_dpdk_lcores'] + + + + + + +def trace_os_reserved_cores(): + """ + Trace os_reserved_cores from Airship deployment + + os_reserved_cores = all_cores - (reserved_vnf_cores + + vswitch_pmd_cores + + vswitch_dpdk_lcores) + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + all_cores = get_cores_by_role(worker_role) + + reserved_vnf_cores = trace_reserved_vnf_cores() + vswitch_pmd_cores = trace_vswitch_pmd_cores() + vswitch_dpdk_lcores = trace_vswitch_dpdk_lcores() + + non_os_cores = [] + non_os_cores.extend(convert_range_to_list(reserved_vnf_cores)) + non_os_cores.extend(convert_range_to_list(vswitch_pmd_cores)) + non_os_cores.extend(convert_range_to_list(vswitch_dpdk_lcores)) + + os_reserved_cores = set(all_cores).difference(set(non_os_cores)) + + # return as string with comma separated value + return ','.join(map(str, list(os_reserved_cores))) + + +def required_os_reserved_cores(): + """ + Returns value of os_reserved_cores from platform_profile used by + Role for worker nodes in PDF + + :return: os_reserved_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['os_reserved_cores'] + + + + + +def trace_nova_scheduler_filters(): + """ + Trace scheduler_filters from Airship deployment + + :return: value traced from `enabled_filters` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + filters = config.get('filter_scheduler', 'enabled_filters') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + filters = '' + + filters = filters.split(',') + map(str.strip, filters) + + return filters + +def required_nova_scheduler_filters(): + """ + Required nova scheduler_filters by the PDF + """ + pdf = settings.getValue('pdf_file') + filters = pdf['vim_functional']['scheduler_filters'] + + filters = filters.split(',') + map(str.strip, filters) + + return filters + + + + + + + +def trace_cpu_allocation_ratio(): + """ + Trace cpu_allocation_ratio from Airship deployment + + :return: value traced from `cpu_allocation_ratio` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + cpu_allocation_ratio = config.get('DEFAULT', 'cpu_allocation_ratio') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + cpu_allocation_ratio = '' + + return float(cpu_allocation_ratio) + +def required_cpu_allocation_ratio(): + """ + Required cpu_allocation_ratio by the PDF + """ + pdf = settings.getValue('pdf_file') + cpu_allocation_ratio = pdf['vim_functional']['cpu_allocation_ratio'] + + return float(cpu_allocation_ratio) + + + + + + + +def get_role(role_name): + """ + Searches and returns role with `role_name` + """ + roles = settings.getValue('pdf_file')['roles'] + + for role in roles: + if role['name'] == role_name: + role_details = role + + return role_details + + +def get_platform_profile(profile_name): + """ + Searches and returns platform_profile with `profile_name` + """ + platform_profiles = settings.getValue('pdf_file')['platform_profiles'] + + for profile in platform_profiles: + if profile['profile_name'] == profile_name: + profile_details = profile + + return profile_details + +def get_processor_profile(profile_name): + """ + Searches and returns processor_profile with `profile_name` + """ + processor_profiles = settings.getValue('pdf_file')['processor_profiles'] + + for profile in processor_profiles: + if profile['profile_name'] == profile_name: + profile_details = profile + + return profile_details + +def get_platform_profile_by_role(role_name): + """ + Returns platform profile details of a role + """ + role = get_role(role_name) + profile = get_platform_profile(role['platform_profile']) + return profile + + +def get_hardware_profile_by_role(role_name): + """ + Returns hardware profile details of a role + """ + role = get_role(role_name) + + hardware_profiles = settings.getValue('pdf_file')['hardware_profiles'] + + for profile in hardware_profiles: + if profile['profile_name'] == role['hardware_profile']: + profile_details = profile + + return profile_details + + +def get_cores_by_role(role_name): + """ + Returns cpu cores list of server hardware used in the role + """ + hardware_profile = get_hardware_profile_by_role(role_name) + processor_profile = hardware_profile['profile_info']['processor_profile'] + profile = get_processor_profile(processor_profile) + + cpus = [] + + for numa in profile['profile_info']['numas']: + cpus.extend(convert_range_to_list(numa['cpu_set'])) + + return cpus + + + + + + + +def get_nova_conf(): + """ + Returns parsed nova.conf + """ + pod = get_pod_with_labels('application=nova,component=compute') + + cmd = ['cat', '/etc/nova/nova.conf'] + response = kube_exec(pod, cmd) + + config = configparser.ConfigParser() + config.read_string(response) + + return config + + +### cpu cores related helper function + +def convert_range_to_list(x): + """ + Returns list of numbers from given range as string + + e.g.: convert_range_to_list('3-5') will give [3, 4, 5] + """ + # pylint: disable=C0103 + result = [] + for part in x.split(','): + if '-' in part: + a, b = part.split('-') + a, b = int(a), int(b) + result.extend(range(a, b + 1)) + elif part != '': + a = int(part) + result.append(a) + # remove duplicates + result = list(dict.fromkeys(result)) + return result + + +def is_ranges_equals(range1, range2): + """ + Checks whether two ranges passed as string are equal + + e.g.: is_ranges_equals('2-5', '2-4,5') returns true + """ + set1 = set(convert_range_to_list(range1)) + set2 = set(convert_range_to_list(range2)) + return set1 == set2 + +def are_lists_equal(list1, list2): + """ + Checks whether two list are identicals + """ + set1 = set(list1) + set2 = set(list2) + return set1 == set2 + + + +def hex_to_comma_list(hex_mask): + """ + Converts CPU mask given in hex to list of cores + """ + binary = bin(int(hex_mask, 16))[2:] + reversed_binary = binary[::-1] + i = 0 + output = "" + for bit in reversed_binary: + if bit == '1': + output = output + str(i) + ',' + i = i + 1 + return output[:-1] + + +def comma_list_to_hex(cpus): + """ + Converts a list of cpu cores in corresponding hex value + of cpu-mask + """ + cpu_arr = cpus.split(",") + binary_mask = 0 + for cpu in cpu_arr: + binary_mask = binary_mask | (1 << int(cpu)) + return format(binary_mask, '02x') + + + +def split_key_value(key_value_str, delimiter='='): + """ + splits given string into key and value based on delimiter + + :param key_value_str: example string `someKey=somevalue` + :param delimiter: default delimiter is `=` + :return: [ key, value] + """ + key, value = key_value_str.split(delimiter) + key = key.strip() + value = value.strip() + return key, value diff --git a/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py b/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py new file mode 100644 index 0000000..3754299 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py @@ -0,0 +1,243 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Monitoring & Logging Agents Related Checks +""" + +import ast + +from tools.kube_utils import kube_curl +from tools.result_api import rfile +from .store_result import store_result + + +def prometheus_check(): + """ + Check health of Prometheus + """ + username = "prometheus" + password = "password123" + service = "prom-metrics" + namespace = "osh-infra" + + health = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/healthy') + if 'Prometheus is Healthy' in res: + health = "pass" + + readiness = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/-/ready') + if 'Prometheus is Ready' in res: + readiness = "pass" + + if health == "pass" and readiness == "pass": + state = "pass" + else: + state = "fail" + + result = {'category': 'platform', + 'case_name': 'prometheus_check', + 'criteria': state, + 'details': {'health': health, 'readiness': readiness} + } + + store_result(result) + return result + + + +def grafana_check(): + """ + Check health of Grafana + """ + username = "grafana" + password = "password123" + service = "grafana-dashboard" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ + "-o", "/dev/null", "-u", \ + f'{username}:{password}', \ + f'{service}.{namespace}:3000/api/health') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'grafana_check', + 'criteria': state + } + + store_result(result) + return result + + +def prometheus_alert_manager_check(): + """ + Check health of Alert Manager + """ + service = "alerts-engine" + namespace = "osh-infra" + + health = "fail" #default + res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/healthy') + if 'Prometheus is Healthy' in res: + health = "pass" + + readiness = "fail" #default + res = kube_curl("-sL", "-m", "3", f'{service}.{namespace}:9093/-/ready') + if 'Prometheus is Ready' in res: + readiness = "pass" + + if health == "pass" and readiness == "pass": + state = "pass" + else: + state = "fail" + + result = {'category': 'platform', + 'case_name': 'prometheus_alert_manager_check', + 'criteria': state, + 'details': {'health': health, 'readiness': readiness} + } + + + store_result(result) + return result + + +def elasticsearch_check(): + """ + Check health of Elasticsearch cluster + """ + username = "elasticsearch" + password = "password123" + service = "elasticsearch" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/_cluster/health') + + if res == '': + res = 'Elasticsearch not reachable' + else: + res = ast.literal_eval(res) + if res['status'] == 'green': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'elasticsearch_check', + 'criteria': state, + 'details': res + } + + store_result(result) + return result + + +def kibana_check(): + """ + Check health of Kibana + """ + username = "elasticsearch" + password = "password123" + service = "kibana-dash" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-u", f'{username}:{password}', f'{service}.{namespace}/api/status') + + if res == '': + res = 'kibana not reachable' + else: + res = ast.literal_eval(res) + if res['status']['overall']['state'] == 'green': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'kibana_check', + 'criteria': state, + 'details': rfile(str(res)) + } + + store_result(result) + return result + + +def nagios_check(): + """ + Check health of Nagios + """ + username = "nagios" + password = "password123" + service = "nagios-metrics" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}",\ + "-o", "/dev/null", "-u", \ + f'{username}:{password}', \ + f'{service}.{namespace}') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'nagios_check', + 'criteria': state + } + + store_result(result) + return result + + +def elasticsearch_exporter_check(): + """ + Check health of Elasticsearch Exporter + """ + service = "elasticsearch-exporter" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9108/metrics') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'elasticsearch_exporter_check', + 'criteria': state + } + + store_result(result) + return result + + +def fluentd_exporter_check(): + """ + Check health of Fluentd Exporter + """ + service = "fluentd-exporter" + namespace = "osh-infra" + + state = "fail" #default + res = kube_curl("-sL", "-m", "3", "-w", "%{http_code}", "-o", "/dev/null", f'{service}.{namespace}:9309/metrics') + if res == '200': + state = "pass" + + result = {'category': 'platform', + 'case_name': 'fluentd_exporter_check', + 'criteria': state + } + + store_result(result) + return result diff --git a/sdv/docker/sdvstate/validator/airship/network_check.py b/sdv/docker/sdvstate/validator/airship/network_check.py new file mode 100644 index 0000000..bddf579 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/network_check.py @@ -0,0 +1,114 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Network Related Checks +""" + + +import configparser + +from tools.conf import settings +from tools.kube_utils import kube_exec, get_pod_with_labels + +from .store_result import store_result + + +def physical_network_check(): + """ + physical_network_check + """ + ml2_config = neutron_ml2_config() + + physical_networks = settings.getValue('pdf_file')['physical_networks'] + + type_drivers = ml2_config.get('ml2', 'type_drivers').split(',') + + flat_networks = ml2_config.get('ml2_type_flat', 'flat_networks').split(',') + + vlan_networks = [] + network_vlan_ranges = ml2_config.get('ml2_type_vlan', 'network_vlan_ranges').split(',') + for network in network_vlan_ranges: + vlan_networks.append(network.split(':')[0]) + + result = {'category': 'network', + 'case_name': 'physical_network_check', + 'criteria': 'pass', + 'details': [] + } + + for physnet in physical_networks: + + res = {'network_name': physnet['name'], + 'type': physnet['type'], + 'criteria': 'fail' + } + + if physnet['type'] in type_drivers: + if physnet['type'] == 'flat': + if physnet['name'] in flat_networks or '*' in flat_networks: + res['criteria'] = 'pass' + else: + res['details'] = 'physical network name not found' + if physnet['type'] == 'vlan': + if physnet['name'] in vlan_networks: + res['criteria'] = 'pass' + else: + res['details'] = 'physical network name not found' + else: + res['details'] = 'physical network type not found' + + result['details'].append(res) + if res['criteria'] == 'fail': + result['criteria'] = 'fail' + + store_result(result) + return result + + + +def neutron_ml2_config(): + """ + Returns parsed ml2 config from neutron + """ + ovs = get_pod_with_labels("application=neutron,component=neutron-ovs-agent") + sriov = get_pod_with_labels("application=neutron,component=neutron-sriov-agent") + + confs = get_neutron_ml2_conf_from_pod(ovs) + confs.extend(get_neutron_ml2_conf_from_pod(sriov)) + + config = configparser.ConfigParser() + for conf in confs: + config.read_string(conf) + + return config + + + + +def get_neutron_ml2_conf_from_pod(pod): + """ + Reads ml2 config from neutron pod + """ + cmd = ['ls', '/etc/neutron/plugins/ml2/'] + response = kube_exec(pod, cmd) + files = response.rstrip("\n").split() + + response = [] + for filename in files: + cmd = ['cat', '/etc/neutron/plugins/ml2/' + filename] + conf = kube_exec(pod, cmd) + response.append(conf) + + return response diff --git a/sdv/docker/sdvstate/validator/airship/pod_health_check.py b/sdv/docker/sdvstate/validator/airship/pod_health_check.py index 34a6747..0093ffc 100644 --- a/sdv/docker/sdvstate/validator/airship/pod_health_check.py +++ b/sdv/docker/sdvstate/validator/airship/pod_health_check.py @@ -13,13 +13,19 @@ # limitations under the License. +""" +Pod Health Checks +""" + + import logging -from kubernetes import client, config from tools.kube_utils import kube_api from tools.conf import settings -from tools.result_api import result_api, rfile +from tools.result_api import rfile + +from .store_result import store_result @@ -29,13 +35,25 @@ def pod_health_check(): """ api = kube_api() namespace_list = settings.getValue('airship_namespace_list') + + result = {'category': 'platform', + 'case_name': 'pod_health_check', + 'criteria': 'pass', + 'details': [] + } + for namespace in namespace_list: pod_list = api.list_namespaced_pod(namespace) for pod in pod_list.items: - result = pod_status(pod) - if result['state'] == 'fail': - result['logs'] = get_logs(pod) - result_api.store(result) + pod_stats = pod_status(pod) + if pod_stats['criteria'] == 'fail': + pod_stats['logs'] = get_logs(pod) + result['criteria'] = 'fail' + result['details'].append(pod_stats) + + + store_result(result) + return result @@ -43,14 +61,13 @@ def pod_status(pod): """ Check health of a pod and returns it's status as result """ - result = {'state': 'ok', - 'kind': 'pod', + result = {'criteria': 'pass', 'name': pod.metadata.name, 'namespace': pod.metadata.namespace, 'node': pod.spec.node_name} if pod.status.container_statuses is None: - result['state'] = 'fail' + result['criteria'] = 'fail' result['pod_details'] = rfile(str(pod)) else: for container in pod.status.container_statuses: @@ -62,14 +79,14 @@ def pod_status(pod): status = container.state.waiting.reason if status not in ('Running', 'Completed'): - result['state'] = 'fail' + result['criteria'] = 'fail' result['pod_details'] = rfile(str(pod)) - info = f'[Health: {result["state"]}] Name: {result["name"]}, ' + info = f'[Health: {result["criteria"]}] Name: {result["name"]}, ' info = info + f'Namespace: {result["namespace"]}, Node: {result["node"]}' logger = logging.getLogger(__name__) - logger.info(info) + logger.debug(info) return result diff --git a/sdv/docker/sdvstate/validator/airship/store_result.py b/sdv/docker/sdvstate/validator/airship/store_result.py new file mode 100644 index 0000000..52f4e10 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/store_result.py @@ -0,0 +1,28 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +store_result function to log and store result +""" +import logging +from tools.result_api import result_api + +def store_result(result): + """ + Logs and stores result + """ + logger = logging.getLogger(__name__) + logger.info(f'[State: {result["criteria"]}] {result["case_name"]}') + + result_api.store(result) |