From 9fa1356bf9aabd50c4adcec082eedf1410f9a7a7 Mon Sep 17 00:00:00 2001 From: Parth Yadav Date: Wed, 26 Aug 2020 00:08:29 +0530 Subject: Add support for TestAPI, Reporting and new checks Adds following new checks: * ceph_health_check * prometheus_check * grafana_check * elasticsearch_check * kibana_check * nagios_check * elasticsearch_exporter_check * fluentd_exporter_check * physical_network_check * reserved_vnf_cores_check * isolated_cores_check * vswitch_pmd_cores_check * vswitch_dpdk_lcore_check * os_reserved_cores_check * nova_scheduler_filters_check * cpu_allocation_ratio_check Updated result reporting format to match with TestAPI result formating Signed-off-by: Parth Yadav Change-Id: I421fd20067d289e8735a2ed3f402c68f45ae69e9 --- .../sdvstate/validator/airship/compute_check.py | 646 +++++++++++++++++++++ 1 file changed, 646 insertions(+) create mode 100644 sdv/docker/sdvstate/validator/airship/compute_check.py (limited to 'sdv/docker/sdvstate/validator/airship/compute_check.py') diff --git a/sdv/docker/sdvstate/validator/airship/compute_check.py b/sdv/docker/sdvstate/validator/airship/compute_check.py new file mode 100644 index 0000000..ff6f6db --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/compute_check.py @@ -0,0 +1,646 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Compute Related Checks +""" + +import configparser +import json + +from tools.kube_utils import kube_exec, get_pod_with_labels +from tools.conf import settings +from .store_result import store_result + + +########### +# Checks +########### + +def isolated_cores_check(): + """ + isolated_cores_check + """ + traced_value = trace_isolated_cores() + required_value = required_isolated_cores() + + result = {'category': 'compute', + 'case_name': 'isolated_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def reserved_vnf_cores_check(): + """ + reserved_vnf_cores_check + """ + traced_value = trace_reserved_vnf_cores() + required_value = required_reserved_vnf_cores() + + result = {'category': 'compute', + 'case_name': 'reserved_vnf_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def vswitch_pmd_cores_check(): + """ + vswitch_pmd_cores_check + """ + traced_value = trace_vswitch_pmd_cores() + required_value = required_vswitch_pmd_cores() + + result = {'category': 'compute', + 'case_name': 'vswitch_pmd_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def vswitch_dpdk_lcores_check(): + """ + vswitch_dpdk_lcores_check + """ + traced_value = trace_vswitch_dpdk_lcores() + required_value = required_vswitch_dpdk_lcores() + + result = {'category': 'compute', + 'case_name': 'vswitch_dpdk_lcores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def os_reserved_cores_check(): + """ + os_reserved_cores_check + """ + traced_value = trace_os_reserved_cores() + required_value = required_os_reserved_cores() + + result = {'category': 'compute', + 'case_name': 'os_reserved_cores_check', + 'details': {'traced_cores': traced_value, + 'required_cores': required_value + } + } + + if is_ranges_equals(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + + store_result(result) + return result + + + +def nova_scheduler_filters_check(): + """ + nova_scheduler_filters_check + """ + traced_value = trace_nova_scheduler_filters() + required_value = required_nova_scheduler_filters() + + result = {'category': 'compute', + 'case_name': 'nova_scheduler_filters_check', + 'details': {'traced_filters': traced_value, + 'required_filters': required_value + } + } + + if are_lists_equal(traced_value, required_value): + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + store_result(result) + return result + + + +def cpu_allocation_ratio_check(): + """ + cpu_allocation_ratio_check + """ + traced_value = trace_cpu_allocation_ratio() + required_value = required_cpu_allocation_ratio() + + result = {'category': 'compute', + 'case_name': 'cpu_allocation_ratio_check', + 'details': {'traced_ratio': traced_value, + 'required_ratio': required_value + } + } + + if traced_value == required_value: + result['criteria'] = 'pass' + else: + result['criteria'] = 'fail' + + store_result(result) + return result + + + + + + + + +############### +# helper functions +############### + + + +def trace_isolated_cores(): + """ + Trace isolated_cores from Airship deployment + + :return: value traced from `isolcpus` key in `/proc/cmdline` + """ + pod = get_pod_with_labels('application=nova,component=compute') + + cmd = ['cat', '/proc/cmdline'] + proc_cmd = kube_exec(pod, cmd) + + for option in proc_cmd.split(): + if 'isolcpus' in option: + _, isolcpus_value = split_key_value(option) + break + + return isolcpus_value + + +def required_isolated_cores(): + """ + Returns value of `isolated_cpus` from platform_profile used by + Role for worker nodes in PDF + + :return: isolated_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['isolated_cpus'] + + + + + + +def trace_reserved_vnf_cores(): + """ + Trace vnf_reserved_cores from Airship deployment + + :return: value traced from `vcpu_pin_set` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + vcpu_pin_set = config.get('DEFAULT', 'vcpu_pin_set') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + vcpu_pin_set = '' + + return vcpu_pin_set + + +def required_reserved_vnf_cores(): + """ + Returns value of vnf_cores from platform_profile used by + Role for worker nodes in PDF + + :return: vnf_reserverd_core value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vnf_cores'] + + + + + + +def trace_vswitch_pmd_cores(): + """ + Trace vswitch_pmd_cores from Airship deployment + + :return: value traced from `other_config:pmd-cpu-mask` in + openvswitchdb using ovs-vsctl + """ + ovs_pod = get_pod_with_labels('application=openvswitch,component=openvswitch-vswitchd') + + cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config'] + response = kube_exec(ovs_pod, cmd) + + response.replace('=', ':') + config = json.loads(response) + + if 'pmd-cpu-mask' in config: + pmd_cores = hex_to_comma_list(config['pmd-cpu-mask']) + else: + pmd_cores = '' + + return pmd_cores + + +def required_vswitch_pmd_cores(): + """ + Returns value of vswitch_pmd_cores from platform_profile used by + Role for worker nodes in PDF + + :return: vswitch_pmd_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vswitch_pmd_cores'] + + + + + + +def trace_vswitch_dpdk_lcores(): + """ + Trace vswitch_dpdk_lcores from Airship deployment + + :return: value traced from `other_config:dpdk-lcore-mask` in + openvswitchdb using ovs-vsctl + """ + ovs_pod = get_pod_with_labels('application=openvswitch,component=openvswitch-vswitchd') + + cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config'] + response = kube_exec(ovs_pod, cmd) + + response.replace('=', ':') + config = json.loads(response) + + if 'dpdk-lcore-mask' in config: + pmd_cores = hex_to_comma_list(config['dpdk-lcore-mask']) + else: + pmd_cores = '' + + return pmd_cores + + +def required_vswitch_dpdk_lcores(): + """ + Returns value of vswitch_dpdk_lcores from platform_profile used by + Role for worker nodes in PDF + + :return: vswitch_dpdk_lcores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['vswitch_dpdk_lcores'] + + + + + + +def trace_os_reserved_cores(): + """ + Trace os_reserved_cores from Airship deployment + + os_reserved_cores = all_cores - (reserved_vnf_cores + + vswitch_pmd_cores + + vswitch_dpdk_lcores) + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + all_cores = get_cores_by_role(worker_role) + + reserved_vnf_cores = trace_reserved_vnf_cores() + vswitch_pmd_cores = trace_vswitch_pmd_cores() + vswitch_dpdk_lcores = trace_vswitch_dpdk_lcores() + + non_os_cores = [] + non_os_cores.extend(convert_range_to_list(reserved_vnf_cores)) + non_os_cores.extend(convert_range_to_list(vswitch_pmd_cores)) + non_os_cores.extend(convert_range_to_list(vswitch_dpdk_lcores)) + + os_reserved_cores = set(all_cores).difference(set(non_os_cores)) + + # return as string with comma separated value + return ','.join(map(str, list(os_reserved_cores))) + + +def required_os_reserved_cores(): + """ + Returns value of os_reserved_cores from platform_profile used by + Role for worker nodes in PDF + + :return: os_reserved_cores value expected by the PDF + """ + worker_role = settings.getValue('WORKER_ROLE_NAME') + profile = get_platform_profile_by_role(worker_role) + return profile['os_reserved_cores'] + + + + + +def trace_nova_scheduler_filters(): + """ + Trace scheduler_filters from Airship deployment + + :return: value traced from `enabled_filters` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + filters = config.get('filter_scheduler', 'enabled_filters') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + filters = '' + + filters = filters.split(',') + map(str.strip, filters) + + return filters + +def required_nova_scheduler_filters(): + """ + Required nova scheduler_filters by the PDF + """ + pdf = settings.getValue('pdf_file') + filters = pdf['vim_functional']['scheduler_filters'] + + filters = filters.split(',') + map(str.strip, filters) + + return filters + + + + + + + +def trace_cpu_allocation_ratio(): + """ + Trace cpu_allocation_ratio from Airship deployment + + :return: value traced from `cpu_allocation_ratio` key in nova.conf + of actual deployment + """ + try: + config = get_nova_conf() + cpu_allocation_ratio = config.get('DEFAULT', 'cpu_allocation_ratio') + except (configparser.NoOptionError, configparser.MissingSectionHeaderError): + cpu_allocation_ratio = '' + + return float(cpu_allocation_ratio) + +def required_cpu_allocation_ratio(): + """ + Required cpu_allocation_ratio by the PDF + """ + pdf = settings.getValue('pdf_file') + cpu_allocation_ratio = pdf['vim_functional']['cpu_allocation_ratio'] + + return float(cpu_allocation_ratio) + + + + + + + +def get_role(role_name): + """ + Searches and returns role with `role_name` + """ + roles = settings.getValue('pdf_file')['roles'] + + for role in roles: + if role['name'] == role_name: + role_details = role + + return role_details + + +def get_platform_profile(profile_name): + """ + Searches and returns platform_profile with `profile_name` + """ + platform_profiles = settings.getValue('pdf_file')['platform_profiles'] + + for profile in platform_profiles: + if profile['profile_name'] == profile_name: + profile_details = profile + + return profile_details + +def get_processor_profile(profile_name): + """ + Searches and returns processor_profile with `profile_name` + """ + processor_profiles = settings.getValue('pdf_file')['processor_profiles'] + + for profile in processor_profiles: + if profile['profile_name'] == profile_name: + profile_details = profile + + return profile_details + +def get_platform_profile_by_role(role_name): + """ + Returns platform profile details of a role + """ + role = get_role(role_name) + profile = get_platform_profile(role['platform_profile']) + return profile + + +def get_hardware_profile_by_role(role_name): + """ + Returns hardware profile details of a role + """ + role = get_role(role_name) + + hardware_profiles = settings.getValue('pdf_file')['hardware_profiles'] + + for profile in hardware_profiles: + if profile['profile_name'] == role['hardware_profile']: + profile_details = profile + + return profile_details + + +def get_cores_by_role(role_name): + """ + Returns cpu cores list of server hardware used in the role + """ + hardware_profile = get_hardware_profile_by_role(role_name) + processor_profile = hardware_profile['profile_info']['processor_profile'] + profile = get_processor_profile(processor_profile) + + cpus = [] + + for numa in profile['profile_info']['numas']: + cpus.extend(convert_range_to_list(numa['cpu_set'])) + + return cpus + + + + + + + +def get_nova_conf(): + """ + Returns parsed nova.conf + """ + pod = get_pod_with_labels('application=nova,component=compute') + + cmd = ['cat', '/etc/nova/nova.conf'] + response = kube_exec(pod, cmd) + + config = configparser.ConfigParser() + config.read_string(response) + + return config + + +### cpu cores related helper function + +def convert_range_to_list(x): + """ + Returns list of numbers from given range as string + + e.g.: convert_range_to_list('3-5') will give [3, 4, 5] + """ + # pylint: disable=C0103 + result = [] + for part in x.split(','): + if '-' in part: + a, b = part.split('-') + a, b = int(a), int(b) + result.extend(range(a, b + 1)) + elif part != '': + a = int(part) + result.append(a) + # remove duplicates + result = list(dict.fromkeys(result)) + return result + + +def is_ranges_equals(range1, range2): + """ + Checks whether two ranges passed as string are equal + + e.g.: is_ranges_equals('2-5', '2-4,5') returns true + """ + set1 = set(convert_range_to_list(range1)) + set2 = set(convert_range_to_list(range2)) + return set1 == set2 + +def are_lists_equal(list1, list2): + """ + Checks whether two list are identicals + """ + set1 = set(list1) + set2 = set(list2) + return set1 == set2 + + + +def hex_to_comma_list(hex_mask): + """ + Converts CPU mask given in hex to list of cores + """ + binary = bin(int(hex_mask, 16))[2:] + reversed_binary = binary[::-1] + i = 0 + output = "" + for bit in reversed_binary: + if bit == '1': + output = output + str(i) + ',' + i = i + 1 + return output[:-1] + + +def comma_list_to_hex(cpus): + """ + Converts a list of cpu cores in corresponding hex value + of cpu-mask + """ + cpu_arr = cpus.split(",") + binary_mask = 0 + for cpu in cpu_arr: + binary_mask = binary_mask | (1 << int(cpu)) + return format(binary_mask, '02x') + + + +def split_key_value(key_value_str, delimiter='='): + """ + splits given string into key and value based on delimiter + + :param key_value_str: example string `someKey=somevalue` + :param delimiter: default delimiter is `=` + :return: [ key, value] + """ + key, value = key_value_str.split(delimiter) + key = key.strip() + value = value.strip() + return key, value -- cgit 1.2.3-korg