From 4f234b8fd2012d1c7c26effefdd41e7636a35738 Mon Sep 17 00:00:00 2001 From: opensource-tnbt Date: Thu, 5 Mar 2020 18:16:28 +0530 Subject: Tools: Version-1 of SDV Added. This patch adds Version-1 of the SDV. Please refer to docs folder for the details. Signed-off-by: Sridhar K. N. Rao Change-Id: I4fefa4752e40a735414d57b45d1dc929afa301b1 --- tools/sdv/NwLinksValid/__init__.py | 16 ++ tools/sdv/NwLinksValid/nwlinksvalidator.py | 37 ++++ tools/sdv/SoftwarePostValid/__init__.py | 18 ++ tools/sdv/SoftwarePostValid/swpostvalidator.py | 41 ++++ tools/sdv/SoftwarePreValid/__init__.py | 18 ++ tools/sdv/SoftwarePreValid/airship.py | 267 +++++++++++++++++++++++++ tools/sdv/SoftwarePreValid/swprevalidator.py | 41 ++++ tools/sdv/conf/00_common.conf | 18 ++ tools/sdv/conf/01_swprevalid.conf | 33 +++ tools/sdv/conf/02_swpostvalid.conf | 4 + tools/sdv/conf/03_nwlinksvalid.conf | 1 + tools/sdv/conf/__init__.py | 257 ++++++++++++++++++++++++ tools/sdv/core/__init__.py | 17 ++ tools/sdv/core/component_factory.py | 31 +++ tools/sdv/core/loader/__init__.py | 16 ++ tools/sdv/core/loader/loader.py | 128 ++++++++++++ tools/sdv/core/loader/loader_servant.py | 182 +++++++++++++++++ tools/sdv/docs/valid.rst | 28 +++ tools/sdv/valid | 147 ++++++++++++++ 19 files changed, 1300 insertions(+) create mode 100755 tools/sdv/NwLinksValid/__init__.py create mode 100644 tools/sdv/NwLinksValid/nwlinksvalidator.py create mode 100755 tools/sdv/SoftwarePostValid/__init__.py create mode 100644 tools/sdv/SoftwarePostValid/swpostvalidator.py create mode 100755 tools/sdv/SoftwarePreValid/__init__.py create mode 100644 tools/sdv/SoftwarePreValid/airship.py create mode 100644 tools/sdv/SoftwarePreValid/swprevalidator.py create mode 100644 tools/sdv/conf/00_common.conf create mode 100644 tools/sdv/conf/01_swprevalid.conf create mode 100644 tools/sdv/conf/02_swpostvalid.conf create mode 100644 tools/sdv/conf/03_nwlinksvalid.conf create mode 100644 tools/sdv/conf/__init__.py create mode 100644 tools/sdv/core/__init__.py create mode 100644 tools/sdv/core/component_factory.py create mode 100644 tools/sdv/core/loader/__init__.py create mode 100644 tools/sdv/core/loader/loader.py create mode 100644 tools/sdv/core/loader/loader_servant.py create mode 100644 tools/sdv/docs/valid.rst create mode 100755 tools/sdv/valid (limited to 'tools/sdv') diff --git a/tools/sdv/NwLinksValid/__init__.py b/tools/sdv/NwLinksValid/__init__.py new file mode 100755 index 0000000..87b6e58 --- /dev/null +++ b/tools/sdv/NwLinksValid/__init__.py @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Nework-Links Validator interface and helpers. +""" + +from NwLinksValid.nwlinksvalidator import * diff --git a/tools/sdv/NwLinksValid/nwlinksvalidator.py b/tools/sdv/NwLinksValid/nwlinksvalidator.py new file mode 100644 index 0000000..3b84595 --- /dev/null +++ b/tools/sdv/NwLinksValid/nwlinksvalidator.py @@ -0,0 +1,37 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Abstract class for N/W Lnks Prevalidations. +Implementors, please inherit from this class. +""" + +class INwLinksValidator(): + """ Model for a Links Validator """ + def __init__(self): + """ Initialization of the Interface """ + self._default_nwlinks_validation = None + + @property + def validation_nwlinks_defaults(self): + """ Default Validation values """ + return True + + def validate_compute_node_links(self): + """ Validating Compute Node Links""" + raise NotImplementedError('Please call an implementation.') + + def validate_control_node_links(self): + """ Validating Controller Node Links""" + raise NotImplementedError('Please call an implementation.') diff --git a/tools/sdv/SoftwarePostValid/__init__.py b/tools/sdv/SoftwarePostValid/__init__.py new file mode 100755 index 0000000..1c6f121 --- /dev/null +++ b/tools/sdv/SoftwarePostValid/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sw Validator interface and helpers. +""" + +from SoftwarePostValid.swpostvalidator import * diff --git a/tools/sdv/SoftwarePostValid/swpostvalidator.py b/tools/sdv/SoftwarePostValid/swpostvalidator.py new file mode 100644 index 0000000..2901cd4 --- /dev/null +++ b/tools/sdv/SoftwarePostValid/swpostvalidator.py @@ -0,0 +1,41 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Abstract class for Software Postvalidations. +Implementors, please inherit from this class. +""" + +class ISwPostValidator(): + """ Model for a Sw Validator """ + def __init__(self): + """ Initialization of the Interface """ + self._default_swpost_validation = None + + @property + def validation_swpost_defaults(self): + """ Default Validation values """ + return True + + def validate_configuration_mandatory(self): + """ + Validating Mandatory Configuration + """ + raise NotImplementedError('Please call an implementation.') + + def validate_configuration_optional(self): + """ + Validating Optional Configuration + """ + raise NotImplementedError('Please call an implementation.') diff --git a/tools/sdv/SoftwarePreValid/__init__.py b/tools/sdv/SoftwarePreValid/__init__.py new file mode 100755 index 0000000..e29c0af --- /dev/null +++ b/tools/sdv/SoftwarePreValid/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sw Validator interface and helpers. +""" + +from SoftwarePreValid.swprevalidator import * diff --git a/tools/sdv/SoftwarePreValid/airship.py b/tools/sdv/SoftwarePreValid/airship.py new file mode 100644 index 0000000..902b654 --- /dev/null +++ b/tools/sdv/SoftwarePreValid/airship.py @@ -0,0 +1,267 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Airship implementation of Software Predeployment Validation +""" + +import os +import shutil +from pathlib import Path +import git +import urllib3 +import yaml +from conf import settings +from SoftwarePreValid import swprevalidator + +def check_link(link): + """ + Function the check the availability of Hyperlinks + """ + timeout = urllib3.util.Timeout(connect=5.0, read=7.0) + http = urllib3.PoolManager(timeout=timeout) + try: + http.request('HEAD', link) + except urllib3.exceptions.LocationValueError as err: + print(err.args) + return False + except urllib3.exceptions.MaxRetryError as err: + print(err.args) + return False + except urllib3.exceptions.RequestError as err: + print(err.args) + return False + except urllib3.exceptions.ConnectTimeoutError as err: + print(err.args) + return False + except urllib3.exceptions.PoolError as err: + print(err.args) + return False + except urllib3.exceptions.HTTPError as err: + print(err.args) + return False + return True + +class Airship(swprevalidator.ISwPreValidator): + """ + Ariship Sw Validation + """ + def __init__(self): + """ Airship class constructor """ + super().__init__() + self.url = settings.getValue('AIRSHIP_MANIFEST_URL') + self.branch = settings.getValue('AIRSHIP_MANIFEST_BRANCH') + self.dl_path = settings.getValue('AIRSHIP_MANIFEST_DOWNLOAD_PATH') + self.site_name = settings.getValue('AIRSHIP_MANIFEST_SITE_NAME') + self.manifest = None + self.dirpath = Path(self.dl_path, 'airship') + self.tmdirpath = Path(self.dl_path, 'treasuremap') + self.locations = [] + + def clone_repo(self): + """ + Cloning the repos + """ + git.Repo.clone_from(self.url, + self.dirpath, + branch=self.branch) + git.Repo.clone_from('https://github.com/airshipit/treasuremap', + self.tmdirpath, + branch=settings.getValue( + 'AIRSHIP_TREASUREMAP_VERSION')) + + def cleanup_manifest(self): + """ + Remove existing manifests + """ + # Next Remove any manifest files, if it exists + if self.dirpath.exists() and self.dirpath.is_dir(): + shutil.rmtree(self.dirpath) + if self.tmdirpath.exists() and self.tmdirpath.is_dir(): + shutil.rmtree(self.tmdirpath) + + def manifest_exists_locally(self): + """ + Check if manifests exists locally + """ + if self.dirpath.exists() and self.dirpath.is_dir(): + return True + return False + + def validate_hyperlinks(self): + """ + Hyperlink Validation + """ + self.cleanup_manifest() + # Next, clone the repo to the provided path. + self.clone_repo() + + if self.dirpath.exists() and self.dirpath.is_dir(): + # Get the file(s) where links are defined. + self.find_locations( + os.path.join(self.dirpath, 'type', + 'cntt', 'software', + 'config', 'versions.yaml')) + for location in self.locations: + if check_link(location): + print("The Link: %s is VALID" % (location)) + else: + print("The Link: %s is INVALID" % (location)) + + + + # pylint: disable=consider-using-enumerate + def find_locations(self, yamlfile): + """ + Find all the hyperlinks in the manifests + """ + with open(yamlfile, 'r') as filep: + lines = filep.readlines() + for index in range(len(lines)): + line = lines[index].strip() + if line.startswith('location:'): + link = line.split(":", 1)[1] + if "opendev" in link: + if ((len(lines) > index+1) and + (lines[index+1].strip().startswith( + 'reference:'))): + ref = lines[index+1].split(":", 1)[1] + link = link + '/commit/' + ref.strip() + if link.strip() not in self.locations: + print(link) + self.locations.append(link.strip()) + if 'docker.' in line: + link = line.split(":", 1)[1] + link = link.replace('"', '') + parts = link.split('/') + if len(parts) == 3: + link = ('https://index.' + + parts[0].strip() + + '/v1/repositories/' + + parts[1] + '/' + parts[2].split(':')[0] + + '/tags/' + parts[2].split(':')[-1]) + if link.strip() not in self.locations: + print(link) + self.locations.append(link.strip()) + + # pylint: disable=too-many-nested-blocks, too-many-boolean-expressions + def validate_configuration_mandatory(self): + """ + Configuration checking of mandatory parameters + """ + if not self.manifest_exists_locally(): + self.clone_repo() + # We will perform validation one-by-one: + # The Operating System Flavor + os_done = False + os_filename = os.path.join(self.tmdirpath, + 'global', + 'software', + 'charts', + 'ucp', + 'drydock', + 'maas.yaml') + with open(os_filename, 'r') as osref: + osfiles = yaml.load_all(osref, Loader=yaml.FullLoader) + for osfile in osfiles: + if ('data' in osfile and + 'values' in osfile['data'] and + 'conf' in osfile['data']['values'] and + 'maas' in osfile['data']['values']['conf'] and + 'images' in osfile['data']['values']['conf']['maas'] and + ('default_os' in + osfile['data']['values']['conf']['maas']['images'])): + if (settings.getValue('OPERATING_SYSTEM') in + osfile['data']['values']['conf']['maas']['images'][ + 'default_os']): + print('Operating System is VALID') + os_done = True + if not os_done: + print("Operating System is INVALID") + + filesdir = os.path.join(self.dirpath, + 'site', + self.site_name, + 'profiles', + 'host') + hostprofile = None + os_ver_done = False + if os.path.isdir(filesdir): + for filename in os.listdir(filesdir): + filename = os.path.join(filesdir, filename) + with open(filename, 'r') as fileref: + hostprofile = yaml.load(fileref, Loader=yaml.FullLoader) + if 'data' in hostprofile: + if 'platform' in hostprofile['data']: + if 'image' in hostprofile['data']['platform']: + if (hostprofile['data']['platform']['image'] in + settings.getValue('OS_VERSION_NAME')): + print('Operating System Version is VALID') + os_ver_done = True + break + if not os_ver_done: + print("Operating System Version is INVALID") + # Virtualization - Hugepages and CPU Isolation + hugepages_size_done = False + hugepages_count_done = False + filesdir = os.path.join(self.dirpath, + 'type', + 'cntt', + 'profiles', + 'hardware') + if os.path.isdir(filesdir): + for filename in os.listdir(filesdir): + filename = os.path.join(filesdir, filename) + with open(filename, 'r') as fileref: + hwprofile = yaml.load(fileref, Loader=yaml.FullLoader) + if ('data' in hwprofile and + 'hugepages' in hwprofile['data'] and + 'dpdk' in hwprofile['data']['hugepages']): + if ('size' in hwprofile['data']['hugepages']['dpdk'] and + (settings.getValue('HUGEPAGES_SIZE') in + hwprofile['data']['hugepages']['dpdk']['size'])): + print('Hugepages Size is VALID') + else: + print('Hugepages Size is INVALID') + hugepages_size_done = True + if ('count' in hwprofile['data']['hugepages']['dpdk'] and + (settings.getValue('HUGEPAGES_COUNT') == + hwprofile['data']['hugepages']['dpdk']['count'])): + print('Hugepages COUNT is VALID') + else: + print('Hugepages COUNT is INVALID') + hugepages_count_done = True + if hugepages_size_done and hugepages_count_done: + break + + # Virtual Switch - Switch and Configuration + # Openstack-Version + filename = os.path.join(self.tmdirpath, + 'global', + 'software', + 'config', + 'versions.yaml') + if os.path.exists(filename): + if settings.getValue('OPENSTACK_VERSION') in open(filename).read(): + print('Openstack Version is valid') + else: + print('Openstack version if INVALID') + # Openstack Services + # Bootstrap + + def validate_configuration_optional(self): + """ + Validate Optional COnfigurations + """ + return False diff --git a/tools/sdv/SoftwarePreValid/swprevalidator.py b/tools/sdv/SoftwarePreValid/swprevalidator.py new file mode 100644 index 0000000..485fc2b --- /dev/null +++ b/tools/sdv/SoftwarePreValid/swprevalidator.py @@ -0,0 +1,41 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Abstract class for Software Prevalidations. +Implementors, please inherit from this class. +""" + +class ISwPreValidator(): + """ Model for a Sw Validator """ + def __init__(self): + """ Initialization of the Interface """ + self._default_swpre_validation = None + + @property + def validation_swpre_defaults(self): + """ Default Validation values """ + return True + + def validate_hyperlinks(self): + """ Validate Hyperlinks""" + raise NotImplementedError('Please call an implementation.') + + def validate_configuration_mandatory(self): + """ Validate Mandatory Configurations """ + raise NotImplementedError('Please call an implementation.') + + def validate_configuration_optional(self): + """ Validate Optional Configurations """ + raise NotImplementedError('Please call an implementation.') diff --git a/tools/sdv/conf/00_common.conf b/tools/sdv/conf/00_common.conf new file mode 100644 index 0000000..5326ecb --- /dev/null +++ b/tools/sdv/conf/00_common.conf @@ -0,0 +1,18 @@ +import os + +# default log output directory for all logs +LOG_DIR = '/tmp' + +# default log for all "small" executables +LOG_FILE_DEFAULT = 'valid-overall.log' + +ROOT_DIR = os.path.normpath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), '../')) +SW_PRE_VALID_DIR = os.path.join(ROOT_DIR, 'SoftwarePreValid') +SW_POST_VALID_DIR = os.path.join(ROOT_DIR, 'SoftwarePostValid') +NW_LINKS_VALID_DIR = os.path.join(ROOT_DIR, 'NwLinksValid') + +# 'debug', 'info', 'warning', 'error', 'critical' +VERBOSITY = 'warning' + +EXCLUDE_MODULES = [''] diff --git a/tools/sdv/conf/01_swprevalid.conf b/tools/sdv/conf/01_swprevalid.conf new file mode 100644 index 0000000..46043ce --- /dev/null +++ b/tools/sdv/conf/01_swprevalid.conf @@ -0,0 +1,33 @@ + +# Modify this value to any Installer projects that have +# manifests (templates and configuration files) +SW_PRE_VALIDATOR = 'Airship' + +# Mandatory Requirements [ Configuration Check ] +OPENSTACK_VERSION = 'ocata' +OPERATING_SYSTEM = 'centos' +OS_VERSION_NAME = 'xenial' +HUGEPAGES_SIZE = '1G' +HUGEPAGES_COUNT = 32 +OPENSTACK_CUSTOM_SERVICE_LIST = ['barbican', 'sahara'] +VIRTUAL_SWITCH_COMPUTE = 'ovs' +VIRTUAL_SWITCH_VERSION = '2.9.2' +BOOTSTRAP_PROTOCOL = 'pxe' +CPU_ISOLATION = '2-19,22-39' + +# Airship Specific configurations. +AIRSHIP_MANIFEST_URL = 'https://gerrit.opnfv.org/gerrit/airship' +AIRSHIP_MANIFEST_BRANCH = 'master' +AIRSHIP_MANIFEST_DOWNLOAD_PATH = '/tmp' +AIRSHIP_MANIFEST_SITE_NAME = 'intel-pod10' +AIRSHIP_TREASUREMAP_VERSION = 'v1.7' + + +# Optional Requirements [Could be Installer Specific ] +OVS_DPDK_ARGUMENTS = 'test' +OVERCLOUD_LOGGING_CLIENT = 'fluentd' +OVERCLOUD_MONITORING_CLIENT = 'collectd' +LMA_SERVER_MONITORING = 'prometheus' +LMA_SERVER_LOGGING = 'efk' +OPENSTACK_CONTAINER_ORCHESTRATION = 'kubernetes' +AIRSHIP_MANIFEST_VERSION = 1.7 diff --git a/tools/sdv/conf/02_swpostvalid.conf b/tools/sdv/conf/02_swpostvalid.conf new file mode 100644 index 0000000..1ed9279 --- /dev/null +++ b/tools/sdv/conf/02_swpostvalid.conf @@ -0,0 +1,4 @@ + +# Modify this value to any Installer projects that have +# manifests (templates and configuration files) +SW_POST_VALIDATOR = 'Airship' diff --git a/tools/sdv/conf/03_nwlinksvalid.conf b/tools/sdv/conf/03_nwlinksvalid.conf new file mode 100644 index 0000000..6e83066 --- /dev/null +++ b/tools/sdv/conf/03_nwlinksvalid.conf @@ -0,0 +1 @@ +NW_LINKS_VALIDATOR = 'lldpd' diff --git a/tools/sdv/conf/__init__.py b/tools/sdv/conf/__init__.py new file mode 100644 index 0000000..c7dadca --- /dev/null +++ b/tools/sdv/conf/__init__.py @@ -0,0 +1,257 @@ +# Copyright 2015-2017 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Settings and configuration handlers. + +Settings will be loaded from several .conf files +and any user provided settings file. +""" + +# pylint: disable=invalid-name + +import copy +import os +import re +import logging +import pprint +import ast +import netaddr + +_LOGGER = logging.getLogger(__name__) + +# regex to parse configuration macros from 04_vnf.conf +# it will select all patterns starting with # sign +# and returns macro parameters and step +# examples of valid macros: +# #VMINDEX +# #MAC(AA:BB:CC:DD:EE:FF) or #MAC(AA:BB:CC:DD:EE:FF,2) +# #IP(192.168.1.2) or #IP(192.168.1.2,2) +# #EVAL(2*#VMINDEX) +_PARSE_PATTERN = r'(#[A-Z]+)(\(([^(),]+)(,([0-9]+))?\))?' + +class Settings(object): + """Holding class for settings. + """ + def __init__(self): + pass + + def _eval_param(self, param): + # pylint: disable=invalid-name + """ Helper function for expansion of references to vsperf parameters + """ + if isinstance(param, str): + # evaluate every #PARAM reference inside parameter itself + macros = re.findall(r'#PARAM\((([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)\)', param) + if macros: + for macro in macros: + # pylint: disable=eval-used + try: + tmp_val = str(eval("self.getValue('{}'){}".format(macro[1], macro[2]))) + param = param.replace('#PARAM({})'.format(macro[0]), tmp_val) + # silently ignore that option required by PARAM macro can't be evaluated; + # It is possible, that referred parameter will be constructed during runtime + # and re-read later. + except IndexError: + pass + except AttributeError: + pass + return param + elif isinstance(param, (list, tuple)): + tmp_list = [] + for item in param: + tmp_list.append(self._eval_param(item)) + return tmp_list + elif isinstance(param, dict): + tmp_dict = {} + for (key, value) in param.items(): + tmp_dict[key] = self._eval_param(value) + return tmp_dict + else: + return param + + def getValue(self, attr): + """Return a settings item value + """ + if attr in self.__dict__: + if attr == 'TEST_PARAMS': + return getattr(self, attr) + else: + master_value = getattr(self, attr) + return self._eval_param(master_value) + else: + raise AttributeError("%r object has no attribute %r" % + (self.__class__, attr)) + + def __setattr__(self, name, value): + """Set a value + """ + # skip non-settings. this should exclude built-ins amongst others + if not name.isupper(): + return + + # we can assume all uppercase keys are valid settings + super(Settings, self).__setattr__(name, value) + + def setValue(self, name, value): + """Set a value + """ + if name is not None and value is not None: + super(Settings, self).__setattr__(name, value) + + def load_from_file(self, path): + """Update ``settings`` with values found in module at ``path``. + """ + import imp + + custom_settings = imp.load_source('custom_settings', path) + + for key in dir(custom_settings): + if getattr(custom_settings, key) is not None: + setattr(self, key, getattr(custom_settings, key)) + + def load_from_dir(self, dir_path): + """Update ``settings`` with contents of the .conf files at ``path``. + + Each file must be named Nfilename.conf, where N is a single or + multi-digit decimal number. The files are loaded in ascending order of + N - so if a configuration item exists in more that one file the setting + in the file with the largest value of N takes precedence. + + :param dir_path: The full path to the dir from which to load the .conf + files. + + :returns: None + """ + regex = re.compile("^(?P[0-9]+)(?P[a-z]?)_.*.conf$") + + def get_prefix(filename): + """ + Provide a suitable function for sort's key arg + """ + match_object = regex.search(os.path.basename(filename)) + return [int(match_object.group('digit_part')), + match_object.group('alfa_part')] + + # get full file path to all files & dirs in dir_path + file_paths = os.listdir(dir_path) + file_paths = [os.path.join(dir_path, x) for x in file_paths] + + # filter to get only those that are a files, with a leading + # digit and end in '.conf' + file_paths = [x for x in file_paths if os.path.isfile(x) and + regex.search(os.path.basename(x))] + + # sort ascending on the leading digits and afla (e.g. 03_, 05a_) + file_paths.sort(key=get_prefix) + + # load settings from each file in turn + for filepath in file_paths: + self.load_from_file(filepath) + + def load_from_dict(self, conf): + """ + Update ``settings`` with values found in ``conf``. + + Unlike the other loaders, this is case insensitive. + """ + for key in conf: + if conf[key] is not None: + if isinstance(conf[key], dict): + # recursively update dict items, e.g. TEST_PARAMS + setattr(self, key.upper(), + merge_spec(getattr(self, key.upper()), conf[key])) + else: + setattr(self, key.upper(), conf[key]) + + def restore_from_dict(self, conf): + """ + Restore ``settings`` with values found in ``conf``. + + Method will drop all configuration options and restore their + values from conf dictionary + """ + self.__dict__.clear() + tmp_conf = copy.deepcopy(conf) + for key in tmp_conf: + self.setValue(key, tmp_conf[key]) + + def load_from_env(self): + """ + Update ``settings`` with values found in the environment. + """ + for key in os.environ: + setattr(self, key, os.environ[key]) + + def __str__(self): + """Provide settings as a human-readable string. + + This can be useful for debug. + + Returns: + A human-readable string. + """ + tmp_dict = {} + for key in self.__dict__: + tmp_dict[key] = self.getValue(key) + + return pprint.pformat(tmp_dict) + + # + # validation methods used by step driven testcases + # + def validate_getValue(self, result, attr): + """Verifies, that correct value was returned + """ + # getValue must be called to expand macros and apply + # values from TEST_PARAM option + assert result == self.getValue(attr) + return True + + def validate_setValue(self, _dummy_result, name, value): + """Verifies, that value was correctly set + """ + assert value == self.__dict__[name] + return True + +settings = Settings() + + +def merge_spec(orig, new): + """Merges ``new`` dict with ``orig`` dict, and returns orig. + + This takes into account nested dictionaries. Example: + + >>> old = {'foo': 1, 'bar': {'foo': 2, 'bar': 3}} + >>> new = {'foo': 6, 'bar': {'foo': 7}} + >>> merge_spec(old, new) + {'foo': 6, 'bar': {'foo': 7, 'bar': 3}} + + You'll notice that ``bar.bar`` is not removed. This is the desired result. + """ + for key in orig: + if key not in new: + continue + + # Not allowing derived dictionary types for now + # pylint: disable=unidiomatic-typecheck + if type(orig[key]) == dict: + orig[key] = merge_spec(orig[key], new[key]) + else: + orig[key] = new[key] + + for key in new: + if key not in orig: + orig[key] = new[key] + + return orig diff --git a/tools/sdv/core/__init__.py b/tools/sdv/core/__init__.py new file mode 100644 index 0000000..6c1db43 --- /dev/null +++ b/tools/sdv/core/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Core structural interfaces and their implementations +""" +import core.component_factory diff --git a/tools/sdv/core/component_factory.py b/tools/sdv/core/component_factory.py new file mode 100644 index 0000000..68ad460 --- /dev/null +++ b/tools/sdv/core/component_factory.py @@ -0,0 +1,31 @@ +# Copyright 2020 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Create Components. +""" + +def create_swprevalidator(swprevalidator_class): + """ Create Pre-Validators""" + return swprevalidator_class() + + +def create_swpostvalidator(swpostvalidator_class): + """ Create Post-Validators""" + return swpostvalidator_class() + + +def create_linkvalidator(linkprevalidator_class): + """ Create Link-Validators""" + return linkprevalidator_class() diff --git a/tools/sdv/core/loader/__init__.py b/tools/sdv/core/loader/__init__.py new file mode 100644 index 0000000..c3b60a4 --- /dev/null +++ b/tools/sdv/core/loader/__init__.py @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Core: Loader Component. +""" +from .loader import Loader diff --git a/tools/sdv/core/loader/loader.py b/tools/sdv/core/loader/loader.py new file mode 100644 index 0000000..ea62d0c --- /dev/null +++ b/tools/sdv/core/loader/loader.py @@ -0,0 +1,128 @@ +# Copyright 2020 Intel Corporation, Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Abstract class for Software Prevalidations. +Implementors, please inherit from this class. +""" + +from conf import settings +from core.loader.loader_servant import LoaderServant +from SoftwarePreValid.swprevalidator import ISwPreValidator +from SoftwarePostValid.swpostvalidator import ISwPostValidator +from NwLinksValid.nwlinksvalidator import INwLinksValidator + +# pylint: disable=too-many-public-methods +class Loader(): + """Loader class - main object context holder. + """ + _swvalidator_loader = None + + def __init__(self): + """Loader ctor - initialization method. + + All data is read from configuration each time Loader instance is + created. It is up to creator to maintain object life cycle if this + behavior is unwanted. + """ + self._swprevalidator_loader = LoaderServant( + settings.getValue('SW_PRE_VALID_DIR'), + settings.getValue('SW_PRE_VALIDATOR'), + ISwPreValidator) + self._swpostvalidator_loader = LoaderServant( + settings.getValue('SW_POST_VALID_DIR'), + settings.getValue('SW_POST_VALIDATOR'), + ISwPostValidator) + self._nwlinksvalidator_loader = LoaderServant( + settings.getValue('NW_LINKS_VALID_DIR'), + settings.getValue('NW_LINKS_VALIDATOR'), + INwLinksValidator) + + def get_swprevalidator(self): + """ Returns a new instance configured Software Validator + :return: ISwPreValidator implementation if available, None otherwise + """ + return self._swprevalidator_loader.get_class()() + + def get_swprevalidator_class(self): + """Returns type of currently configured Software Validator. + + :return: Type of ISwPreValidator implementation if available. + None otherwise. + """ + return self._swprevalidator_loader.get_class() + + def get_swprevalidators(self): + """ + Get Prevalidators + """ + return self._swprevalidator_loader.get_classes() + + def get_swprevalidators_printable(self): + """ + Get Prevalidators for printing + """ + return self._swprevalidator_loader.get_classes_printable() + + def get_swpostvalidator(self): + """ Returns a new instance configured Software Validator + :return: ISwPostValidator implementation if available, None otherwise + """ + return self._swpostvalidator_loader.get_class()() + + def get_swpostvalidator_class(self): + """Returns type of currently configured Software Validator. + + :return: Type of ISwPostValidator implementation if available. + None otherwise. + """ + return self._swpostvalidator_loader.get_class() + + def get_swpostvalidators(self): + """ + Get Postvalidators + """ + return self._swpostvalidator_loader.get_classes() + + def get_swpostvalidators_printable(self): + """ + Get Postvalidators for printing + """ + return self._swpostvalidator_loader.get_classes_printable() + + def get_nwlinksvalidator(self): + """ Returns a new instance configured Nw-Links Validator + :return: INwLinksValidator implementation if available, None otherwise + """ + return self._nwlinksvalidator_loader.get_class()() + + def get_nwlinksvalidator_class(self): + """Returns type of currently configured Nw-Links Validator. + + :return: Type of NwLinksValidator implementation if available. + None otherwise. + """ + return self._nwlinksvalidator_loader.get_class() + + def get_nwlinkvalidators(self): + """ + Get Linkvalidators + """ + return self._nwlinksvalidator_loader.get_classes() + + def get_nwlinkvalidators_printable(self): + """ + Get Linkvalidators for printing + """ + return self._nwlinksvalidator_loader.get_classes_printable() diff --git a/tools/sdv/core/loader/loader_servant.py b/tools/sdv/core/loader/loader_servant.py new file mode 100644 index 0000000..ba175d5 --- /dev/null +++ b/tools/sdv/core/loader/loader_servant.py @@ -0,0 +1,182 @@ +# Copyright 2020 Intel Corporation, Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Loader Support Module. +""" + +import os +from os import sys +import imp +import fnmatch +import logging +from conf import settings + +class LoaderServant(): + """Class implements basic dynamic import operations. + """ + _class_name = None + _path = None + _interface = None + + def __init__(self, path, class_name, interface): + """LoaderServant constructor + + Intializes all data needed for import operations. + + Attributes: + path: path to directory which contains implementations derived from + interface. + class_name: Class name which will be returned in get_class + method, if such definition exists in directory + represented by path, + interface: interface type. Every object which doesn't + implement this particular interface will be + filtered out. + """ + self._class_name = class_name + self._path = path + self._interface = interface + + def get_class(self): + """Returns class type based on parameters passed in __init__. + + :return: Type of the found class. + None if class hasn't been found + """ + + return self.load_module(path=self._path, + interface=self._interface, + class_name=self._class_name) + + def get_classes(self): + """Returns all classes in path derived from interface + + :return: Dictionary with following data: + - key: String representing class name, + - value: Class type. + """ + return self.load_modules(path=self._path, + interface=self._interface) + + def get_classes_printable(self): + """Returns all classes derived from _interface found in path + + :return: String - list of classes in printable format. + """ + + out = self.load_modules(path=self._path, + interface=self._interface) + results = [] + + # sort modules to produce the same output everytime + for (name, mod) in sorted(out.items()): + desc = (mod.__doc__ or 'No description').strip().split('\n')[0] + results.append((name, desc)) + + header = 'Classes derived from: ' + self._interface.__name__ + output = [header + '\n' + '=' * len(header) + '\n'] + + for (name, desc) in results: + output.append('* %-18s%s' % ('%s:' % name, desc)) + + output.append('') + + output.append('') + + return '\n'.join(output) + + @staticmethod + def load_module(path, interface, class_name): + """Imports everything from given path and returns class type + + This is based on following conditions: + - Class is derived from interface, + - Class type name matches class_name. + + :return: Type of the found class. + None if class hasn't been found + """ + + results = LoaderServant.load_modules( + path=path, interface=interface) + + if class_name in results: + logging.info( + "Class found: %s.", class_name) + return results.get(class_name) + + return None + + @staticmethod + def load_modules(path, interface): + """Returns dictionary of class name/class type found in path + + This is based on following conditions: + - classes found under path are derived from interface. + - class is not interface itself. + + :return: Dictionary with following data: + - key: String representing class name, + - value: Class type. + """ + result = {} + + for _, mod in LoaderServant._load_all_modules(path): + # find all classes derived from given interface, but suppress + # interface itself and any abstract class starting with iface name + gens = dict((k, v) for (k, v) in list(mod.__dict__.items()) + if isinstance(v, type) and + issubclass(v, interface) and + not k.startswith(interface.__name__)) + if gens: + for (genname, gen) in list(gens.items()): + result[genname] = gen + return result + + @staticmethod + def _load_all_modules(path): + """Load all modules from ``path`` directory. + + This is based on the design used by OFTest: + https://github.com/floodlight/oftest/blob/master/oft + + :param path: Path to a folder of modules. + + :return: List of modules in a folder. + """ + mods = [] + + for root, _, filenames in os.walk(path): + # Iterate over each python file + for filename in fnmatch.filter(filenames, '[!.]*.py'): + modname = os.path.splitext(os.path.basename(filename))[0] + + # skip module load if it is excluded by configuration + if modname in settings.getValue('EXCLUDE_MODULES'): + continue + + try: + if modname in sys.modules: + mod = sys.modules[modname] + else: + mod = imp.load_module( + modname, *imp.find_module(modname, [root])) + except ImportError: + logging.error('Could not import file %s', filename) + raise + + mods.append((modname, mod)) + + return mods diff --git a/tools/sdv/docs/valid.rst b/tools/sdv/docs/valid.rst new file mode 100644 index 0000000..6aeb8a2 --- /dev/null +++ b/tools/sdv/docs/valid.rst @@ -0,0 +1,28 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Intel Corporation, AT&T and others. + +CIRV Software Validation Tool +============================= +This tool is designed to perform Software Configuration Validation, which covers: + +1. Pre-Deployment (of VIM or Container Management Software) Validation of Software Configuration +2. Post-Deployment (of VIM or Container Management Software) Validation of Software Configuration +3. Network-Link Checking - Validating VLAN and IP configurations + + +Installation +************ +This tool does not have any installation. However, there are requirements in terms of Python packages, which can be installed using pip3. Refer to requirements.txt file for the package list. + +Usage +***** +Example Commands: + +1. To run all the validations: ./valid +2. Help: ./valid --help +3. Version Check: ./valid --version +4. List Sofware Pre-Deployment validators: ./valid --list-swpredepv +5. List Sofware Post-Deployment validators: ./valid --list-swpostdepv +6. List all validations: ./valid --list-validations +7. Run only single validation [WORK IN PROGRESS] diff --git a/tools/sdv/valid b/tools/sdv/valid new file mode 100755 index 0000000..1a9a252 --- /dev/null +++ b/tools/sdv/valid @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Spirent Communications +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""VALID main script. +""" + +import logging +import os +import sys +import argparse +import time +import datetime +from conf import settings +import core.component_factory as component_factory +from core.loader import Loader + +VERBOSITY_LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL +} + +_CURR_DIR = os.path.dirname(os.path.realpath(__file__)) +_LOGGER = logging.getLogger() + +def parse_arguments(): + """ + Parse command line arguments. + """ + parser = argparse.ArgumentParser(prog=__file__, formatter_class= + argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--version', action='version', version='%(prog)s 0.1') + parser.add_argument('--list-validations', action='store_true', + help='list all validations') + parser.add_argument('--list-swpredepv', action='store_true', + help='list all Software pre-dep validations and exit') + parser.add_argument('--list-swpostdepv', action='store_true', + help='list all Software post-dep validations and exit') + parser.add_argument('--list-nwlinksv', action='store_true', + help='list all Link validations and exit') + parser.add_argument('exact_validation_name', nargs='*', help='Exact names of\ + validations to run. E.g "valid nwlinks"\ + runs only nwlink-validations.\ + To run all omit positional arguments') + args = vars(parser.parse_args()) + + return args + + +def configure_logging(level): + """Configure logging. + """ + name, ext = os.path.splitext(settings.getValue('LOG_FILE_DEFAULT')) + rename_default = "{name}_{uid}{ex}".format(name=name, + uid=settings.getValue( + 'LOG_TIMESTAMP'), + ex=ext) + log_file_default = os.path.join( + settings.getValue('RESULTS_PATH'), rename_default) + _LOGGER.setLevel(logging.DEBUG) + stream_logger = logging.StreamHandler(sys.stdout) + stream_logger.setLevel(VERBOSITY_LEVELS[level]) + stream_logger.setFormatter(logging.Formatter( + '[%(levelname)-5s] %(asctime)s : (%(name)s) - %(message)s')) + _LOGGER.addHandler(stream_logger) + file_logger = logging.FileHandler(filename=log_file_default) + file_logger.setLevel(logging.DEBUG) + file_logger.setFormatter(logging.Formatter( + '%(asctime)s : %(message)s')) + _LOGGER.addHandler(file_logger) + +def handle_list_options(args): + """ Process --list cli arguments if needed + + :param args: A dictionary with all CLI arguments + """ + if args['list_swpredepv']: + print(Loader().get_swprevalidators_printable()) + sys.exit(0) + + if args['list_swpostdepv']: + print(Loader().get_swpostvalidators_printable()) + sys.exit(0) + + if args['list_nwlinksv']: + print(Loader().get_nwlinkvalidators_printable()) + sys.exit(0) + + +def main(): + """Main function. + """ + args = parse_arguments() + + # define the timestamp to be used by logs and results + date = datetime.datetime.fromtimestamp(time.time()) + timestamp = date.strftime('%Y-%m-%d_%H-%M-%S') + settings.setValue('LOG_TIMESTAMP', timestamp) + + + # configure settings + settings.load_from_dir(os.path.join(_CURR_DIR, 'conf')) + + # if required, handle list-* operations + handle_list_options(args) + + results_dir = "results_" + timestamp + results_path = os.path.join(settings.getValue('LOG_DIR'), results_dir) + settings.setValue('RESULTS_PATH', results_path) + # create results directory + if not os.path.exists(results_path): + os.makedirs(results_path) + + configure_logging(settings.getValue('VERBOSITY')) + + loader = Loader() + swprevalidators = loader.get_swprevalidators() + if settings.getValue('SW_PRE_VALIDATOR') not in swprevalidators: + _LOGGER.error('There are no swvalidators matching \'%s\' found in' + ' \'%s\'. Exiting...', settings.getValue('SW_PRE_VALIDATOR'), + settings.getValue('SW_PRE_VALID_DIR')) + sys.exit(1) + swv_pre_ctrl = component_factory.create_swprevalidator( + loader.get_swprevalidator_class()) + # First validate hyperlinks + swv_pre_ctrl.validate_hyperlinks() + # Next validate mandatory configuration + swv_pre_ctrl.validate_configuration_mandatory() + + +if __name__ == "__main__": + main() -- cgit 1.2.3-korg