diff options
Diffstat (limited to 'sdv/docker/sdvstate/tools')
| -rw-r--r-- | sdv/docker/sdvstate/tools/conf/__init__.py | 120 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/kube_utils.py | 85 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/__init__.py | 24 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/result_api.py | 73 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/rfile.py | 52 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/storage/__init__.py | 18 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py | 17 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/storage/local/local.py | 133 | ||||
| -rw-r--r-- | sdv/docker/sdvstate/tools/result_api/storage/storage_api.py | 46 |
9 files changed, 568 insertions, 0 deletions
diff --git a/sdv/docker/sdvstate/tools/conf/__init__.py b/sdv/docker/sdvstate/tools/conf/__init__.py new file mode 100644 index 0000000..67c808f --- /dev/null +++ b/sdv/docker/sdvstate/tools/conf/__init__.py @@ -0,0 +1,120 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + + +"""Settings and configuration handlers. + +Settings will be loaded from several .yaml or .yml files +and any user provided settings file. +""" + +import os +import ast + +import yaml + +# pylint: disable=invalid-name + +class Settings(): + """Holding class for settings. + """ + def __init__(self): + pass + + + def getValue(self, attr): + """ + Return a settings item value + """ + try: + attr = attr.lower() + return getattr(self, attr) + except AttributeError: + raise AttributeError("{obj} object has no attribute \ + {attr}".format(obj=self.__class__, attr=attr)) + + + def setValue(self, name, value): + """Set a value + """ + if name is not None and value is not None: + super(Settings, self).__setattr__(name.lower(), value) + + + def load_from_file(self, path): + """Update ``settings`` with values found in module at ``path``. + """ + with open(path) as file: + configs = yaml.load_all(file, Loader=yaml.SafeLoader) + for conf in configs: + for name, value in conf.items(): + self.setValue(name, value) + + + def load_from_env(self): + """ + Update ``settings`` with values found in the environment. + """ + for key in os.environ: + value = os.environ[key] + + #evaluate string to python type + try: + value = ast.literal_eval(os.environ[key]) + except (ValueError, SyntaxError): + pass #already string + + self.setValue(key, value) + + + def load_from_dir(self, dir_path): + """Update ``settings`` with contents of the yaml files at ``path``. + + Files are read in ascending order, hence if a configuration item + exists in more than one file, then the setting in the file that + occurs in the last read file will have high precedence and + overwrite previous values. + + Same precedence logic for sub-directories. + Also, child directory will have more precedence than it's parent + + :param dir_path: The full path to the dir from which to load the + yaml files. + + :returns: None + """ + files = list_yamls(dir_path) + + for file in files: + self.load_from_file(file) + + +settings = Settings() + + +def list_yamls(dir_path): + """Get all yaml files recursively in ``dir_path`` + """ + files = [] + dir_list = [x[0] for x in os.walk(dir_path)] + dir_list.sort() + for path in dir_list: + dir_files = [path+'/'+f for f in os.listdir(path) + if f.endswith('.yaml') or f.endswith('.yml')] + if dir_files is not None: + dir_files.sort() + files.extend(dir_files) + return files diff --git a/sdv/docker/sdvstate/tools/kube_utils.py b/sdv/docker/sdvstate/tools/kube_utils.py new file mode 100644 index 0000000..3468ac6 --- /dev/null +++ b/sdv/docker/sdvstate/tools/kube_utils.py @@ -0,0 +1,85 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +""" +Kubernetes cluster api helper functions +""" + +from kubernetes import client, config +from kubernetes.stream import stream + +from tools.conf import settings # pylint: disable=import-error + + +def load_kube_api(): + """ + Loads kubernetes api + """ + config.load_kube_config(settings.getValue('kube_config')) + api = client.CoreV1Api() + settings.setValue('kube_api', api) + + +def kube_api(): + """ + Returns kube_api object + """ + return settings.getValue('kube_api') + + +def get_pod_with_labels(labels): + """ + Returns json details any one pod with matching labels + + :param labels: labels to find matching pod + :return: pod details + """ + api = kube_api() + pod = api.list_pod_for_all_namespaces(label_selector=labels).items[0] + return pod + + +def kube_exec(pod, cmd): + """ + Executes `cmd` inside `pod` and returns response + + :param pod: pod object + :param cmd: command to execute inside pod + :return: response from pod + """ + api = kube_api() + response = stream(api.connect_get_namespaced_pod_exec, + pod.metadata.name, pod.metadata.namespace, command=cmd, + stderr=True, stdin=False, stdout=True, tty=False) + return response + + +def kube_curl(*args): + """ + executes curl cmd in kubernetes network + + :param args: comma separated list of args to pass to curl + :return: http response + """ + args = list(args) + args.insert(0, "curl") + + labels = "application=prometheus-openstack-exporter,component=exporter" + pod = get_pod_with_labels(labels) + + response = kube_exec(pod, args) + + return response diff --git a/sdv/docker/sdvstate/tools/result_api/__init__.py b/sdv/docker/sdvstate/tools/result_api/__init__.py new file mode 100644 index 0000000..0633103 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +""" +Result Manager Package +""" +from .result_api import result_api +from .rfile import rfile + +from .storage.storage_api import StorageApi +from .storage.local.local import Local diff --git a/sdv/docker/sdvstate/tools/result_api/result_api.py b/sdv/docker/sdvstate/tools/result_api/result_api.py new file mode 100644 index 0000000..95dfd56 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/result_api.py @@ -0,0 +1,73 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Result API +Main entry point to use results manager +""" + +import logging +from .storage.storage_api import StorageApi + +class ResultApi(): + """ + Result API + Provides various storage options to use implemented as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + self._logger = logging.getLogger(__name__) + self._storage_handles = [] + + def register_storage(self, storage_api): + """ + Registers ``storage_api`` as an active storage option to use. + """ + self._logger.debug("Loading new Storage API...") + if not isinstance(storage_api, StorageApi): + raise TypeError("incorrect storage type, Required StorageAPI obj") + + storage_api.load_settings() + self._storage_handles.append(storage_api) + self._logger.info(f'{storage_api.name} api registered') + + def unregister_storage(self, storage_api): + """ + Removes registered ``storage_api`` if exists + """ + while storage_api in self._storage_handles: + self._storage_handles.remove(storage_api) + + def unregister_all(self): + """ + Removes all registered storage endpoints + """ + for storage_api in self._storage_handles: + self.unregister_storage(storage_api) + + def store(self, data): + """ + Calls all active storage_api and stores ``data`` in all of them + """ + for api in self._storage_handles: + api.store(data) + + +# pylint: disable=invalid-name +result_api = ResultApi() diff --git a/sdv/docker/sdvstate/tools/result_api/rfile.py b/sdv/docker/sdvstate/tools/result_api/rfile.py new file mode 100644 index 0000000..71a0924 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/rfile.py @@ -0,0 +1,52 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +"""rfile is Object representation of a file for Result_api and storage_api +""" + +import logging + +# pylint: disable=invalid-name + +class rfile(): + """ + rfile object to represent files in Result API + """ + + def __init__(self, data): + """ + Initialisation function + """ + self._logger = logging.getLogger(__name__) + self.hold_data(data) + + def get_data(self): + """ + Returns stored data + """ + if self._data == '': + self._logger.warning('Reading from a empty \'rfile\'') + return self._data + + + def hold_data(self, data): + """ + Holds data of a file + """ + if data is None: + self._logger.warning('Storing an empty \'rfile\'') + data = '' + self._data = data diff --git a/sdv/docker/sdvstate/tools/result_api/storage/__init__.py b/sdv/docker/sdvstate/tools/result_api/storage/__init__.py new file mode 100644 index 0000000..d647f0f --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Abstract class for Storage API +""" diff --git a/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py b/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py new file mode 100644 index 0000000..185ec01 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Local storage api +""" diff --git a/sdv/docker/sdvstate/tools/result_api/storage/local/local.py b/sdv/docker/sdvstate/tools/result_api/storage/local/local.py new file mode 100644 index 0000000..d7dc67b --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/local/local.py @@ -0,0 +1,133 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Local Storage Api +""" + +import logging +import os +import random +import string +import json +from tools.conf import settings # pylint: disable=import-error + +from ..storage_api import StorageApi +from ... import rfile + + + +class Local(StorageApi): + """ + Storage API + Provides abstract class for various storage options to implement as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + super(Local, self).__init__() + self.name = 'Local Storage' + self._logger = logging.getLogger(__name__) + self._path = '' + self._filename = '' + + def store(self, data): + """ + stores ``data``, ``data`` should be a dict + + :param data: dict object to store + """ + if not isinstance(data, dict): + raise TypeError("incorrect data type to store, dict required") + + if not os.path.isfile(self._filename): + with open(self._filename, 'w') as fhandle: + json.dump([], fhandle) + + with open(self._filename, 'r') as fhandle: + records = json.load(fhandle) + temp_data = data.copy() + eval_rfile(temp_data) + records.append(temp_data) + + with open(self._filename, 'w') as fhandle: + self._logger.info(f'{self.name}: New record saved') + json.dump(records, fhandle, indent=4, sort_keys=True, default=str) + + + def load_settings(self): + """ + Load all required settings otherwise set to default + Settings to load: + * ``result_path`` (default: /tmp/local/) + * ``results_filename`` (default: results.json) + """ + try: + path = settings.getValue('results_path') + except AttributeError: + path = '/tmp/local/' + settings.setValue('results_path', path) + + try: + filename = settings.getValue('results_filename') + except AttributeError: + filename = 'results.json' + settings.setValue('results_filename', filename) + + if not os.path.exists(path): + os.makedirs(path) + + self._path = path + self._filename = path + filename + + + + + +def eval_rfile(data): + """ + Find all values of a type rfile in data dict/list and evals them into + actual file + """ + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, rfile): + data[key] = rfile_save(value, key) + else: + eval_rfile(value) + if isinstance(data, list): + for i, _ in enumerate(data): + if isinstance(data[i], rfile): + data[i] = rfile_save(data[i]) + else: + eval_rfile(data[i]) + + +def rfile_save(rfile_obj, prefix='zz'): + """ + Takes rfile Object and stores it into random file returning filename + """ + letters = string.ascii_lowercase + suffix = ''.join(random.choice(letters) for i in range(6)) + filename = settings.getValue('results_path') + f'{prefix}-{suffix}.txt' + if os.path.isfile(filename): + return rfile_save(rfile_obj, prefix) + else: + with open(filename, 'w') as fhandle: + fhandle.write(rfile_obj.get_data()) + return f'{prefix}-{suffix}.txt' diff --git a/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py b/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py new file mode 100644 index 0000000..5ff0d8f --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py @@ -0,0 +1,46 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Abstract class for Storage API +""" + +import logging + + +class StorageApi(): + """ + Storage API + Provides abstract class for various storage options to implement as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + self._logger = logging.getLogger(__name__) + + def store(self, data): + """ + stores ``data`` + """ + raise NotImplementedError() + + def load_settings(self): + """ + Load all required settings + """ + raise NotImplementedError() |