diff options
| author |   Parth Yadav <parth.yadav@ramanujan.du.ac.in> | 2020-08-07 18:16:54 +0530 |
|---|---|---|
| committer | Parth Yadav <parth.yadav@ramanujan.du.ac.in> | 2020-08-17 19:11:14 +0530 |
| commit | 69f4c631e49c359338cff5c9f5b2c96c6fe6b280 (patch) | |
| tree | d7fc7e207aca6ae5b9899d1f1eba27fe29c4c273 | |
| parent | 42af6fa2ef682d28d12952831aa0c74fd647daad (diff) | |
Init SDV-State
This patch adds post cloud-software deployment state validation
tool. Currently supports pod_health_check for Airship deployment.
Signed-off-by: Parth Yadav<parthyadav3105@gmail.com>
Change-Id: I28eeff520f7a00419620bf50cc38fd4793aa31b8
34 files changed, 1623 insertions, 0 deletions
@@ -234,6 +234,8 @@ expected-line-ending-format= # Logging modules to check that the string format arguments are in logging # function parameter format logging-modules=logging +logging-format-style=fstr + [MISCELLANEOUS] diff --git a/sdv/docker/sdvstate/.dockerignore b/sdv/docker/sdvstate/.dockerignore new file mode 100644 index 0000000..1ed3459 --- /dev/null +++ b/sdv/docker/sdvstate/.dockerignore @@ -0,0 +1,23 @@ +# Byte-compiled / optimized / DLL files +**/__pycache__/ +**/*.py[cod] +**/*.confc + +# C extensions +**/*.so + + +# Installer logs +**/pip-log.txt +**/pip-delete-this-directory.txt + + +# Log files: +**/*.log + +# SublimeText +**/*.sublime-project +**/*.sublime-workspace + +**/*.md +**/*.png diff --git a/sdv/docker/sdvstate/Dockerfile b/sdv/docker/sdvstate/Dockerfile new file mode 100644 index 0000000..7d18e7a --- /dev/null +++ b/sdv/docker/sdvstate/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.8-slim-buster + +MAINTAINER Parth Yadav <parthyadav3105@gmail.com> + +WORKDIR /state/ + +COPY requirements.txt /state/requirements.txt +RUN pip install -r requirements.txt +RUN rm requirements.txt + +COPY core/ /state/core/ +COPY settings/ /state/settings/ +COPY tools/ /state/tools/ +COPY validator/ /state/validator/ +COPY state /state/ +COPY server /state/ + +ENV RESULTS_PATH /tmp/ +ENV RESULTS_FILENAME results.json +ENV LOG_VERBOSITY info +ENV SAVE_RESULTS_LOCALLY True + +WORKDIR /data/ + +CMD [ "python", "/state/server" ] diff --git a/sdv/docker/sdvstate/core/__init__.py b/sdv/docker/sdvstate/core/__init__.py new file mode 100644 index 0000000..ed33752 --- /dev/null +++ b/sdv/docker/sdvstate/core/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Core package +contains all program specific dependencies +""" + +from .load_pdf import load_pdf diff --git a/sdv/docker/sdvstate/core/load_pdf.py b/sdv/docker/sdvstate/core/load_pdf.py new file mode 100644 index 0000000..6ce22f0 --- /dev/null +++ b/sdv/docker/sdvstate/core/load_pdf.py @@ -0,0 +1,40 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Loads PDF file into settings +""" + +import json +import yaml + +from tools.conf import settings + +def load_pdf(): + """ + Updates settings with PDF data + """ + filename = settings.getValue('pdf_file') + with open(filename) as handle: + data = handle.read() + + try: + pdf = json.loads(data) + except json.decoder.JSONDecodeError: + try: + pdf = yaml.safe_load(data) + except yaml.parser.ParserError: + raise Exception(f"Invalid PDF file: {filename}") + + settings.setValue('pdf_file', pdf) diff --git a/sdv/docker/sdvstate/example/kubepod10 b/sdv/docker/sdvstate/example/kubepod10 new file mode 100644 index 0000000..2717fc6 --- /dev/null +++ b/sdv/docker/sdvstate/example/kubepod10 @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +clusters: +- cluster: + server: https://10.10.100.21:6553 + insecure-skip-tls-verify: true + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: admin + name: admin@kubernetes +current-context: admin@kubernetes +kind: Config +preferences: {} +users: +- name: admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURZRENDQWtpZ0F3SUJBZ0lVQmN1akh5bmUzMFBnMUw5MnNJZERmWEtlVm5Vd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0tqRVRNQkVHQTFVRUNoTUtTM1ZpWlhKdVpYUmxjekVUTUJFR0ExVUVBeE1LYTNWaVpYSnVaWFJsY3pBZQpGdzB4T1RFd01UY3hOakUzTURCYUZ3MHlNREV3TVRZeE5qRTNNREJhTUNreEZ6QVZCZ05WQkFvVERuTjVjM1JsCmJUcHRZWE4wWlhKek1RNHdEQVlEVlFRREV3VmhaRzFwYmpDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVAKQURDQ0FRb0NnZ0VCQUxRVnpUeG1UTUZHRkdiWks1OGwyUXZkT2hUZVZ2dzVKTWJyVE8wY2hhd1BtdmJXeXczSApMeUFpNTJsZkU5VGdONXBBVzVrVzJmS2tkREMwRnNXZXF2VDV4SFVvbVFGa3RRM2RWMEJnMXRXYVNIdnVHMXQwCndac2hIQWN6RTl0ZS93dFR6ajhkdFl0ZXdIbXpzd1J1bk9sRnFaUVZZT1hReENPYkEvZ2Z1V0o5RUFKNlduZDcKcUhZdEJvbzR0RkhVTmFocDRwUXNNS1VlbDZPUnA4NEM0WnNIenYyZm9Jb2pYd1V2TmJMNUE1VlZjallrK0taZwpCc3IyMWowT0c4N1F3Q0ZuOThMelJqUU92L01FTFRPOEoxemIrK3pvbkg0ZkpDckc1Q2RKNUFQbU81UnBEMGluCmJKNnFOR2QyY0kxaGdVWWx2aWI4QURXc21VelRkWU5wa0JrQ0F3RUFBYU4vTUgwd0RnWURWUjBQQVFIL0JBUUQKQWdXZ01CMEdBMVVkSlFRV01CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFNQmdOVkhSTUJBZjhFQWpBQQpNQjBHQTFVZERnUVdCQlRKSXRERnJwcGR3RExOTWFWY2IzQ1JiNVhBZURBZkJnTlZIU01FR0RBV2dCUjcwU1Z4Ck8wVlpzNXVRQkpaOEZ0cmluT25EaERBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQUFBRUQrQXNaaEFsWnRucTkKR0w5a1dvWDd5d0ZmR2FRdEgvSnFaT0VGYkx0bFdLVmlHc3gyVWlSRTN2U3l6VlFpMVBhNGR3cXF1MXc4bVNIVworc1REVlN1aGE5Q2NlbzcvT3F4dnl3ME43c0t2L0NPeml6YWF5djlXTmlBYXhFNjRMSk1sTWlrS2wrZG1zSlVMCktVUXJLVzhvcnlhZk4zSzZnd3NLS2FtU2Mzb1J4TG9wWHVveEo5a2lyVG5DOWpMVGdWSU1EM0I5aEtleEtLQ3YKb1hKVkUyMWViVnNiOExiSUcyaldRcWlnVktxWEFRN3gwcEt6RFcvN1dIc1JyRFRkbFpYU0ZUZS9IQUpZd2tuVwp1cmd2blJkZ1BYUHl6cHJhWU9iTCtTV3dvejRTS216OGV5TWpQcFd0TkFZQTdIYm5XT3RqU2NXNFJKWnpaQ1V3CldicStNZz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBdEJYTlBHWk13VVlVWnRrcm55WFpDOTA2Rk41Vy9Ea2t4dXRNN1J5RnJBK2E5dGJMCkRjY3ZJQ0xuYVY4VDFPQTNta0JibVJiWjhxUjBNTFFXeFo2cTlQbkVkU2laQVdTMURkMVhRR0RXMVpwSWUrNGIKVzNUQm15RWNCek1UMjE3L0MxUE9QeDIxaTE3QWViT3pCRzZjNlVXcGxCVmc1ZERFSTVzRCtCKzVZbjBRQW5wYQpkM3VvZGkwR2lqaTBVZFExcUduaWxDd3dwUjZYbzVHbnpnTGhtd2ZPL1orZ2lpTmZCUzgxc3ZrRGxWVnlOaVQ0CnBtQUd5dmJXUFE0Ynp0REFJV2Yzd3ZOR05BNi84d1F0TTd3blhOdjc3T2ljZmg4a0tzYmtKMG5rQStZN2xHa1AKU0tkc25xbzBaM1p3aldHQlJpVytKdndBTmF5WlROTjFnMm1RR1FJREFRQUJBb0lCQUYxcnlmS0JaMlFFUk9VdAoyQkZSZ3cxQ2tMVHV1dFZSbDZDUnhIQmxFWHMzQlQ3MElwWmRORGRKcEI3bnNkUUhGUkV5UGRKbkpsMVhydWJ0Ckpic1RHc0RIS1lGVnREb2kwa0lGQnhSZ3FGSmJIU3NkVkpmWE0vQ1Q5b1JOblFsNmVIaVoyeTZtN04wR0pIZCsKSDJvM0w3TmI3aUxpREVoc1NyUGw0T05CSWR6VEFQYy91b3hQbVQxQ2ZiQ3hVU051d1EzOS9mWHJVNzJTOFU4ZAoybXd2dDZpczQ2c09IWkNkNG0xNGJENE11Y2VsUG83V2ptT0hRZlUzd0g1NTE5Q0FtV0hDRFA0ZndNY3dYWlJUClZWUHcyU1VZRW9lMS9DM083cVVqMlRTMldJdysveHZOQml6WFpqajZTdmd5ME1FREtzamhsbWM2OE81MVAvajgKcmh3dFp1RUNnWUVBeTE1c2NuRFVidWFob1dJNzlCVzAxdWlyY0V1Vnp2TXhmc3ptbEJhZnp3dDZPL0FNQ3l1NwpKS2ZNR1JFQmxXR1RGMUhiUlREbGZzK3lTNFpjKzRQUDlmZVNVNFI1NWJkczlZRU44K3liSjJvbzVPcFlGOVFkCmtoL2JQRUZkN3pTbVQ0R2l2V2lxNklqVlFrTGNOVWFlczN4WlZ2d2NqUXd5cENtTU1aRlJtN1VDZ1lFQTRyREsKSTZITUdDcTR4eWZuaUhhT0ZkNlVEYjdmS3ZJNTJZYnU5R0IyR0JXdmkrcEhOZ1V1dU44SG9GT0hQS2NuYnkweQoyS0FwRjVaYTFSZUNHNGdSTE8vMjMybU42VnVCMERGWlNnUEFITTJKd1BtOFUrYjlWZFZaZEI1ZWJrTDhxNXlkCmZqM3F3S2NRVTZMR09wVy83ZEhsMUVUWE9kMUYzTi8wNFdzcGlWVUNnWUJtb0UrNXNKYURJSCtRSVRLQUNqUW4KLzJJRVdTQlFQd2xMSTd0NEg3S2xtUFVtS2d6cDFqZXFWOEwzSTAzWlJGUW1BSGpXZ2NaT0tDR2hXenl3NytPUwpERTBiT0U4TFRYVCtyeEdMZG1zVmlNejZPQWdjZmo0dDcwV0RNcmxrYlAxQVFmc04ralBGQk1nWm1BUG9IcXNYCmlEak5YSXhMNFV2czY4cURlUUhsd1FLQmdGTGs3UFg4cTJKRzlReTJsZDc3NDFjeDdoZmNyVVRLRU1kdnBSK3QKeW1GaVJMQTRPbFFScnhVaFVXdWFQOEM1S3gxbmZNbGtQOEtGVTYvS2llUkJiRzV2VFdwQzhnYmNWR3JxTU1sMAo5NkpRc3NmalNxK3Zyd0hkSTNubnhRWXk3cXhlZCtUN0JVWHZrWFBUK1FMaFVhN0lhMitrd01OREc5SDUvMVVTCjE3eUZBb0dBZjBubW53RjJRMTZVYXhhanI2M3hjUFlQS09FY1pHTFcxalhoMHVpNFJnK3lscEdSZ25xdVJzMk8KL3RDYTlUYm1JcG9KZHA3aWFPOTIzenI1MWphcnlBOCtuWWhoZ2dRQ29IdWNIY0ZBR213Ryt6R2NyMlBZYklseAo5TkVsUEFZM2pndFNWTW4yUkhMak0wUWVuTUQ1aG1HcHQvWVJOd3hPNkNBdXhaNUhzOTQ9Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/sdv/docker/sdvstate/example/kubepod15 b/sdv/docker/sdvstate/example/kubepod15 new file mode 100644 index 0000000..7710cbc --- /dev/null +++ b/sdv/docker/sdvstate/example/kubepod15 @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +clusters: +- cluster: + server: https://10.10.150.21:6553 + insecure-skip-tls-verify: true + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: admin + name: admin@kubernetes +current-context: admin@kubernetes +kind: Config +preferences: {} +users: +- name: admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURZRENDQWtpZ0F3SUJBZ0lVRFZ1T2IvczEyKzR1dGNTRnBuOEhQbFlIVWFBd0RRWUpLb1pJaHZjTkFRRUwKQlFBd0tqRVRNQkVHQTFVRUNoTUtTM1ZpWlhKdVpYUmxjekVUTUJFR0ExVUVBeE1LYTNWaVpYSnVaWFJsY3pBZQpGdzB4T1RFeU1UVXdNelExTURCYUZ3MHlNREV5TVRRd016UTFNREJhTUNreEZ6QVZCZ05WQkFvVERuTjVjM1JsCmJUcHRZWE4wWlhKek1RNHdEQVlEVlFRREV3VmhaRzFwYmpDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVAKQURDQ0FRb0NnZ0VCQUo0Z3R2ZUMyVktnV1BwRHRMTWE3d2k0aE1hZlBEZUVveUg2LzBrWUdKWEF6TTRuVE55NApZdXVldCtBZUdDNnJ6cHNDRG1GcVBrVkRRM1ZkMEsrd05VSXZmOGpZVXlMbWVXUEZxNStqV25SaHpSbUVyT2VBCk9UK0lMa0pFMUN3T2hPbEtjMlB0TjhPUzdFbVR1NmxkZHQ4OXM5Z1M5aXNmbm5JQmY2YkhNdGdqWWJrZEEzbEQKb0VLL282VS9LdkpydTN2L01IRXl6VUwwbjB4UHpHK3ZPVDRpRVZRV3A3M3o0d2gzalN4SENvQmJ4RU9hTk5mSgpoQjNFMUZhSTZMY3U4VHdWdnZ3WStHc3Z3NURXblJ5VXczL0REUXpNMGNQZkc2WUNmeWhjQkVJSUJ5ZEtwUTdYCi9NZ0p4MWV1QmRHdVFheHNaNHhvS0taZW4vQWhCbWZDTUVjQ0F3RUFBYU4vTUgwd0RnWURWUjBQQVFIL0JBUUQKQWdXZ01CMEdBMVVkSlFRV01CUUdDQ3NHQVFVRkJ3TUJCZ2dyQmdFRkJRY0RBakFNQmdOVkhSTUJBZjhFQWpBQQpNQjBHQTFVZERnUVdCQlJMYUVVWXRRaGxMQnFCQUtJdTRrUDRwWWhRTlRBZkJnTlZIU01FR0RBV2dCU2g3bE54CmJXZ1pUSjZKRkUwdHhTdGdIS3hqd3pBTkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQXBpWXFVaUMyNFVLM2YxUW8KMnp1YTlTKzVaeWZXOTgvcG9zWDVFZ0x2c25uYmJoaXRFeXptc1RQaG1yNTZ2WmNkVlVjb3B4NFc2M0luaDRUYQpHQlBUMjVwdGZIVEE1bTNURDIrb1dFQXlKMHhBbjR3M0VpdzRhYmY0aCs1Q0JlTm9ldXJlOXhMYlIzNnZZSG9aCnQ0aVk1Q3BraHhud3VLV0FZTnE4a2lsQTlvUzV2bm5ndUMxYVJEckQ5bTJLZlk1aWtiRndGWWUzRzRLTXAyaUgKWVpiMUxhZ3BlZHRjbTJSNnhNZ0RVSktKbkN5WFpIcXp1WHMzT1h1TTFRVzZlMVl2VU1aQUdMV25NYkJ2S3MzNQpyMUdsdFY5OUh0WHBoTnBqeFd6a1RNS0s2K0wwQ0xxNXducVZjVzNUK1Y1V05HbkhWMThBMkhEM0NUc3NRWmxBCm5pbGVXdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb2dJQkFBS0NBUUVBbmlDMjk0TFpVcUJZK2tPMHN4cnZDTGlFeHA4OE40U2pJZnIvU1JnWWxjRE16aWRNCjNMaGk2NTYzNEI0WUxxdk9td0lPWVdvK1JVTkRkVjNRcjdBMVFpOS95TmhUSXVaNVk4V3JuNk5hZEdITkdZU3MKNTRBNVA0Z3VRa1RVTEE2RTZVcHpZKzAzdzVMc1NaTzdxVjEyM3oyejJCTDJLeCtlY2dGL3BzY3kyQ05odVIwRAplVU9nUXIranBUOHE4bXU3ZS84d2NUTE5RdlNmVEUvTWI2ODVQaUlSVkJhbnZmUGpDSGVOTEVjS2dGdkVRNW8wCjE4bUVIY1RVVm9qb3R5N3hQQlcrL0JqNGF5L0RrTmFkSEpURGY4TU5ETXpSdzk4YnBnSi9LRndFUWdnSEowcWwKRHRmOHlBbkhWNjRGMGE1QnJHeG5qR2dvcGw2ZjhDRUdaOEl3UndJREFRQUJBb0lCQUJ5cFkyQ3p0LzZSRCsrMAo3QUQyNWRMWDEwRkZSWjN1amI4d0JxdlNFVXE3bXFQWFhjZzRKNzM3aytxc3FjZHozc3diOEUxWis1V0VYcXJjCmFXSWU5MWhhMGJldTlrckNLY2lhNE1QYjBSNTlSN2JUWkovRmp4cmo3VGFYMFRsM0hFSkkrMmRtYlJBbkJtdEQKdXVVMUNzSG1KajRKR2RPeE5JQUhvNEt3WXBmb2NPME9acFhVZFlOQktwSUhseFhOWjJ4RkJiVzh6a1FRekZ6MAplWmQ3YzZNUmlaZFRPd0pqNWl4c0FWSytBTXVGRkFSamNmc1FBZktlS2J2YUdDTFBvQmFSak5US0Z6MEVhWmlZClNTM2NYMDRCTnk0NWNPVWlHK3RsSlgxeEhGT04weittMlArZWQ1dCtHSHB3UUg4ak1ZQzhkZlJCUVdSeGNCTjcKemd3NWp5RUNnWUVBeEM4REVPWm9QNXhOcFZwbXB6OVAvY2NiVE9XekdUTlZpZzFhcmdSMC90RWRRK0lKQlpDdgpqMzlWVk9FeDUrSlJHVmhmSG9NczlXZVZuSmJSZTFyN1QrWU1WOVFCOVYvbDVhdkJoc0Vhc1NpaThsVEpzT3dXCmJRTDQyMDd1QVhGemFoMHhPS253T2gwN3p0TTBFMXQwWFVrR1BFSzE2bldPbFc4K3IyOEEvSmNDZ1lFQXpsZEgKSlo5RTUxT0dmZU96Y2VQdWV0ZU9PYnNvWVU4eHhOaUg0MitKWVpKNFVqNVY2RGw4OHdEaS91NVNVOVBtUWM1dAozOFpncXdRRjRFWklTbk0rbElKZnluUmhmWU9YZXB4bEJnVFBVb2dUeGY1bm5jZjNOWGZrVVJxUWViZURqTEdjClBrbU1LbE9kK21jRnYxdGI4eXhzbVEwbjhFVWplVkdLT3JKVjc5RUNnWUJ1NC9QanRaanZlN1lYNVFWcE84eEgKTWlnb1N4MzAvS001S1ZzOFNhQ24rQ09HbjFsaUgrcGNQaWxKbFJEVWRZUkp3ejNnelZ5NFNoaXpManl5Y1RiawpickJEWkw3R3A3SVhKQUo3M09MdGlINnlZMkt0OG9TcWthZUFyeGl4RUNPZ3MyZURFK3VKcmNTRW43VXJ5K0gyCmFMUnhrM09vVjFLRS9TQjlvVXo1ZVFLQmdFZ1g4b0hRbmhCOC9IYXJ3aHkrMktvTyttQnRaZlJwNlNldngvck4KRTZFRnZnaHVRekc2TkUvck5XU0EvRDdSd0plcGVuWS9KN05ZMm55NzBiSkJoZEg1bzJKbk8xRFJVM0hCaHdLTgpWNnFzWk13KzBSRXR0cy8xcmM0d2k5NGJJbGxjRFEwdVFVemdua2ZKQ3hjSzRwdWFIKzl4eTB5RnU1azl4aUF3CkF4cWhBb0dBUlc3Qno1UjlSOWZKVUp3ZEhvMGpRVmhrL3lzSWlmQWRxQ3d6blpOcVM4cU9KMXpsSExhWkozMXcKbVdjNzA3UUN6Q3BOMk1YV2lnMzc2VVJpdXFtcEJTZW14bzFRendhQWJhK0Yvd1I1VzlncndzTmZ2RDR6TkVHbgp2dFllSS9taXlJOVFaay9PVkcrblRLL1ZIZExha3FOVFNKQUl6WSttZ2Y4SWphUTUrVW89Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
\ No newline at end of file diff --git a/sdv/docker/sdvstate/example/state.yml b/sdv/docker/sdvstate/example/state.yml new file mode 100644 index 0000000..1ca61e1 --- /dev/null +++ b/sdv/docker/sdvstate/example/state.yml @@ -0,0 +1,17 @@ +# This is a comment + +# values are stored in key:value format +## keys are case-insensitive +## values can be int, float, string, dict, list, bool + + +## Path to PDF file +PDF_FILE: example/intel-pod10.json + +############# +# Airship arguments +############# + +# Path to kube-config file +KUBE_CONFIG : example/kubepod10 + diff --git a/sdv/docker/sdvstate/requirements.txt b/sdv/docker/sdvstate/requirements.txt new file mode 100644 index 0000000..415b2ae --- /dev/null +++ b/sdv/docker/sdvstate/requirements.txt @@ -0,0 +1,4 @@ +PyYAML == 5.3.1 +pylint == 2.4.4 +tornado == 6.0.4 +kubernetes == 11.0.0 diff --git a/sdv/docker/sdvstate/server b/sdv/docker/sdvstate/server new file mode 100755 index 0000000..ca37eca --- /dev/null +++ b/sdv/docker/sdvstate/server @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Server +""" + + +import logging +import subprocess + +from tornado.web import Application +from tornado.ioloop import IOLoop +import tornado.concurrent +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web +import tornado.log + + +# pylint: disable=W0223 + +class SwPostStateValidator(tornado.web.RequestHandler): + """ + Post Cloud Software Deployment State Validation Tool REST Request Handler + """ + + # def set_default_headers(self): + # """ + # """ + # self.set_header('Content-Type', 'application/json') + + + def get(self): + """ + get request + + usage: + option 1: passing parameters as key=value list + /?params='KEY1=value1,KEY2=value2' + + option 2: passing parameters in conf-file + /?config=filename + + :return: logs from test results + """ + conf_params = self.get_argument('params', None) + if conf_params is not None: + conf_params = conf_params.replace(',', ';') + conf_file = self.get_argument('config', None) + pdf_file = self.get_argument('pdf', None) + + cmd = ['python3', '/state/state'] + + if conf_params is not None: + cmd.extend(['--conf-params', f'{conf_params}']) + + if conf_file is not None: + cmd.extend(['--conf-file', f'/data/{conf_file}']) + + if pdf_file is not None: + cmd.extend(['--pdf-file', f'/data/{pdf_file}']) + + + logger = logging.getLogger(__name__) + command = " ".join(cmd) + logger.info(f'# Executing: [ {command} ]') + + + process = subprocess.Popen(cmd, stdout=subprocess.PIPE) + output = process.communicate()[0] + returncode = process.returncode + + if returncode == 0: + results = output + else: + results = {'error': 'failed to retrieve results'} + + self.finish(results) + + + +class UploadHandler(tornado.web.RequestHandler): + """ + Upload File + """ + + def post(self): + """ + post request + + usage: + + /upload + pass a file in `file` variable in header + """ + data = self.request.files['file'][0] + fname = data['filename'] + + with open("/data/" + fname, 'wb+') as ofile: + ofile.write(data['body']) + + self.finish("{upload : ok}") + + + +def server_main_block(): + """ + Main Function + """ + + app = Application([('/', SwPostStateValidator), + ('/upload', UploadHandler)]) + + # Cli Config + tornado.options.define("port", default=80, help="run on the given port", type=int) + tornado.options.parse_command_line() + + + # Server Config + http_server = tornado.httpserver.HTTPServer(app) + http_server.listen(tornado.options.options.port) + + + # Tornado's event loop handles it from here + print("# Servering.... \n [Ctrl + C] to quit") + + + # Logging + log_file_filename = "/var/log/tornado.log" + handler = logging.FileHandler(log_file_filename) + app_log = logging.getLogger("tornado.general") + tornado.log.enable_pretty_logging() + app_log.addHandler(handler) + + + try: + tornado.ioloop.IOLoop.instance().start() + except KeyboardInterrupt: + tornado.ioloop.IOLoop.instance().stop() + + # start + IOLoop.instance().start() + + +if __name__ == "__main__": + server_main_block() diff --git a/sdv/docker/sdvstate/settings/common.yml b/sdv/docker/sdvstate/settings/common.yml new file mode 100644 index 0000000..65f131c --- /dev/null +++ b/sdv/docker/sdvstate/settings/common.yml @@ -0,0 +1,27 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +################################## +# Program behavior configurations +################################## + +# Logging +log_filename: state-validation.log +log_verbosity: info + +# Results +results_dir: /tmp/state/ +save_results_locally: True
\ No newline at end of file diff --git a/sdv/docker/sdvstate/settings/installer.yml b/sdv/docker/sdvstate/settings/installer.yml new file mode 100644 index 0000000..6a97c6b --- /dev/null +++ b/sdv/docker/sdvstate/settings/installer.yml @@ -0,0 +1,29 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +##################### +## Airship +###################### + +# Default Airship namespace list +# list all kubernetes namespace that have Airship components +airship_namespace_list: + - kube-system + - ceph + - ucp + - osh-infra + - tenant-ceph + - openstack
\ No newline at end of file diff --git a/sdv/docker/sdvstate/settings/result.yml b/sdv/docker/sdvstate/settings/result.yml new file mode 100644 index 0000000..409fd0f --- /dev/null +++ b/sdv/docker/sdvstate/settings/result.yml @@ -0,0 +1,19 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +# Note all paths must end with '/' +results_path: /tmp/state/ +results_filename: results.json
\ No newline at end of file diff --git a/sdv/docker/sdvstate/state b/sdv/docker/sdvstate/state new file mode 100755 index 0000000..41d17a4 --- /dev/null +++ b/sdv/docker/sdvstate/state @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + + +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +"""(Post)Deployment State Validation tool +""" + +import argparse +import logging +import os +import re +import ast +import sys +from datetime import datetime + +from tools.conf import settings +from tools.result_api import result_api, Local +from core import load_pdf +from validator import AirshipValidator + + +VERBOSITY_LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL +} + +_LOGGER = logging.getLogger() +_CURR_DIR = os.path.dirname(os.path.realpath(__file__)) + + + +def parse_param_string(values): + """ + Parse and split a single '--test-params' argument. + + This expects either 'x=y', 'x=y,z' or 'x' (implicit true) + values. For multiple overrides use a ; separated list for + e.g. --test-params 'x=z; y=(a,b)' + """ + results = {} + + if values == '': + return {} + + for param, _, value in re.findall('([^;=]+)(=([^;]+))?', values): + param = param.strip() + value = value.strip() + if param: + if value: + # values are passed inside string from CLI, so we must retype them accordingly + try: + results[param] = ast.literal_eval(value) + except ValueError: + # for backward compatibility, we have to accept strings without quotes + _LOGGER.warning(f'Adding missing quotes around string value: {param} = {str(value)}') + results[param] = str(value) + else: + results[param] = True + return results + +def parse_arguments(): + """ + Parse command line arguments. + """ + + class _SplitTestParamsAction(argparse.Action): + """ + Parse and split '--conf-params' arguments. + + This expects either a single list of ; separated overrides + as 'x=y', 'x=y,z' or 'x' (implicit true) values. + e.g. --test-params 'x=z; y=(a,b)' + Or a list of these ; separated lists with overrides for + multiple tests. + e.g. --test-params "['x=z; y=(a,b)','x=z']" + """ + def __call__(self, parser, namespace, values, option_string=None): + if values[0] == '[': + input_list = ast.literal_eval(values) + parameter_list = [] + for test_params in input_list: + parameter_list.append(parse_param_string(test_params)) + else: + parameter_list = parse_param_string(values) + results = parameter_list + setattr(namespace, self.dest, results) + + + class _ValidateFileAction(argparse.Action): + """Validate a file can be read from before using it. + """ + def __call__(self, parser, namespace, values, option_string=None): + if not os.path.isfile(values): + raise argparse.ArgumentTypeError( + 'the path \'%s\' is not a valid path' % values) + if not os.access(values, os.R_OK): + raise argparse.ArgumentTypeError( + 'the path \'%s\' is not accessible' % values) + + setattr(namespace, self.dest, values) + + def list_logging_levels(): + """Give a summary of all available logging levels. + + :return: List of verbosity level names in decreasing order of + verbosity + """ + return sorted(VERBOSITY_LEVELS.keys(), + key=lambda x: VERBOSITY_LEVELS[x]) + + parser = argparse.ArgumentParser(description='(Post)Deployment State Validation tool') + + parser.add_argument('--version', action='version', version='%(prog)s 0.1') + + parser.add_argument('--log-verbosity', choices=list_logging_levels(), + help='logging level') + parser.add_argument('-s', '--save-results-locally', action="store_true", + default=None, help='turn on local storage of results') + + group = parser.add_argument_group('test configuration options') + group.add_argument('--conf-file', action=_ValidateFileAction, + help='settings file') + group.add_argument('--conf-params', action=_SplitTestParamsAction, + help='csv list of conf-file parameters: key=val; e.g. ' + 'KUBE_CONFIG=/path/to/kubeconfig/file;PDF_FILE=path/to/pdf/file' + ' or a list of csv lists of conf-file parameters: key=val; e.g. ' + '[\'KUBE_CONFIG=/path/to/kubeconfig/file\',' + '\'PDF_FILE=path/to/pdf/file\']') + + group = parser.add_argument_group('override conf-file options') + group.add_argument('--pdf-file', help='Path to PDF file') + + + + args = vars(parser.parse_args()) + + return args + + +def main(): + """Main Function + """ + args = parse_arguments() + + ################################## + # Load settings: + # precedence: cli > env > conf-file > settings-dir + ################################## + + settings.load_from_dir(os.path.join(_CURR_DIR, 'settings')) + + if args["conf_file"]: + settings.load_from_file(os.path.join(_CURR_DIR, args["conf_file"])) + + settings.load_from_env() + + if args['conf_params']: + args = {**args.pop('conf_params'), **args} #merge 2 dicts + for key in args: + settings.setValue(key, args[key]) + + + ################################## + # Results settings: + ################################## + now = datetime.now().strftime("%d-%m-%Y_%H-%M-%S/") + results_path = settings.getValue('results_dir') + now + settings.setValue('results_path', results_path) + if not os.path.exists(results_path): + os.makedirs(results_path) + + + ################################## + # Logs settings: + ################################## + + log_file = results_path + settings.getValue('log_filename') + stream_handler = logging.StreamHandler(sys.stdout) + file_handler = logging.FileHandler(filename=log_file) + + level = settings.getValue('LOG_VERBOSITY') + _LOGGER.setLevel(logging.DEBUG) + stream_handler.setLevel(VERBOSITY_LEVELS[level]) + file_handler.setLevel(VERBOSITY_LEVELS[level]) + + form = '[%(levelname)-8s] : %(msg)s' + if level == 'debug': + form = '["%(asctime)s"][%(levelname)-8s][%(pathname)s:%(funcName)s:%(lineno)-4s] : %(msg)s' + stream_handler.setFormatter(logging.Formatter(form)) + file_handler.setFormatter(logging.Formatter(form)) + + _LOGGER.addHandler(stream_handler) + _LOGGER.addHandler(file_handler) + + + ################################## + # ResultAPI settings: + ################################## + + if settings.getValue('save_results_locally'): + result_api.register_storage(Local()) + + + #### + # Still Developing + #### + load_pdf() + + installer = settings.getValue('pdf_file')["deployment_info"]["installer_used"].lower() + + if installer == 'airship': + airship = AirshipValidator() + airship.validate() + + + + +if __name__ == "__main__": + main() diff --git a/sdv/docker/sdvstate/tools/conf/__init__.py b/sdv/docker/sdvstate/tools/conf/__init__.py new file mode 100644 index 0000000..67c808f --- /dev/null +++ b/sdv/docker/sdvstate/tools/conf/__init__.py @@ -0,0 +1,120 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + + +"""Settings and configuration handlers. + +Settings will be loaded from several .yaml or .yml files +and any user provided settings file. +""" + +import os +import ast + +import yaml + +# pylint: disable=invalid-name + +class Settings(): + """Holding class for settings. + """ + def __init__(self): + pass + + + def getValue(self, attr): + """ + Return a settings item value + """ + try: + attr = attr.lower() + return getattr(self, attr) + except AttributeError: + raise AttributeError("{obj} object has no attribute \ + {attr}".format(obj=self.__class__, attr=attr)) + + + def setValue(self, name, value): + """Set a value + """ + if name is not None and value is not None: + super(Settings, self).__setattr__(name.lower(), value) + + + def load_from_file(self, path): + """Update ``settings`` with values found in module at ``path``. + """ + with open(path) as file: + configs = yaml.load_all(file, Loader=yaml.SafeLoader) + for conf in configs: + for name, value in conf.items(): + self.setValue(name, value) + + + def load_from_env(self): + """ + Update ``settings`` with values found in the environment. + """ + for key in os.environ: + value = os.environ[key] + + #evaluate string to python type + try: + value = ast.literal_eval(os.environ[key]) + except (ValueError, SyntaxError): + pass #already string + + self.setValue(key, value) + + + def load_from_dir(self, dir_path): + """Update ``settings`` with contents of the yaml files at ``path``. + + Files are read in ascending order, hence if a configuration item + exists in more than one file, then the setting in the file that + occurs in the last read file will have high precedence and + overwrite previous values. + + Same precedence logic for sub-directories. + Also, child directory will have more precedence than it's parent + + :param dir_path: The full path to the dir from which to load the + yaml files. + + :returns: None + """ + files = list_yamls(dir_path) + + for file in files: + self.load_from_file(file) + + +settings = Settings() + + +def list_yamls(dir_path): + """Get all yaml files recursively in ``dir_path`` + """ + files = [] + dir_list = [x[0] for x in os.walk(dir_path)] + dir_list.sort() + for path in dir_list: + dir_files = [path+'/'+f for f in os.listdir(path) + if f.endswith('.yaml') or f.endswith('.yml')] + if dir_files is not None: + dir_files.sort() + files.extend(dir_files) + return files diff --git a/sdv/docker/sdvstate/tools/kube_utils.py b/sdv/docker/sdvstate/tools/kube_utils.py new file mode 100644 index 0000000..3468ac6 --- /dev/null +++ b/sdv/docker/sdvstate/tools/kube_utils.py @@ -0,0 +1,85 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +""" +Kubernetes cluster api helper functions +""" + +from kubernetes import client, config +from kubernetes.stream import stream + +from tools.conf import settings # pylint: disable=import-error + + +def load_kube_api(): + """ + Loads kubernetes api + """ + config.load_kube_config(settings.getValue('kube_config')) + api = client.CoreV1Api() + settings.setValue('kube_api', api) + + +def kube_api(): + """ + Returns kube_api object + """ + return settings.getValue('kube_api') + + +def get_pod_with_labels(labels): + """ + Returns json details any one pod with matching labels + + :param labels: labels to find matching pod + :return: pod details + """ + api = kube_api() + pod = api.list_pod_for_all_namespaces(label_selector=labels).items[0] + return pod + + +def kube_exec(pod, cmd): + """ + Executes `cmd` inside `pod` and returns response + + :param pod: pod object + :param cmd: command to execute inside pod + :return: response from pod + """ + api = kube_api() + response = stream(api.connect_get_namespaced_pod_exec, + pod.metadata.name, pod.metadata.namespace, command=cmd, + stderr=True, stdin=False, stdout=True, tty=False) + return response + + +def kube_curl(*args): + """ + executes curl cmd in kubernetes network + + :param args: comma separated list of args to pass to curl + :return: http response + """ + args = list(args) + args.insert(0, "curl") + + labels = "application=prometheus-openstack-exporter,component=exporter" + pod = get_pod_with_labels(labels) + + response = kube_exec(pod, args) + + return response diff --git a/sdv/docker/sdvstate/tools/result_api/__init__.py b/sdv/docker/sdvstate/tools/result_api/__init__.py new file mode 100644 index 0000000..0633103 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +""" +Result Manager Package +""" +from .result_api import result_api +from .rfile import rfile + +from .storage.storage_api import StorageApi +from .storage.local.local import Local diff --git a/sdv/docker/sdvstate/tools/result_api/result_api.py b/sdv/docker/sdvstate/tools/result_api/result_api.py new file mode 100644 index 0000000..95dfd56 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/result_api.py @@ -0,0 +1,73 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Result API +Main entry point to use results manager +""" + +import logging +from .storage.storage_api import StorageApi + +class ResultApi(): + """ + Result API + Provides various storage options to use implemented as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + self._logger = logging.getLogger(__name__) + self._storage_handles = [] + + def register_storage(self, storage_api): + """ + Registers ``storage_api`` as an active storage option to use. + """ + self._logger.debug("Loading new Storage API...") + if not isinstance(storage_api, StorageApi): + raise TypeError("incorrect storage type, Required StorageAPI obj") + + storage_api.load_settings() + self._storage_handles.append(storage_api) + self._logger.info(f'{storage_api.name} api registered') + + def unregister_storage(self, storage_api): + """ + Removes registered ``storage_api`` if exists + """ + while storage_api in self._storage_handles: + self._storage_handles.remove(storage_api) + + def unregister_all(self): + """ + Removes all registered storage endpoints + """ + for storage_api in self._storage_handles: + self.unregister_storage(storage_api) + + def store(self, data): + """ + Calls all active storage_api and stores ``data`` in all of them + """ + for api in self._storage_handles: + api.store(data) + + +# pylint: disable=invalid-name +result_api = ResultApi() diff --git a/sdv/docker/sdvstate/tools/result_api/rfile.py b/sdv/docker/sdvstate/tools/result_api/rfile.py new file mode 100644 index 0000000..71a0924 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/rfile.py @@ -0,0 +1,52 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +"""rfile is Object representation of a file for Result_api and storage_api +""" + +import logging + +# pylint: disable=invalid-name + +class rfile(): + """ + rfile object to represent files in Result API + """ + + def __init__(self, data): + """ + Initialisation function + """ + self._logger = logging.getLogger(__name__) + self.hold_data(data) + + def get_data(self): + """ + Returns stored data + """ + if self._data == '': + self._logger.warning('Reading from a empty \'rfile\'') + return self._data + + + def hold_data(self, data): + """ + Holds data of a file + """ + if data is None: + self._logger.warning('Storing an empty \'rfile\'') + data = '' + self._data = data diff --git a/sdv/docker/sdvstate/tools/result_api/storage/__init__.py b/sdv/docker/sdvstate/tools/result_api/storage/__init__.py new file mode 100644 index 0000000..d647f0f --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Abstract class for Storage API +""" diff --git a/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py b/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py new file mode 100644 index 0000000..185ec01 --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/local/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Local storage api +""" diff --git a/sdv/docker/sdvstate/tools/result_api/storage/local/local.py b/sdv/docker/sdvstate/tools/result_api/storage/local/local.py new file mode 100644 index 0000000..d7dc67b --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/local/local.py @@ -0,0 +1,133 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Local Storage Api +""" + +import logging +import os +import random +import string +import json +from tools.conf import settings # pylint: disable=import-error + +from ..storage_api import StorageApi +from ... import rfile + + + +class Local(StorageApi): + """ + Storage API + Provides abstract class for various storage options to implement as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + super(Local, self).__init__() + self.name = 'Local Storage' + self._logger = logging.getLogger(__name__) + self._path = '' + self._filename = '' + + def store(self, data): + """ + stores ``data``, ``data`` should be a dict + + :param data: dict object to store + """ + if not isinstance(data, dict): + raise TypeError("incorrect data type to store, dict required") + + if not os.path.isfile(self._filename): + with open(self._filename, 'w') as fhandle: + json.dump([], fhandle) + + with open(self._filename, 'r') as fhandle: + records = json.load(fhandle) + temp_data = data.copy() + eval_rfile(temp_data) + records.append(temp_data) + + with open(self._filename, 'w') as fhandle: + self._logger.info(f'{self.name}: New record saved') + json.dump(records, fhandle, indent=4, sort_keys=True, default=str) + + + def load_settings(self): + """ + Load all required settings otherwise set to default + Settings to load: + * ``result_path`` (default: /tmp/local/) + * ``results_filename`` (default: results.json) + """ + try: + path = settings.getValue('results_path') + except AttributeError: + path = '/tmp/local/' + settings.setValue('results_path', path) + + try: + filename = settings.getValue('results_filename') + except AttributeError: + filename = 'results.json' + settings.setValue('results_filename', filename) + + if not os.path.exists(path): + os.makedirs(path) + + self._path = path + self._filename = path + filename + + + + + +def eval_rfile(data): + """ + Find all values of a type rfile in data dict/list and evals them into + actual file + """ + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, rfile): + data[key] = rfile_save(value, key) + else: + eval_rfile(value) + if isinstance(data, list): + for i, _ in enumerate(data): + if isinstance(data[i], rfile): + data[i] = rfile_save(data[i]) + else: + eval_rfile(data[i]) + + +def rfile_save(rfile_obj, prefix='zz'): + """ + Takes rfile Object and stores it into random file returning filename + """ + letters = string.ascii_lowercase + suffix = ''.join(random.choice(letters) for i in range(6)) + filename = settings.getValue('results_path') + f'{prefix}-{suffix}.txt' + if os.path.isfile(filename): + return rfile_save(rfile_obj, prefix) + else: + with open(filename, 'w') as fhandle: + fhandle.write(rfile_obj.get_data()) + return f'{prefix}-{suffix}.txt' diff --git a/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py b/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py new file mode 100644 index 0000000..5ff0d8f --- /dev/null +++ b/sdv/docker/sdvstate/tools/result_api/storage/storage_api.py @@ -0,0 +1,46 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Abstract class for Storage API +""" + +import logging + + +class StorageApi(): + """ + Storage API + Provides abstract class for various storage options to implement as + plugin to storage api + """ + + def __init__(self): + """ + Initialization function + """ + self._logger = logging.getLogger(__name__) + + def store(self, data): + """ + stores ``data`` + """ + raise NotImplementedError() + + def load_settings(self): + """ + Load all required settings + """ + raise NotImplementedError() diff --git a/sdv/docker/sdvstate/validator/__init__.py b/sdv/docker/sdvstate/validator/__init__.py new file mode 100644 index 0000000..0e1fb38 --- /dev/null +++ b/sdv/docker/sdvstate/validator/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Package for Validators +""" + +from .validator import Validator +from .airship.airship import AirshipValidator diff --git a/sdv/docker/sdvstate/validator/airship/airship.py b/sdv/docker/sdvstate/validator/airship/airship.py new file mode 100644 index 0000000..e77f06f --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/airship.py @@ -0,0 +1,51 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Airship Validator +""" + +import logging +import ast +import json + +from tools.conf import settings +from tools.result_api import result_api, rfile +from tools.kube_utils import * +from validator.validator import Validator + +## Checks +from .pod_health_check import pod_health_check + + + +class AirshipValidator(Validator): + """Class for Airship Validation + """ + + def __init__(self): + """ + Initialisation function. + """ + super(AirshipValidator, self).__init__() + self._logger = logging.getLogger(__name__) + + load_kube_api() + + + def validate(self): + """ + """ + pod_health_check() diff --git a/sdv/docker/sdvstate/validator/airship/pod_health_check.py b/sdv/docker/sdvstate/validator/airship/pod_health_check.py new file mode 100644 index 0000000..34a6747 --- /dev/null +++ b/sdv/docker/sdvstate/validator/airship/pod_health_check.py @@ -0,0 +1,94 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +import logging +from kubernetes import client, config + +from tools.kube_utils import kube_api +from tools.conf import settings +from tools.result_api import result_api, rfile + + + +def pod_health_check(): + """ + Check health of all pods and get logs of failed pods + """ + api = kube_api() + namespace_list = settings.getValue('airship_namespace_list') + for namespace in namespace_list: + pod_list = api.list_namespaced_pod(namespace) + for pod in pod_list.items: + result = pod_status(pod) + if result['state'] == 'fail': + result['logs'] = get_logs(pod) + result_api.store(result) + + + +def pod_status(pod): + """ + Check health of a pod and returns it's status as result + """ + result = {'state': 'ok', + 'kind': 'pod', + 'name': pod.metadata.name, + 'namespace': pod.metadata.namespace, + 'node': pod.spec.node_name} + + if pod.status.container_statuses is None: + result['state'] = 'fail' + result['pod_details'] = rfile(str(pod)) + else: + for container in pod.status.container_statuses: + if container.state.running is not None: + status = 'Running' + if container.state.terminated is not None: + status = container.state.terminated.reason + if container.state.waiting is not None: + status = container.state.waiting.reason + + if status not in ('Running', 'Completed'): + result['state'] = 'fail' + result['pod_details'] = rfile(str(pod)) + + info = f'[Health: {result["state"]}] Name: {result["name"]}, ' + info = info + f'Namespace: {result["namespace"]}, Node: {result["node"]}' + + logger = logging.getLogger(__name__) + logger.info(info) + return result + + +def get_logs(pod): + """ + Collects logs of all containers in ``pod`` + """ + api = kube_api() + logs = [] + if pod.status.container_statuses is not None: + for container in pod.status.container_statuses: + con = {'container': container.name} + if container.state.waiting is not None and \ + container.state.waiting.reason == 'PodInitializing': + log = 'Not found, status: waiting, reason: PodInitializing' + else: + log = api.read_namespaced_pod_log(name=pod.metadata.name, + namespace=pod.metadata.namespace, + container=container.name) + con['log'] = rfile(log) + logs.append(con) + return logs diff --git a/sdv/docker/sdvstate/validator/validator.py b/sdv/docker/sdvstate/validator/validator.py new file mode 100644 index 0000000..4f36008 --- /dev/null +++ b/sdv/docker/sdvstate/validator/validator.py @@ -0,0 +1,27 @@ +# Copyright 2020 University Of Delhi. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Interface for Software Validators +""" + +class Validator(): + """ + Interface for Software to Validate + """ + def __init__(self): + """Initialisation function. + """ +
\ No newline at end of file diff --git a/sdv/docs/docker/state/development/design/index.rst b/sdv/docs/docker/state/development/design/index.rst new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sdv/docs/docker/state/development/design/index.rst diff --git a/sdv/docs/docker/state/development/overview/index.rst b/sdv/docs/docker/state/development/overview/index.rst new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sdv/docs/docker/state/development/overview/index.rst diff --git a/sdv/docs/docker/state/development/overview/result-api-architecture.png b/sdv/docs/docker/state/development/overview/result-api-architecture.png Binary files differnew file mode 100644 index 0000000..e464187 --- /dev/null +++ b/sdv/docs/docker/state/development/overview/result-api-architecture.png diff --git a/sdv/docs/docker/state/development/overview/result_api.rst b/sdv/docs/docker/state/development/overview/result_api.rst new file mode 100644 index 0000000..12fd62c --- /dev/null +++ b/sdv/docs/docker/state/development/overview/result_api.rst @@ -0,0 +1,178 @@ +Result API +========== + +Table of contents +^^^^^^^^^^^^^^^^^ + +- `Description <#Description>`__ +- `Architecture <#Architecture>`__ +- `Managing configuration <#Managing%20configuration>`__ +- `How to Use <#How%20to%20Use>`__ + +---------------------- + +Description +~~~~~~~~~~~ + +Result API is very simple write-only API for storing results of any size +to multiple storage endpoints. + +Results API focuses on only sending test results to required +destinations so that they can be later viewed by user but do not +implements read functionality as this is not required while running +tests + +Usage is very simple: +''''''''''''''''''''' + +.. code:: python + + from result_api import result_api as api + from result_api import Local + + # First register storage enpoints + endpoint = Local() + api.register_storage(endpoint) + + # Now, use anywhere in program + api.store(data) + +Note: In above example we used only one storage endpoints so, whenever +``api`` will make call to ``store()`` method it will be used. But if +register multiple storage endpoints then whenever ``api`` will make call +to ``store()`` method all endpoints will be called to store data. This +package doesn't allows to selectively make calls to different storage +endpoint for storing. Instead follows **one api -> many endpoint (one to +many)** design. + +---------------------- + +Architecture +~~~~~~~~~~~~ + +|img| + +*ResultsAPI exports data to various storage endpoints like harddisk, +SQL, Mongo, etc. exposed by StorageAPI in write-only mode* + +---------------------- + +Managing configuration +~~~~~~~~~~~~~~~~~~~~~~ + +Result API uses `Conf: Program Settings handler <../conf/readme.md>`__ +to manage all it's settings. So, user can change any settings of +ResultAPI as they need. ``Conf`` allows to change settings in two ways +either read from file ``settings.load_from_file(path/to/file)`` or set +inside program ``settings.setValue(key, Value)`` + +Configurations available: + ++-----------------+---------------------+-------------------------------------+ +| Storage | setting | optional | ++=================+=====================+=====================================+ +| Local Storage | results\_path | Yes, defaults to ``/tmp/local/`` | ++-----------------+---------------------+-------------------------------------+ +| Local Storage | results\_filename | Yes, defaults to ``results.json`` | ++-----------------+---------------------+-------------------------------------+ + +---------------------- + +How to Use +~~~~~~~~~~ + +For using ResultAPI successfully the following steps are required + +#. **Import result\_api instance** from ResultAPI package as this + instance will be commonly used across our program. +#. **Import required StorageAPI** +#. **Set Storage settings** or load settings from file +#. **Create Storage Endpoints from StorageAPI** +#. **Register Storage Endpoints with result\_api** + +Now, result\_api is ready to use. We can send values to all registered +storage endpoints by making a simple call to ```store()`` +method <#store()%20method>`__ + +store() method +'''''''''''''' + +``result_api`` has ``store(dict)`` method for storing data. + +.. code:: python + + # data must be a dict + data = { 'key1': "value1", 'key2': 5, 'dumykeytext': "dummy string value"} + result_api.store(data) + +rfile +''''' + +rfile stands for result file. When you have extremely big value for a +key you would like to tell your storage endpoint to store them in +separate file and refer them in actual key. For storing value of type +file in result\_api use rfile construct. + +.. code:: python + + data = { 'name': 'example', 'myfile': rfile('Text to store in this file')} + result_api.store(data) + # Respective StorageAPI will evaluate "data" for all rfile values and store their text in some separate file/storage-object and put there refernece in "data" + +Use Local Storage with Result API +''''''''''''''''''''''''''''''''' + +.. code:: python + + from result_api import result_api as api + from result_api import Local + from result_api import rfile + + def main(): + # Update settings required for Local storage + settings.setValue('results_path', '/tmp/myresults/') + settings.setValue('results_filename', 'results.json') + + # Register Local() to result_api, this will load above settings automatically + api.register_storage(Local()) + + data = { 'testcase': "RA1.24", 'value': 'Pass', 'logs': rfile('These are logs')} + + # Now, store any data + api.store(data) + +Register Storage Endpoint +''''''''''''''''''''''''' + +.. code:: python + + from result_api.storage.mystorageapi import MyStorageAPI + + # Set required settings values for storage endpoint + settings.setValue('mysetting1', 'value') + settings.setValue('mysetting2', 'value2') + + #Now Register StorageAPI to ResultAPI + endpoint = MyStorageAPI() + api.register_storage(endpoint) + +Changing settings +''''''''''''''''' + +.. code:: python + + # Set values in a yaml file and load it + settings.load_from_file('result.yml') + + # Or set in program + settings.setValue('key', 'value') + + # Note above steps will only change settings values but will not impact any previously registered storage endpoints + # To use endpoints with new value, register new endpoints + endpoint = MyStorageAPI() + api.register_storage(endpoint) + + # And do not forget to unregister old endpoint as they have old settings + api.unregister_storage(old_endpoint) + +.. |img| image:: result-api-architecture.png diff --git a/sdv/docs/docker/state/development/overview/settings.rst b/sdv/docs/docker/state/development/overview/settings.rst new file mode 100644 index 0000000..c70f21e --- /dev/null +++ b/sdv/docs/docker/state/development/overview/settings.rst @@ -0,0 +1,38 @@ +Program Settings handler +------------------------ + +``from tools.conf import settings`` + +Settings will be loaded from several ``.yaml`` or ``.yml`` files and any +user provided settings files. + +So, that user can use these settings values across program. + +This utility loads settings from yaml files in form of key and value +where key is always ``string`` while value can be of type python: + +- ``int`` e.g. 5, 45, 1234 +- ``str`` e.g. hello, world +- ``float`` e.g. 34.56, 12.7 +- ``list`` e.g. [ ‘month’ , ‘is’, 45 ] +- ``dict`` e.g. {‘program’: ‘sdv’, ‘language’: ‘python’} +- ``bool`` e.g. True, False + +keys are case-insensitive +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The utility is case-insensitive to keys used as it automatically +converts all keys to lower case. + +E.g. ``Program : sdv``, ``program : sdv``, ``PrOgRam : sdv`` all are +same. + +- ``settings.load_from_file(path/to/file)`` +- ``settings.load_from_env()`` +- ``settings.load_from_dir(directory/to/search/yamls)`` + +``settings.load_from_dir()`` reads all yaml files in given directory and +all it’s sub-directory recursively in ascending order, hence if a +configuration item exists in more than one file, then the setting in the +file that occurs in the last read file will have high precedence and +overwrite previous values. .
\ No newline at end of file diff --git a/sdv/docs/docker/state/release/configguide/index.rst b/sdv/docs/docker/state/release/configguide/index.rst new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sdv/docs/docker/state/release/configguide/index.rst diff --git a/sdv/docs/docker/state/release/userguide/index.rst b/sdv/docs/docker/state/release/userguide/index.rst new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/sdv/docs/docker/state/release/userguide/index.rst |
