Initialize k8s test suite for Airship

This patch updates sdv framework to support multiple test suites.
Current test suites:
    * default
    * k8s
Current Cloud Installers:
    * Airship

The patch also adds makefile for sdv framework to ease development work.
Some useful makefile target:
    * make sandbox
    * make run
    * make lint
    * make bash
Run `make help` for complete usage guide.

The patch restructures the codebases directory structure.

Signed-off-by: Parth Yadav<parthyadav3105@gmail.com>
Change-Id: I109d13f84334ec1cfa4f9c17b74d38a979272ea5

19 files changed, 207 insertions, 40 deletions

diff --git a/sdv/docker/sdvstate/Dockerfile b/sdv/docker/sdvstate/Dockerfile
index 7d18e7a..10bc678 100644
--- a/sdv/docker/sdvstate/Dockerfile
+++ b/sdv/docker/sdvstate/Dockerfile
@@ -2,18 +2,17 @@ FROM python:3.8-slim-buster
 
 MAINTAINER Parth Yadav <parthyadav3105@gmail.com>
 
-WORKDIR /state/
+WORKDIR /sdv/
 
-COPY requirements.txt /state/requirements.txt
+COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt
 RUN rm requirements.txt
 
-COPY core/ /state/core/
-COPY settings/ /state/settings/
-COPY tools/ /state/tools/
-COPY validator/ /state/validator/
-COPY state /state/
-COPY server /state/
+COPY internal/ internal/
+COPY settings/ settings/
+COPY tools/ tools/
+COPY state .
+COPY server .
 
 ENV RESULTS_PATH /tmp/
 ENV RESULTS_FILENAME results.json
diff --git a/sdv/docker/sdvstate/Makefile b/sdv/docker/sdvstate/Makefile
new file mode 100644
index 0000000..1cd9b78
--- /dev/null
+++ b/sdv/docker/sdvstate/Makefile
@@ -0,0 +1,133 @@
+# Set the shell to bash always
+SHELL := /bin/bash
+
+
+#############
+# Options
+############
+
+IMG_TAG="latest"
+WORKDIR="workdir"
+
+###################
+
+# helper for printing yellow-bold formatted messages in makefile.
+# Usage: $S some text message $E
+S=printf "\n\e[1;33m[makefile]:
+E=\e[0m\n"
+
+
+define HELP_MSG
+
+Usage: make [target] arguments=value
+_________________________________________________________________
+Targets:
+
+make sandbox
+	creates new sandbox container.
+
+make sandbox-clean
+	deletes sandbox container.
+
+make build
+	builds container image for sdv.
+
+make bash
+	opens a bash to sandbox contianer
+
+make run
+	runs sdvstate checks inside sandbox container.
+
+make lint
+	lints sdvstate code and generates report.
+
+make setup-dev
+	creates workdir for local developement files.
+
+_________________________________________________________________
+Optional arguments:
+
+IMG_TAG
+	Currently set to '$(IMG_TAG)'
+
+endef
+export HELP_MSG
+help:
+	@echo "$$HELP_MSG"
+
+
+# The following target enables all commands to run in same shell. This enables
+# exit to kill the program
+.ONESHELL:
+
+
+define CONF
+## Path to PDF file
+PDF_FILE: github.com/nfvid/sites/blob/master/intel-pod10.json
+
+#############
+# Airship arguments
+#############
+
+# Path to kube-config file
+KUBE_CONFIG : /sdv/workdir/example/config
+
+MASTER_ROLE_NAME : masters
+WORKER_ROLE_NAME : workers
+endef
+export CONF
+setup-dev:
+	@if [ ! -d "$(WORKDIR)" ]; then \
+		mkdir $(WORKDIR); \
+		mkdir "$(WORKDIR)/example"; \
+		echo "$$CONF" >> "$(WORKDIR)/example/conf.yaml"; \
+		$S: Created local workdir. $E; \
+	fi
+
+
+build:
+	$S: building sdv image... $E
+	@docker build . -t sdv:$(IMG_TAG) -f Dockerfile
+
+
+sandbox-clean:
+	@if [[ "$(shell docker container ls --format "{{.Names}}" | grep sdvsandbox)" == "sdvsandbox" ]]; then \
+	  docker container stop sdvsandbox; \
+	fi
+
+sandbox: build sandbox-clean
+	docker container run --rm -d --name sdvsandbox \
+	-v $(PWD):/sdv/ \
+	-v $(PWD)/workdir/result:/tmp/state \
+	sdv:latest /bin/bash -c "while true; do sleep 10; done;";
+
+
+bash:
+	docker container exec -it sdvsandbox /bin/bash
+
+define sandbox_bash
+	if [[ "$(shell docker container ls --format "{{.Names}}" | grep sdvsandbox)" == "" ]]; then \
+		echo "Sandbox container does not exists. Creating Sandbox..."; \
+		make sandbox; \
+	fi
+	docker container exec -it sdvsandbox bash -c $(1);
+endef
+
+run:
+	@$(call sandbox_bash, "/sdv/state --conf-file /sdv/workdir/example/conf.yaml")
+
+lint:
+	@cp ../../../pylintrc $(WORKDIR)/pylintrc
+	@$(call sandbox_bash, \
+	"pylint --rcfile=/sdv/workdir/pylintrc /sdv/state /sdv/server /sdv/internal /sdv/tools")
+
+
+test:
+	@echo "Todo"
+
+all: help
+
+
+.PHONY: help setup-dev build sandbox-clean sandbox bash lint test run all 
+
+
diff --git a/sdv/docker/sdvstate/example/state.yml b/sdv/docker/sdvstate/example/state.yml
deleted file mode 100644
index 89dc548..0000000
--- a/sdv/docker/sdvstate/example/state.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# This is a comment
-
-# values are stored in key:value format
-## keys are case-insensitive
-## values can be int, float, string, dict, list, bool
-
-
-## Path to PDF file
-PDF_FILE: example/intel-pod10.json
-
-#############
-# Airship arguments
-#############
-
-# Path to kube-config file
-KUBE_CONFIG : example/kubepod10
-
-MASTER_ROLE_NAME : masters
-WORKER_ROLE_NAME : workers
-
diff --git a/sdv/docker/sdvstate/core/__init__.py b/sdv/docker/sdvstate/internal/__init__.py
index 47830c5..47830c5 100644
--- a/sdv/docker/sdvstate/core/__init__.py
+++ b/sdv/docker/sdvstate/internal/__init__.py
diff --git a/sdv/docker/sdvstate/core/display_report.py b/sdv/docker/sdvstate/internal/display_report.py
index 97ccb55..97ccb55 100644
--- a/sdv/docker/sdvstate/core/display_report.py
+++ b/sdv/docker/sdvstate/internal/display_report.py
diff --git a/sdv/docker/sdvstate/core/load_pdf.py b/sdv/docker/sdvstate/internal/load_pdf.py
index fa2bf7e..fff2d67 100644
--- a/sdv/docker/sdvstate/core/load_pdf.py
+++ b/sdv/docker/sdvstate/internal/load_pdf.py
@@ -17,9 +17,9 @@
 """
 
 import json
-import yaml
-import requests
 import os
+import requests
+import yaml
 
 from tools.conf import settings
 
@@ -28,14 +28,16 @@ def load_pdf():
     Updates settings with PDF data
     """
     path = settings.getValue('pdf_file')
-    data=""
+    data = ""
     if os.path.exists(path):
         with open(path) as handle:
             data = handle.read()
     else:
-        if (path.find("github.com") != -1):
+        if path.find("github.com") != -1:
             path = path.replace("github.com", "raw.githubusercontent.com")
             path = path.replace("/blob", "")
+            if path[:8] != "https://":
+                path = "https://" + path
         try:
             resp = requests.get(path)
             if resp.status_code == requests.codes.ok:
diff --git a/sdv/docker/sdvstate/validator/__init__.py b/sdv/docker/sdvstate/internal/validator/__init__.py
index 0e1fb38..0e1fb38 100644
--- a/sdv/docker/sdvstate/validator/__init__.py
+++ b/sdv/docker/sdvstate/internal/validator/__init__.py
diff --git a/sdv/docker/sdvstate/validator/airship/__init__.py b/sdv/docker/sdvstate/internal/validator/airship/__init__.py
index 78e42c4..78e42c4 100644
--- a/sdv/docker/sdvstate/validator/airship/__init__.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/__init__.py
diff --git a/sdv/docker/sdvstate/validator/airship/airship.py b/sdv/docker/sdvstate/internal/validator/airship/airship.py
index f2bdebd..6f1109d 100644
--- a/sdv/docker/sdvstate/validator/airship/airship.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/airship.py
@@ -22,7 +22,7 @@ from datetime import datetime as dt
 
 from tools.conf import settings
 from tools.kube_utils import load_kube_api, delete_kube_curl_pod
-from validator.validator import Validator
+from internal.validator.validator import Validator
 
 from . import *
 
@@ -59,9 +59,27 @@ class AirshipValidator(Validator):
         """
 
         self._report['scenario'] = 'none'
-        self._report['case_name'] = 'ook_airship'
         self._report['start_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S')
 
+        test_suite = settings.getValue("test_suite")
+
+        if test_suite == "default":
+            self._report['case_name'] = 'ook_airship'
+            self.default_suite()
+
+        if test_suite == "k8s":
+            self._report['case_name'] = 'k8s_airship'
+            self.k8s_suite()
+
+        delete_kube_curl_pod()
+
+        self._report['stop_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S')
+
+
+    def default_suite(self):
+        """
+        Default Test Suite
+        """
 
         # PLATFORM CHECKS
         self.update_report(pod_health_check())
@@ -92,9 +110,10 @@ class AirshipValidator(Validator):
         self.update_report(nova_scheduler_filters_check())
         self.update_report(cpu_allocation_ratio_check())
 
-        delete_kube_curl_pod()
-
-        self._report['stop_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S')
+    def k8s_suite(self):
+        """
+        Kubernetes Platform Test Suite
+        """
 
 
     def update_report(self, result):
diff --git a/sdv/docker/sdvstate/validator/airship/ceph_check.py b/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py
index b33e876..b33e876 100644
--- a/sdv/docker/sdvstate/validator/airship/ceph_check.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/ceph_check.py
diff --git a/sdv/docker/sdvstate/validator/airship/compute_check.py b/sdv/docker/sdvstate/internal/validator/airship/compute_check.py
index a602471..a602471 100644
--- a/sdv/docker/sdvstate/validator/airship/compute_check.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/compute_check.py
diff --git a/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py b/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py
index 3754299..3754299 100644
--- a/sdv/docker/sdvstate/validator/airship/monitoring_logging_agent_check.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/monitoring_logging_agent_check.py
diff --git a/sdv/docker/sdvstate/validator/airship/network_check.py b/sdv/docker/sdvstate/internal/validator/airship/network_check.py
index bddf579..bddf579 100644
--- a/sdv/docker/sdvstate/validator/airship/network_check.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/network_check.py
diff --git a/sdv/docker/sdvstate/validator/airship/pod_health_check.py b/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py
index 0093ffc..0093ffc 100644
--- a/sdv/docker/sdvstate/validator/airship/pod_health_check.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/pod_health_check.py
diff --git a/sdv/docker/sdvstate/validator/airship/store_result.py b/sdv/docker/sdvstate/internal/validator/airship/store_result.py
index 52f4e10..52f4e10 100644
--- a/sdv/docker/sdvstate/validator/airship/store_result.py
+++ b/sdv/docker/sdvstate/internal/validator/airship/store_result.py
diff --git a/sdv/docker/sdvstate/validator/validator.py b/sdv/docker/sdvstate/internal/validator/validator.py
index 4f36008..4f36008 100644
--- a/sdv/docker/sdvstate/validator/validator.py
+++ b/sdv/docker/sdvstate/internal/validator/validator.py
diff --git a/sdv/docker/sdvstate/settings/testsuits.yaml b/sdv/docker/sdvstate/settings/testsuits.yaml
new file mode 100644
index 0000000..dd154b9
--- /dev/null
+++ b/sdv/docker/sdvstate/settings/testsuits.yaml
@@ -0,0 +1,15 @@
+# Copyright 2020 University Of Delhi.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+test_suite: default
\ No newline at end of file
diff --git a/sdv/docker/sdvstate/state b/sdv/docker/sdvstate/state
index 353df71..0df9592 100755
--- a/sdv/docker/sdvstate/state
+++ b/sdv/docker/sdvstate/state
@@ -31,9 +31,9 @@ import requests
 
 from tools.conf import settings
 from tools.result_api import result_api, Local
-from core import load_pdf
-from core import display_report
-from validator import AirshipValidator
+from internal import load_pdf
+from internal import display_report
+from internal.validator import AirshipValidator
 
 
 VERBOSITY_LEVELS = {
@@ -146,6 +146,7 @@ def parse_arguments():
                        ' or a list of csv lists of conf-file parameters: key=val; e.g. '
                        '[\'KUBE_CONFIG=/path/to/kubeconfig/file\','
                        '\'PDF_FILE=path/to/pdf/file\']')
+    group.add_argument('--test-suite', help='set of checks to perform. values: default, k8s')
 
     group = parser.add_argument_group('override conf-file options')
     group.add_argument('--pdf-file', help='Path to PDF file')
diff --git a/sdv/docker/sdvstate/tools/__init__.py b/sdv/docker/sdvstate/tools/__init__.py
new file mode 100644
index 0000000..b8cd56d
--- /dev/null
+++ b/sdv/docker/sdvstate/tools/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2020 University Of Delhi.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+Tools package
+"""