Bug Fixes: Fix failing http based health checks && Vswitch core checks

This Patch ensures sdvstate support for Airship 1.7 and Airship 1.8.

* Http based health check:
        Added a sandbox pod for curl utility for sending https requests
        to pods within cluster. Few Airship components are required to
        be checked by sending http request on their status endpoints,
        like Kibana.

* Fix config file parsing bug for:
        1. vswitch_pmd_cores_check and
        2. vswitch_dpdk_lcores_check

Signed-off-by: Parth Yadav<parthyadav3105@gmail.com>
Change-Id: I0864b9fd269c189b6e9807123da43dcf1fe1cbef

3 files changed, 83 insertions, 6 deletions

diff --git a/sdv/docker/sdvstate/tools/kube_utils.py b/sdv/docker/sdvstate/tools/kube_utils.py
index 3468ac6..9c543db 100644
--- a/sdv/docker/sdvstate/tools/kube_utils.py
+++ b/sdv/docker/sdvstate/tools/kube_utils.py
@@ -18,6 +18,8 @@
 Kubernetes cluster api helper functions
 """
 
+import time
+
 from kubernetes import client, config
 from kubernetes.stream import stream
 
@@ -77,9 +79,67 @@ def kube_curl(*args):
     args = list(args)
     args.insert(0, "curl")
 
-    labels = "application=prometheus-openstack-exporter,component=exporter"
-    pod = get_pod_with_labels(labels)
 
-    response = kube_exec(pod, args)
+    try:
+        pod = get_pod_with_labels("application=sdvstate-curl")
+    except IndexError:
+        create_kube_curl_pod()
+        pod = get_pod_with_labels("application=sdvstate-curl")
+    finally:
+        response = kube_exec(pod, args)
 
     return response
+
+
+def create_kube_curl_pod():
+    """
+    Create a sandbox pod(image: curlimages/curl:7.76.1) for
+    curl utility inside kubernetes cluster.
+    """
+    print(("Creating pod sdvstate-curl..."))
+    pod_manifest = {
+        'apiVersion': 'v1',
+        'kind': 'Pod',
+        'metadata': {
+            'name': 'sdvstate-curl',
+            'labels': {
+                'application': 'sdvstate-curl'
+            }
+        },
+        'spec': {
+            'containers': [{
+                'image': 'curlimages/curl:7.76.1',
+                'name': 'sdvstate-curl',
+                'command': ["/bin/sh"],
+                "args": [
+                    "-c",
+                    "while true; do sleep 5; done"
+                ]
+            }]
+        }
+    }
+
+    api = kube_api()
+    response = api.create_namespaced_pod(body=pod_manifest,
+                                         namespace='default')
+    # wait 1 minute or less for pod to create.
+    seconds_left = 60
+    while seconds_left:
+        resp = api.read_namespaced_pod(name='sdvstate-curl',
+                                       namespace='default')
+        if resp.status.phase == 'Running':
+            break
+        time.sleep(3)
+        seconds_left -= 3
+
+    if seconds_left == 0:
+        raise Exception("sdvstate-curl pod taking took long to create, tests failed...")
+
+
+def delete_kube_curl_pod():
+    """
+    Cleans curl utility pod
+    """
+    api = kube_api()
+    api.delete_namespaced_pod(name='sdvstate-curl', body={},
+                              namespace='default')
diff --git a/sdv/docker/sdvstate/validator/airship/airship.py b/sdv/docker/sdvstate/validator/airship/airship.py
index 18de66d..f2bdebd 100644
--- a/sdv/docker/sdvstate/validator/airship/airship.py
+++ b/sdv/docker/sdvstate/validator/airship/airship.py
@@ -21,7 +21,7 @@ import logging
 from datetime import datetime as dt
 
 from tools.conf import settings
-from tools.kube_utils import load_kube_api
+from tools.kube_utils import load_kube_api, delete_kube_curl_pod
 from validator.validator import Validator
 
 from . import *
@@ -92,6 +92,8 @@ class AirshipValidator(Validator):
         self.update_report(nova_scheduler_filters_check())
         self.update_report(cpu_allocation_ratio_check())
 
+        delete_kube_curl_pod()
+
         self._report['stop_date'] = dt.now().strftime('%Y-%m-%d %H:%M:%S')
 
 
diff --git a/sdv/docker/sdvstate/validator/airship/compute_check.py b/sdv/docker/sdvstate/validator/airship/compute_check.py
index ff6f6db..a602471 100644
--- a/sdv/docker/sdvstate/validator/airship/compute_check.py
+++ b/sdv/docker/sdvstate/validator/airship/compute_check.py
@@ -18,6 +18,7 @@ Compute Related Checks
 
 import configparser
 import json
+import re
 
 from tools.kube_utils import kube_exec, get_pod_with_labels
 from tools.conf import settings
@@ -291,7 +292,14 @@ def trace_vswitch_pmd_cores():
     cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config']
     response = kube_exec(ovs_pod, cmd)
 
-    response.replace('=', ':')
+    # convert config str to json str
+    match = re.findall("[a-zA-Z0-9-]+=", response)
+    for key in match:
+        response = response.replace(key, '"' + key[:-1] + '":')
+    match = re.findall(":[a-zA-Z0-9-]+", response)
+    for key in match:
+        response = response.replace(key[1:], '"' + key[1:] + '"')
+
     config = json.loads(response)
 
     if 'pmd-cpu-mask' in config:
@@ -330,7 +338,14 @@ def trace_vswitch_dpdk_lcores():
     cmd = ['ovs-vsctl', '-t', '5', 'get', 'Open_vSwitch', '.', 'other_config']
     response = kube_exec(ovs_pod, cmd)
 
-    response.replace('=', ':')
+    # convert config str to json str
+    match = re.findall("[a-zA-Z0-9-]+=", response)
+    for key in match:
+        response = response.replace(key, '"' + key[:-1] + '":')
+    match = re.findall(":[a-zA-Z0-9-]+", response)
+    for key in match:
+        response = response.replace(key[1:], '"' + key[1:] + '"')
+
     config = json.loads(response)
 
     if 'dpdk-lcore-mask' in config: