Clover initial commit for servicemesh/route_rules,

orchestration/kube_client, and tools/clover_validate_rr

Add an 'orchestration' directory. Please note that
'orchestration' does NOT mean Clover does any orchestration ---
similar to how Clover doesn't by itself implement tracing or
logging, orchestration is a directory for code related to Docker
orchestration client --- such as k8s client

kube_client utilizes the Kubernetes python client (a dependency)
to perform tasks against Kubernetes API server. For this commit,
it is only tested for weighted route rule verification, it does
three tasks:
(1) get a list of pods under a namespace --- pod dictionary now
    only contains pod name and label dictionary: used to match
    pod name with the node name in traces from OpenTracing
(2) check to see if a particular pod is up in a particular
    namespace: used to check if Istio pods are running in
    istio-system namespace
(3) check if a container exists in a list of pods under a
    namespace: used to check if application pods have
    istio-proxy container running

route_rule directly invokes istioctl as there isn't any Istio
Python client yet.  Currently it reads and parses routerules
from Istio, and validates if a particular trace result matches
the routerules

Finally, a sample tool clover_validate_rr is provided. This
tool assumes a previous test has been ran (with an id with
both the route-rule-under-test and corresponding traces are
stored --- currently the assumption is tests were ran with
redis-master running on system). The tool can be invoked:

python clover_validate_rr.py -t <test-id> -s <service name>

where test-id is the ID of the test (most likely uuid) and
service name is the name of the service running in the
Kubernetes cluster upon which test traces should be fetched
against

Change-Id: Ic8ab6efc23c71ac4643bee796ef986a86f6fc7dd
Signed-off-by: Stephen Wong <stephen.kf.wong@gmail.com>

4 files changed, 300 insertions, 0 deletions

diff --git a/clover/orchestration/Pipfile b/clover/orchestration/Pipfile
new file mode 100644
index 0000000..12f776c
--- /dev/null
+++ b/clover/orchestration/Pipfile
@@ -0,0 +1,19 @@
+[[source]]
+
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+
+[dev-packages]
+
+
+
+[packages]
+
+kubernetes = "*"
+
+
+[requires]
+
+python_version = "2.7"
diff --git a/clover/orchestration/Pipfile.lock b/clover/orchestration/Pipfile.lock
new file mode 100644
index 0000000..4831a48
--- /dev/null
+++ b/clover/orchestration/Pipfile.lock
@@ -0,0 +1,178 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "97b3bd99fc2b3c80b1109170700a7d2a8e0b4330eaba769f7bc7aaa7f4a42925"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "2.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "cachetools": {
+            "hashes": [
+                "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe",
+                "sha256:ede01f2d3cbd6ddc9e35e16c2b0ce011d8bb70ce0dbaf282f5b4df24b213bc5d"
+            ],
+            "version": "==2.0.1"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296",
+                "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"
+            ],
+            "version": "==2018.1.18"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "google-auth": {
+            "hashes": [
+                "sha256:34088434cb2a2409360b8f3cbc04195a465df1fb2aafad71ebbded77cbf08803",
+                "sha256:9051802d3dae256036cca9e34633a32c0ed1427730d4ebc513dff91ec8b6dd45"
+            ],
+            "version": "==1.4.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f",
+                "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4"
+            ],
+            "version": "==2.6"
+        },
+        "ipaddress": {
+            "hashes": [
+                "sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"
+            ],
+            "version": "==1.0.19"
+        },
+        "kubernetes": {
+            "hashes": [
+                "sha256:2f1a05a9bb2549d6afb6d138b2767d61d8aeb735a7a12bf554440524205e2894",
+                "sha256:f81f145882471a1dd9d23360e99bd77027f07744729ef2728af4af7130cd19fd"
+            ],
+            "index": "pypi",
+            "version": "==5.0.0"
+        },
+        "oauthlib": {
+            "hashes": [
+                "sha256:ce57b501e906ff4f614e71c36a3ab9eacbb96d35c24d1970d2539bbc3ec70ce1"
+            ],
+            "version": "==2.0.6"
+        },
+        "pyasn1": {
+            "hashes": [
+                "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89",
+                "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee",
+                "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5",
+                "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43",
+                "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b",
+                "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b",
+                "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626",
+                "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15",
+                "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1",
+                "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2",
+                "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc",
+                "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b"
+            ],
+            "version": "==0.4.2"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:041e9fbafac548d095f5b6c3b328b80792f006196e15a232b731a83c93d59493",
+                "sha256:0cdca76a68dcb701fff58c397de0ef9922b472b1cb3ea9695ca19d03f1869787",
+                "sha256:0cea139045c38f84abaa803bcb4b5e8775ea12a42af10019d942f227acc426c3",
+                "sha256:0f2e50d20bc670be170966638fa0ae603f0bc9ed6ebe8e97a6d1d4cef30cc889",
+                "sha256:47fb6757ab78fe966e7c58b2030b546854f78416d653163f0ce9290cf2278e8b",
+                "sha256:598a6004ec26a8ab40a39ea955068cf2a3949ad9c0030da970f2e1ca4c9f1cc9",
+                "sha256:72fd8b0c11191da088147c6e4678ec53e573923ecf60b57eeac9e97433e09fc2",
+                "sha256:854700bbdd01394e2ada9c1bfbd0ed9f5d0c551350dbbd023e88b11d2771ae06",
+                "sha256:af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc",
+                "sha256:b1f395cae2d669e0830cb023aa86f9f283b7a9aa32317d7f80d8e78aa2745812",
+                "sha256:c6747146e95d2b14cc2a8399b2b0bde3f93778f8f9ec704690d2b589c376c137",
+                "sha256:f53fe5bcebdf318f51399b250fe8325ef3a26d927f012cc0c8e0f9e9af7f9deb"
+            ],
+            "version": "==0.2.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:07009062406cffd554a9b4135cd2ff167c9bf6b7aac61fe946c93e69fad1bbd8",
+                "sha256:8f95bb7e6edbb2456a51a1fb58c8dca942024b4f5844cae62c90aa88afe6e300"
+            ],
+            "version": "==2.7.0"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
+                "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736",
+                "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f",
+                "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608",
+                "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8",
+                "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab",
+                "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7",
+                "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3",
+                "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1",
+                "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6",
+                "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8",
+                "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4",
+                "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca",
+                "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269"
+            ],
+            "version": "==3.12"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
+                "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
+            ],
+            "version": "==2.18.4"
+        },
+        "requests-oauthlib": {
+            "hashes": [
+                "sha256:50a8ae2ce8273e384895972b56193c7409601a66d4975774c60c2aed869639ca",
+                "sha256:883ac416757eada6d3d07054ec7092ac21c7f35cb1d2cf82faf205637081f468"
+            ],
+            "version": "==0.8.0"
+        },
+        "rsa": {
+            "hashes": [
+                "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5",
+                "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd"
+            ],
+            "version": "==3.4.2"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
+                "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
+            ],
+            "version": "==1.22"
+        },
+        "websocket-client": {
+            "hashes": [
+                "sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6",
+                "sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149"
+            ],
+            "version": "==0.47.0"
+        }
+    },
+    "develop": {}
+}
diff --git a/clover/orchestration/__init__.py b/clover/orchestration/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/orchestration/__init__.py
diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py
new file mode 100644
index 0000000..e5f1d89
--- /dev/null
+++ b/clover/orchestration/kube_client.py
@@ -0,0 +1,103 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from os import path
+import yaml
+
+from kubernetes import client, config
+
+class KubeClient(object):
+
+    def __init__(self):
+        config.load_kube_config()
+        self.core_v1 = client.CoreV1Api()
+        self.extensions_v1beta1 = client.ExtensionsV1beta1Api()
+
+    def find_svc_by_namespace(self, svc_name, namespace='default'):
+        ret_dict = {}
+        try:
+            svc = self.core_v1.read_namespaced_service(name=svc_name,
+                                                       namespace=namespace)
+        except client.rest.ApiException:
+            svc = None
+        if not svc:
+            print('found no service %s in namespace %s' \
+                   % (svc_name, namespace))
+            return None
+        ret_dict[svc.metadata.name] = {}
+        ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels
+        ret_dict[svc.metadata.name]['selector'] = svc.spec.selector
+
+        return ret_dict
+
+    def find_pod_by_namespace(self, namespace='default'):
+        ret_dict = {}
+        pods = self.core_v1.list_namespaced_pod(namespace=namespace)
+        if not pods:
+            print('found no pod')
+            return None
+        for pod in pods.items:
+            if pod.metadata.name not in ret_dict:
+                ret_dict[pod.metadata.name] = {}
+            ret_dict[pod.metadata.name]['labels'] = pod.metadata.labels
+
+        return ret_dict
+
+    def _check_pod(self, pod_name, namespace='defualt', container_name=None):
+        ret = self.core_v1.list_namespaced_pod(namespace=namespace)
+        ret_code = False
+        new_pod_name = None
+        for i in ret.items:
+            if pod_name in i.metadata.name:
+                if i.status.container_statuses and len(i.status.container_statuses) > 0:
+                    container_up = False
+                    for container in i.status.container_statuses:
+                        check_state = True
+                        if container_name:
+                            if container_name != container.name:
+                                check_state = False
+                        if check_state and container.state.running is not None:
+                            container_up = True
+                        else:
+                            if container_up:
+                                container_up = False
+                                break
+                    if container_up:
+                        ret_code = True
+                        new_pod_name = i.metadata.name
+        return ret_code, new_pod_name
+
+    def check_pod_up(self, pod_name, namespace='default'):
+        return self._check_pod(pod_name, namespace)
+
+    def check_container_in_pods(self, container_name, pods, namespace='default'):
+        ret = False
+        for pod in pods:
+            ret, _ = self._check_pod(pod, namespace, container_name)
+            if not ret:
+                return ret
+        return ret
+
+    def create_deployment_yaml(self, deployment_yaml_path, namespace='default'):
+        with open(deployment_yaml_path) as fp:
+            body = yaml.load(fp)
+            resp = self.extensions_v1beta1.create_namespaced_deployment(
+                    body=body, namespace=namespace)
+            print('Deployment created. Status=%s' % str(resp.status))
+
+            dep_name = body.get('metadata').get('name')
+            return dep_name
+
+    def create_service_yaml(self, service_yaml_path, namespace='default'):
+        with open(service_yaml_path) as fp:
+            body = yaml.load(fp)
+            resp = self.extensions_v1beta1.create_namespaced_service(
+                    body=body, namespace=namespace)
+            print('Service created. Status=%s' % str(resp.status))
+
+            svc_name = body.get('metadata').get('name')
+            return svc_name