Clover initial commit for servicemesh/route_rules,

orchestration/kube_client, and tools/clover_validate_rr

Add an 'orchestration' directory. Please note that
'orchestration' does NOT mean Clover does any orchestration ---
similar to how Clover doesn't by itself implement tracing or
logging, orchestration is a directory for code related to Docker
orchestration client --- such as k8s client

kube_client utilizes the Kubernetes python client (a dependency)
to perform tasks against Kubernetes API server. For this commit,
it is only tested for weighted route rule verification, it does
three tasks:
(1) get a list of pods under a namespace --- pod dictionary now
    only contains pod name and label dictionary: used to match
    pod name with the node name in traces from OpenTracing
(2) check to see if a particular pod is up in a particular
    namespace: used to check if Istio pods are running in
    istio-system namespace
(3) check if a container exists in a list of pods under a
    namespace: used to check if application pods have
    istio-proxy container running

route_rule directly invokes istioctl as there isn't any Istio
Python client yet.  Currently it reads and parses routerules
from Istio, and validates if a particular trace result matches
the routerules

Finally, a sample tool clover_validate_rr is provided. This
tool assumes a previous test has been ran (with an id with
both the route-rule-under-test and corresponding traces are
stored --- currently the assumption is tests were ran with
redis-master running on system). The tool can be invoked:

python clover_validate_rr.py -t <test-id> -s <service name>

where test-id is the ID of the test (most likely uuid) and
service name is the name of the service running in the
Kubernetes cluster upon which test traces should be fetched
against

Change-Id: Ic8ab6efc23c71ac4643bee796ef986a86f6fc7dd
Signed-off-by: Stephen Wong <stephen.kf.wong@gmail.com>

8 files changed, 578 insertions, 0 deletions

diff --git a/clover/orchestration/Pipfile b/clover/orchestration/Pipfile
new file mode 100644
index 0000000..12f776c
--- /dev/null
+++ b/clover/orchestration/Pipfile
@@ -0,0 +1,19 @@
+[[source]]
+
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+
+[dev-packages]
+
+
+
+[packages]
+
+kubernetes = "*"
+
+
+[requires]
+
+python_version = "2.7"
diff --git a/clover/orchestration/Pipfile.lock b/clover/orchestration/Pipfile.lock
new file mode 100644
index 0000000..4831a48
--- /dev/null
+++ b/clover/orchestration/Pipfile.lock
@@ -0,0 +1,178 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "97b3bd99fc2b3c80b1109170700a7d2a8e0b4330eaba769f7bc7aaa7f4a42925"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "2.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.python.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "cachetools": {
+            "hashes": [
+                "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe",
+                "sha256:ede01f2d3cbd6ddc9e35e16c2b0ce011d8bb70ce0dbaf282f5b4df24b213bc5d"
+            ],
+            "version": "==2.0.1"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296",
+                "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"
+            ],
+            "version": "==2018.1.18"
+        },
+        "chardet": {
+            "hashes": [
+                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+            ],
+            "version": "==3.0.4"
+        },
+        "google-auth": {
+            "hashes": [
+                "sha256:34088434cb2a2409360b8f3cbc04195a465df1fb2aafad71ebbded77cbf08803",
+                "sha256:9051802d3dae256036cca9e34633a32c0ed1427730d4ebc513dff91ec8b6dd45"
+            ],
+            "version": "==1.4.1"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f",
+                "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4"
+            ],
+            "version": "==2.6"
+        },
+        "ipaddress": {
+            "hashes": [
+                "sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"
+            ],
+            "version": "==1.0.19"
+        },
+        "kubernetes": {
+            "hashes": [
+                "sha256:2f1a05a9bb2549d6afb6d138b2767d61d8aeb735a7a12bf554440524205e2894",
+                "sha256:f81f145882471a1dd9d23360e99bd77027f07744729ef2728af4af7130cd19fd"
+            ],
+            "index": "pypi",
+            "version": "==5.0.0"
+        },
+        "oauthlib": {
+            "hashes": [
+                "sha256:ce57b501e906ff4f614e71c36a3ab9eacbb96d35c24d1970d2539bbc3ec70ce1"
+            ],
+            "version": "==2.0.6"
+        },
+        "pyasn1": {
+            "hashes": [
+                "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89",
+                "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee",
+                "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5",
+                "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43",
+                "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b",
+                "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b",
+                "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626",
+                "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15",
+                "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1",
+                "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2",
+                "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc",
+                "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b"
+            ],
+            "version": "==0.4.2"
+        },
+        "pyasn1-modules": {
+            "hashes": [
+                "sha256:041e9fbafac548d095f5b6c3b328b80792f006196e15a232b731a83c93d59493",
+                "sha256:0cdca76a68dcb701fff58c397de0ef9922b472b1cb3ea9695ca19d03f1869787",
+                "sha256:0cea139045c38f84abaa803bcb4b5e8775ea12a42af10019d942f227acc426c3",
+                "sha256:0f2e50d20bc670be170966638fa0ae603f0bc9ed6ebe8e97a6d1d4cef30cc889",
+                "sha256:47fb6757ab78fe966e7c58b2030b546854f78416d653163f0ce9290cf2278e8b",
+                "sha256:598a6004ec26a8ab40a39ea955068cf2a3949ad9c0030da970f2e1ca4c9f1cc9",
+                "sha256:72fd8b0c11191da088147c6e4678ec53e573923ecf60b57eeac9e97433e09fc2",
+                "sha256:854700bbdd01394e2ada9c1bfbd0ed9f5d0c551350dbbd023e88b11d2771ae06",
+                "sha256:af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc",
+                "sha256:b1f395cae2d669e0830cb023aa86f9f283b7a9aa32317d7f80d8e78aa2745812",
+                "sha256:c6747146e95d2b14cc2a8399b2b0bde3f93778f8f9ec704690d2b589c376c137",
+                "sha256:f53fe5bcebdf318f51399b250fe8325ef3a26d927f012cc0c8e0f9e9af7f9deb"
+            ],
+            "version": "==0.2.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:07009062406cffd554a9b4135cd2ff167c9bf6b7aac61fe946c93e69fad1bbd8",
+                "sha256:8f95bb7e6edbb2456a51a1fb58c8dca942024b4f5844cae62c90aa88afe6e300"
+            ],
+            "version": "==2.7.0"
+        },
+        "pyyaml": {
+            "hashes": [
+                "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
+                "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736",
+                "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f",
+                "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608",
+                "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8",
+                "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab",
+                "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7",
+                "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3",
+                "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1",
+                "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6",
+                "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8",
+                "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4",
+                "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca",
+                "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269"
+            ],
+            "version": "==3.12"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
+                "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
+            ],
+            "version": "==2.18.4"
+        },
+        "requests-oauthlib": {
+            "hashes": [
+                "sha256:50a8ae2ce8273e384895972b56193c7409601a66d4975774c60c2aed869639ca",
+                "sha256:883ac416757eada6d3d07054ec7092ac21c7f35cb1d2cf82faf205637081f468"
+            ],
+            "version": "==0.8.0"
+        },
+        "rsa": {
+            "hashes": [
+                "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5",
+                "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd"
+            ],
+            "version": "==3.4.2"
+        },
+        "six": {
+            "hashes": [
+                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+            ],
+            "version": "==1.11.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
+                "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
+            ],
+            "version": "==1.22"
+        },
+        "websocket-client": {
+            "hashes": [
+                "sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6",
+                "sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149"
+            ],
+            "version": "==0.47.0"
+        }
+    },
+    "develop": {}
+}
diff --git a/clover/orchestration/__init__.py b/clover/orchestration/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/orchestration/__init__.py
diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py
new file mode 100644
index 0000000..e5f1d89
--- /dev/null
+++ b/clover/orchestration/kube_client.py
@@ -0,0 +1,103 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from os import path
+import yaml
+
+from kubernetes import client, config
+
+class KubeClient(object):
+
+    def __init__(self):
+        config.load_kube_config()
+        self.core_v1 = client.CoreV1Api()
+        self.extensions_v1beta1 = client.ExtensionsV1beta1Api()
+
+    def find_svc_by_namespace(self, svc_name, namespace='default'):
+        ret_dict = {}
+        try:
+            svc = self.core_v1.read_namespaced_service(name=svc_name,
+                                                       namespace=namespace)
+        except client.rest.ApiException:
+            svc = None
+        if not svc:
+            print('found no service %s in namespace %s' \
+                   % (svc_name, namespace))
+            return None
+        ret_dict[svc.metadata.name] = {}
+        ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels
+        ret_dict[svc.metadata.name]['selector'] = svc.spec.selector
+
+        return ret_dict
+
+    def find_pod_by_namespace(self, namespace='default'):
+        ret_dict = {}
+        pods = self.core_v1.list_namespaced_pod(namespace=namespace)
+        if not pods:
+            print('found no pod')
+            return None
+        for pod in pods.items:
+            if pod.metadata.name not in ret_dict:
+                ret_dict[pod.metadata.name] = {}
+            ret_dict[pod.metadata.name]['labels'] = pod.metadata.labels
+
+        return ret_dict
+
+    def _check_pod(self, pod_name, namespace='defualt', container_name=None):
+        ret = self.core_v1.list_namespaced_pod(namespace=namespace)
+        ret_code = False
+        new_pod_name = None
+        for i in ret.items:
+            if pod_name in i.metadata.name:
+                if i.status.container_statuses and len(i.status.container_statuses) > 0:
+                    container_up = False
+                    for container in i.status.container_statuses:
+                        check_state = True
+                        if container_name:
+                            if container_name != container.name:
+                                check_state = False
+                        if check_state and container.state.running is not None:
+                            container_up = True
+                        else:
+                            if container_up:
+                                container_up = False
+                                break
+                    if container_up:
+                        ret_code = True
+                        new_pod_name = i.metadata.name
+        return ret_code, new_pod_name
+
+    def check_pod_up(self, pod_name, namespace='default'):
+        return self._check_pod(pod_name, namespace)
+
+    def check_container_in_pods(self, container_name, pods, namespace='default'):
+        ret = False
+        for pod in pods:
+            ret, _ = self._check_pod(pod, namespace, container_name)
+            if not ret:
+                return ret
+        return ret
+
+    def create_deployment_yaml(self, deployment_yaml_path, namespace='default'):
+        with open(deployment_yaml_path) as fp:
+            body = yaml.load(fp)
+            resp = self.extensions_v1beta1.create_namespaced_deployment(
+                    body=body, namespace=namespace)
+            print('Deployment created. Status=%s' % str(resp.status))
+
+            dep_name = body.get('metadata').get('name')
+            return dep_name
+
+    def create_service_yaml(self, service_yaml_path, namespace='default'):
+        with open(service_yaml_path) as fp:
+            body = yaml.load(fp)
+            resp = self.extensions_v1beta1.create_namespaced_service(
+                    body=body, namespace=namespace)
+            print('Service created. Status=%s' % str(resp.status))
+
+            svc_name = body.get('metadata').get('name')
+            return svc_name
diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py
new file mode 100644
index 0000000..cc2ee0c
--- /dev/null
+++ b/clover/servicemesh/route_rules.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+import os
+import redis
+import subprocess
+import sys
+import yaml
+
+#istioctl='$HOME/istio-0.6.0/bin/istioctl'
+# The assumption is that istioctl is already in the user's path
+ISTIOCTL='istioctl'
+
+def cmd_exists(cmd):
+    return any(
+        os.access(os.path.join(path, cmd), os.X_OK)
+        for path in os.environ["PATH"].split(os.pathsep)
+    )
+
+def load_route_rules(rr_yaml_path):
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return False
+
+    # TODO(s3wong): load yaml and verify it does indeed contain route rule
+    cmd = ISTIOCTL + ' create -f ' + rr_yaml_path
+    output = subprocess.check_output(cmd, shell=True)
+    if not output:
+        print('Route rule creation failed: %s' % output)
+        return False
+    return True
+
+def delete_route_rules(rr_yaml_path, namespace):
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return False
+
+    # TODO(s3wong): load yaml and verify it does indeed contain route rule
+    cmd = ISTIOCTL + ' delete -f ' + rr_yaml_path + ' -n ' + namespace
+    output = subprocess.check_output(cmd, shell=True)
+    if not output or not 'Deleted' in output:
+        print('Route rule deletion failed: %s' % output)
+        return False
+    return True
+
+def get_route_rules():
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return None
+    cmd = ISTIOCTL + ' get routerules -o yaml'
+    output = subprocess.check_output(cmd, shell=True)
+    if not output:
+        print('No route rule configured')
+        return None
+    docs = []
+    for raw_doc in output.split('\n---'):
+        try:
+            docs.append(yaml.load(raw_doc))
+        except SyntaxError:
+            print('syntax error: %s' % raw_doc)
+    return docs
+
+def parse_route_rules(routerules):
+    ret_list = []
+    if not routerules:
+        print('No routerules')
+        return ret_list
+    for routerule in routerules:
+        if not routerule or routerule == 'None':  continue
+        print('routerule is %s' % routerule)
+        if routerule.get('kind') != 'RouteRule': continue
+        ret_rr_dict = {}
+        spec = routerule.get('spec')
+        if not spec: continue
+        ret_rr_dict['service'] = spec.get('destination').get('name')
+        ret_rr_dict['rules'] = spec.get('route')
+        ret_list.append(ret_rr_dict)
+    return ret_list
+
+def _derive_key_from_test_id(test_id):
+    return 'route-rules-' + str(test_id)
+
+def set_route_rules(test_id):
+    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    key = _derive_key_from_test_id(test_id)
+    rr = get_route_rules()
+    r.set(key, rr)
+
+def fetch_route_rules(test_id):
+    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    key = _derive_key_from_test_id(test_id)
+    rr = r.get(key)
+    return yaml.load(rr)
+
+'''
+    The format of result_dict is expected to be:
+    {
+        'service':  <service name>,
+        <version string 1>: <integer representation of version string 1 occurrances during test>,
+        <version string 2>: <integer representation of version string 2 occurrances during test>,
+        ...
+    }
+'''
+def validate_weighted_route_rules(result_dict, test_id=None):
+    print('validate_weighted_route_rules: test id %s' % test_id)
+    svc_name = result_dict.get('service')
+    if not test_id:
+        rr_list = parse_route_rules(get_route_rules())
+    else:
+        rr_list = parse_route_rules(fetch_route_rules(test_id))
+    errors = []
+    ret = True
+    for rr in rr_list:
+        route_rules = rr.get('rules')
+        if not route_rules:
+            break
+        for rule in route_rules:
+            version = rule.get('labels').get('version')
+            weight = rule.get('weight')
+            if not weight: weight = 1
+            if abs(weight - result_dict[version]) > 10:
+                err = 'svc %s version %s expected to get %d, but got %d' % (svc_name, version, weight, result_dict[version])
+                ret = False
+            else:
+                err = 'svc %s version %s expected to get %d, got %d. Validation succeeded' % (svc_name, version, weight, result_dict[version])
+            errors.append(err)
+    return ret, errors
+
+
diff --git a/clover/tools/__init__.py b/clover/tools/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/tools/__init__.py
diff --git a/clover/tools/clover_validate_rr.py b/clover/tools/clover_validate_rr.py
new file mode 100644
index 0000000..ff1f8b4
--- /dev/null
+++ b/clover/tools/clover_validate_rr.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import getopt
+import sys
+
+sys.path.insert(0, '..')
+
+from orchestration import kube_client
+import servicemesh.route_rules as rr
+from tracing.tracing import Tracing
+from validate_rr import ValidateWRR
+
+def main(argv):
+    service_name = None
+    test_id = None
+    help_str = 'clover_validate_rr.py -t <test-id> -s <service name>'
+    try:
+        opts, args = getopt.getopt(argv,"hs:t:",["service-name", "test-id"])
+    except getopt.GetoptError:
+        print help_str
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            print help_str
+            sys.exit()
+        elif opt in ("-t", "--test-id"):
+            test_id = str(arg)
+        elif opt in ("-s", "--service-name"):
+            service_name = str(arg)
+
+    if not service_name or not test_id:
+        print help_str
+        sys.exit(3)
+
+    validate_wrr = ValidateWRR(test_id)
+
+    istio_pods = ['istio-ingress', 'istio-mixer', 'istio-pilot']
+    jaeger_pods = ['jaeger-deployment']
+
+    if not validate_wrr.check_pods_up(istio_pods + jaeger_pods,
+                                      'istio-system'):
+        sys.exit(4)
+
+    ret, errors = validate_wrr.validate(service_name)
+    for err in errors:
+        print err
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/clover/tools/validate_rr.py b/clover/tools/validate_rr.py
new file mode 100644
index 0000000..0e7b9ed
--- /dev/null
+++ b/clover/tools/validate_rr.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+import sys
+
+sys.path.insert(0, '..')
+
+from orchestration import kube_client
+import servicemesh.route_rules as rr
+from tracing.tracing import Tracing
+
+class ValidateWRR(object):
+
+    def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298'):
+        self._k8s_client = kube_client.KubeClient()
+        self._test_id = test_id
+        self._tracing = Tracing(tracing_ip, tracing_port)
+
+    def check_pods_up(self, pod_list, namespace='default'):
+        for pod in pod_list:
+            up, name = self._k8s_client.check_pod_up(pod, namespace)
+            if not up:
+                print('pod %s in namespace %s not up' % (pod, namespace))
+                return False
+        return True
+
+    def set_test_id(self, test_id):
+        self._test_id = test_id
+
+    def validate(self, service_name):
+        total = 0
+        svc = self._k8s_client.find_svc_by_namespace(svc_name=service_name)
+        if not svc:
+            err_msg = 'Failed to locate service %s in default namespace' % service_name
+            print err_msg
+            return False, [err_msg]
+        pods = self._k8s_client.find_pod_by_namespace()
+        if not pods:
+            err_msg = 'No pod found in default namespace'
+            return False, [err_msg]
+        svc_pods = {}
+        for p,l in pods.items():
+            pod_labels = l.get('labels')
+            svc_selector_dict = svc[service_name].get('selector')
+            for svc_select_key in svc_selector_dict:
+                if svc_select_key in pod_labels:
+                    if svc_selector_dict[svc_select_key] == pod_labels[svc_select_key]:
+                        svc_pods[p] = l
+
+        trace_ids = self._tracing.getRedisTraceids(self._test_id)
+        rr_dict = {'service': service_name}
+        ver_count_dict = {}
+        for trace_id in trace_ids:
+            span_ids = self._tracing.getRedisSpanids(trace_id)
+            for span in span_ids:
+                # count only the received side --- i.e., messages sent TO
+                # service
+                node_id = self._tracing.getRedisTagsValue(span, trace_id, 'node_id')
+                direction = self._tracing.getRedisTagsValue(span, trace_id, 'upstream_cluster')
+                if direction.startswith('in.'):
+                    for pod_name in svc_pods:
+                        if pod_name in node_id:
+                            total += 1
+                            labels = svc_pods[pod_name]['labels']
+                            print('node %s pod %s labels %s' % (node_id, pod_name, labels))
+                            if 'version' in labels:
+                                version = labels.get('version')
+                                if version in ver_count_dict:
+                                    ver_count_dict[version] += 1
+                                else:
+                                    ver_count_dict[version] = 1
+
+        print('total is %d, ver_count_dict is %s' % (total, ver_count_dict))
+        if ver_count_dict and total > 0:
+            for version in ver_count_dict:
+                rr_dict[version] = float(ver_count_dict[version]) / float(total) * 100
+
+            return(rr.validate_weighted_route_rules(rr_dict, self._test_id))
+        else:
+            err_msg = 'No version label found on any pod'
+            print(err_msg)
+            return False, [err_msg]
+