summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--clover/orchestration/Pipfile19
-rw-r--r--clover/orchestration/Pipfile.lock178
-rw-r--r--clover/orchestration/__init__.py0
-rw-r--r--clover/orchestration/kube_client.py103
-rw-r--r--clover/servicemesh/route_rules.py134
-rw-r--r--clover/tools/__init__.py0
-rw-r--r--clover/tools/clover_validate_rr.py56
-rw-r--r--clover/tools/validate_rr.py88
8 files changed, 578 insertions, 0 deletions
diff --git a/clover/orchestration/Pipfile b/clover/orchestration/Pipfile
new file mode 100644
index 0000000..12f776c
--- /dev/null
+++ b/clover/orchestration/Pipfile
@@ -0,0 +1,19 @@
+[[source]]
+
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+
+[dev-packages]
+
+
+
+[packages]
+
+kubernetes = "*"
+
+
+[requires]
+
+python_version = "2.7"
diff --git a/clover/orchestration/Pipfile.lock b/clover/orchestration/Pipfile.lock
new file mode 100644
index 0000000..4831a48
--- /dev/null
+++ b/clover/orchestration/Pipfile.lock
@@ -0,0 +1,178 @@
+{
+ "_meta": {
+ "hash": {
+ "sha256": "97b3bd99fc2b3c80b1109170700a7d2a8e0b4330eaba769f7bc7aaa7f4a42925"
+ },
+ "pipfile-spec": 6,
+ "requires": {
+ "python_version": "2.7"
+ },
+ "sources": [
+ {
+ "name": "pypi",
+ "url": "https://pypi.python.org/simple",
+ "verify_ssl": true
+ }
+ ]
+ },
+ "default": {
+ "cachetools": {
+ "hashes": [
+ "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe",
+ "sha256:ede01f2d3cbd6ddc9e35e16c2b0ce011d8bb70ce0dbaf282f5b4df24b213bc5d"
+ ],
+ "version": "==2.0.1"
+ },
+ "certifi": {
+ "hashes": [
+ "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296",
+ "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d"
+ ],
+ "version": "==2018.1.18"
+ },
+ "chardet": {
+ "hashes": [
+ "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
+ "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
+ ],
+ "version": "==3.0.4"
+ },
+ "google-auth": {
+ "hashes": [
+ "sha256:34088434cb2a2409360b8f3cbc04195a465df1fb2aafad71ebbded77cbf08803",
+ "sha256:9051802d3dae256036cca9e34633a32c0ed1427730d4ebc513dff91ec8b6dd45"
+ ],
+ "version": "==1.4.1"
+ },
+ "idna": {
+ "hashes": [
+ "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f",
+ "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4"
+ ],
+ "version": "==2.6"
+ },
+ "ipaddress": {
+ "hashes": [
+ "sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81"
+ ],
+ "version": "==1.0.19"
+ },
+ "kubernetes": {
+ "hashes": [
+ "sha256:2f1a05a9bb2549d6afb6d138b2767d61d8aeb735a7a12bf554440524205e2894",
+ "sha256:f81f145882471a1dd9d23360e99bd77027f07744729ef2728af4af7130cd19fd"
+ ],
+ "index": "pypi",
+ "version": "==5.0.0"
+ },
+ "oauthlib": {
+ "hashes": [
+ "sha256:ce57b501e906ff4f614e71c36a3ab9eacbb96d35c24d1970d2539bbc3ec70ce1"
+ ],
+ "version": "==2.0.6"
+ },
+ "pyasn1": {
+ "hashes": [
+ "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89",
+ "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee",
+ "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5",
+ "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43",
+ "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b",
+ "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b",
+ "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626",
+ "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15",
+ "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1",
+ "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2",
+ "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc",
+ "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b"
+ ],
+ "version": "==0.4.2"
+ },
+ "pyasn1-modules": {
+ "hashes": [
+ "sha256:041e9fbafac548d095f5b6c3b328b80792f006196e15a232b731a83c93d59493",
+ "sha256:0cdca76a68dcb701fff58c397de0ef9922b472b1cb3ea9695ca19d03f1869787",
+ "sha256:0cea139045c38f84abaa803bcb4b5e8775ea12a42af10019d942f227acc426c3",
+ "sha256:0f2e50d20bc670be170966638fa0ae603f0bc9ed6ebe8e97a6d1d4cef30cc889",
+ "sha256:47fb6757ab78fe966e7c58b2030b546854f78416d653163f0ce9290cf2278e8b",
+ "sha256:598a6004ec26a8ab40a39ea955068cf2a3949ad9c0030da970f2e1ca4c9f1cc9",
+ "sha256:72fd8b0c11191da088147c6e4678ec53e573923ecf60b57eeac9e97433e09fc2",
+ "sha256:854700bbdd01394e2ada9c1bfbd0ed9f5d0c551350dbbd023e88b11d2771ae06",
+ "sha256:af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc",
+ "sha256:b1f395cae2d669e0830cb023aa86f9f283b7a9aa32317d7f80d8e78aa2745812",
+ "sha256:c6747146e95d2b14cc2a8399b2b0bde3f93778f8f9ec704690d2b589c376c137",
+ "sha256:f53fe5bcebdf318f51399b250fe8325ef3a26d927f012cc0c8e0f9e9af7f9deb"
+ ],
+ "version": "==0.2.1"
+ },
+ "python-dateutil": {
+ "hashes": [
+ "sha256:07009062406cffd554a9b4135cd2ff167c9bf6b7aac61fe946c93e69fad1bbd8",
+ "sha256:8f95bb7e6edbb2456a51a1fb58c8dca942024b4f5844cae62c90aa88afe6e300"
+ ],
+ "version": "==2.7.0"
+ },
+ "pyyaml": {
+ "hashes": [
+ "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8",
+ "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736",
+ "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f",
+ "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608",
+ "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8",
+ "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab",
+ "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7",
+ "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3",
+ "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1",
+ "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6",
+ "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8",
+ "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4",
+ "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca",
+ "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269"
+ ],
+ "version": "==3.12"
+ },
+ "requests": {
+ "hashes": [
+ "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b",
+ "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e"
+ ],
+ "version": "==2.18.4"
+ },
+ "requests-oauthlib": {
+ "hashes": [
+ "sha256:50a8ae2ce8273e384895972b56193c7409601a66d4975774c60c2aed869639ca",
+ "sha256:883ac416757eada6d3d07054ec7092ac21c7f35cb1d2cf82faf205637081f468"
+ ],
+ "version": "==0.8.0"
+ },
+ "rsa": {
+ "hashes": [
+ "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5",
+ "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd"
+ ],
+ "version": "==3.4.2"
+ },
+ "six": {
+ "hashes": [
+ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
+ "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+ ],
+ "version": "==1.11.0"
+ },
+ "urllib3": {
+ "hashes": [
+ "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b",
+ "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f"
+ ],
+ "version": "==1.22"
+ },
+ "websocket-client": {
+ "hashes": [
+ "sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6",
+ "sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149"
+ ],
+ "version": "==0.47.0"
+ }
+ },
+ "develop": {}
+}
diff --git a/clover/orchestration/__init__.py b/clover/orchestration/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/orchestration/__init__.py
diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py
new file mode 100644
index 0000000..e5f1d89
--- /dev/null
+++ b/clover/orchestration/kube_client.py
@@ -0,0 +1,103 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from os import path
+import yaml
+
+from kubernetes import client, config
+
+class KubeClient(object):
+
+ def __init__(self):
+ config.load_kube_config()
+ self.core_v1 = client.CoreV1Api()
+ self.extensions_v1beta1 = client.ExtensionsV1beta1Api()
+
+ def find_svc_by_namespace(self, svc_name, namespace='default'):
+ ret_dict = {}
+ try:
+ svc = self.core_v1.read_namespaced_service(name=svc_name,
+ namespace=namespace)
+ except client.rest.ApiException:
+ svc = None
+ if not svc:
+ print('found no service %s in namespace %s' \
+ % (svc_name, namespace))
+ return None
+ ret_dict[svc.metadata.name] = {}
+ ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels
+ ret_dict[svc.metadata.name]['selector'] = svc.spec.selector
+
+ return ret_dict
+
+ def find_pod_by_namespace(self, namespace='default'):
+ ret_dict = {}
+ pods = self.core_v1.list_namespaced_pod(namespace=namespace)
+ if not pods:
+ print('found no pod')
+ return None
+ for pod in pods.items:
+ if pod.metadata.name not in ret_dict:
+ ret_dict[pod.metadata.name] = {}
+ ret_dict[pod.metadata.name]['labels'] = pod.metadata.labels
+
+ return ret_dict
+
+ def _check_pod(self, pod_name, namespace='defualt', container_name=None):
+ ret = self.core_v1.list_namespaced_pod(namespace=namespace)
+ ret_code = False
+ new_pod_name = None
+ for i in ret.items:
+ if pod_name in i.metadata.name:
+ if i.status.container_statuses and len(i.status.container_statuses) > 0:
+ container_up = False
+ for container in i.status.container_statuses:
+ check_state = True
+ if container_name:
+ if container_name != container.name:
+ check_state = False
+ if check_state and container.state.running is not None:
+ container_up = True
+ else:
+ if container_up:
+ container_up = False
+ break
+ if container_up:
+ ret_code = True
+ new_pod_name = i.metadata.name
+ return ret_code, new_pod_name
+
+ def check_pod_up(self, pod_name, namespace='default'):
+ return self._check_pod(pod_name, namespace)
+
+ def check_container_in_pods(self, container_name, pods, namespace='default'):
+ ret = False
+ for pod in pods:
+ ret, _ = self._check_pod(pod, namespace, container_name)
+ if not ret:
+ return ret
+ return ret
+
+ def create_deployment_yaml(self, deployment_yaml_path, namespace='default'):
+ with open(deployment_yaml_path) as fp:
+ body = yaml.load(fp)
+ resp = self.extensions_v1beta1.create_namespaced_deployment(
+ body=body, namespace=namespace)
+ print('Deployment created. Status=%s' % str(resp.status))
+
+ dep_name = body.get('metadata').get('name')
+ return dep_name
+
+ def create_service_yaml(self, service_yaml_path, namespace='default'):
+ with open(service_yaml_path) as fp:
+ body = yaml.load(fp)
+ resp = self.extensions_v1beta1.create_namespaced_service(
+ body=body, namespace=namespace)
+ print('Service created. Status=%s' % str(resp.status))
+
+ svc_name = body.get('metadata').get('name')
+ return svc_name
diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py
new file mode 100644
index 0000000..cc2ee0c
--- /dev/null
+++ b/clover/servicemesh/route_rules.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+import os
+import redis
+import subprocess
+import sys
+import yaml
+
+#istioctl='$HOME/istio-0.6.0/bin/istioctl'
+# The assumption is that istioctl is already in the user's path
+ISTIOCTL='istioctl'
+
+def cmd_exists(cmd):
+ return any(
+ os.access(os.path.join(path, cmd), os.X_OK)
+ for path in os.environ["PATH"].split(os.pathsep)
+ )
+
+def load_route_rules(rr_yaml_path):
+ if not cmd_exists(ISTIOCTL):
+ print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+ return False
+
+ # TODO(s3wong): load yaml and verify it does indeed contain route rule
+ cmd = ISTIOCTL + ' create -f ' + rr_yaml_path
+ output = subprocess.check_output(cmd, shell=True)
+ if not output:
+ print('Route rule creation failed: %s' % output)
+ return False
+ return True
+
+def delete_route_rules(rr_yaml_path, namespace):
+ if not cmd_exists(ISTIOCTL):
+ print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+ return False
+
+ # TODO(s3wong): load yaml and verify it does indeed contain route rule
+ cmd = ISTIOCTL + ' delete -f ' + rr_yaml_path + ' -n ' + namespace
+ output = subprocess.check_output(cmd, shell=True)
+ if not output or not 'Deleted' in output:
+ print('Route rule deletion failed: %s' % output)
+ return False
+ return True
+
+def get_route_rules():
+ if not cmd_exists(ISTIOCTL):
+ print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+ return None
+ cmd = ISTIOCTL + ' get routerules -o yaml'
+ output = subprocess.check_output(cmd, shell=True)
+ if not output:
+ print('No route rule configured')
+ return None
+ docs = []
+ for raw_doc in output.split('\n---'):
+ try:
+ docs.append(yaml.load(raw_doc))
+ except SyntaxError:
+ print('syntax error: %s' % raw_doc)
+ return docs
+
+def parse_route_rules(routerules):
+ ret_list = []
+ if not routerules:
+ print('No routerules')
+ return ret_list
+ for routerule in routerules:
+ if not routerule or routerule == 'None': continue
+ print('routerule is %s' % routerule)
+ if routerule.get('kind') != 'RouteRule': continue
+ ret_rr_dict = {}
+ spec = routerule.get('spec')
+ if not spec: continue
+ ret_rr_dict['service'] = spec.get('destination').get('name')
+ ret_rr_dict['rules'] = spec.get('route')
+ ret_list.append(ret_rr_dict)
+ return ret_list
+
+def _derive_key_from_test_id(test_id):
+ return 'route-rules-' + str(test_id)
+
+def set_route_rules(test_id):
+ r = redis.StrictRedis(host='localhost', port=6379, db=0)
+ key = _derive_key_from_test_id(test_id)
+ rr = get_route_rules()
+ r.set(key, rr)
+
+def fetch_route_rules(test_id):
+ r = redis.StrictRedis(host='localhost', port=6379, db=0)
+ key = _derive_key_from_test_id(test_id)
+ rr = r.get(key)
+ return yaml.load(rr)
+
+'''
+ The format of result_dict is expected to be:
+ {
+ 'service': <service name>,
+ <version string 1>: <integer representation of version string 1 occurrances during test>,
+ <version string 2>: <integer representation of version string 2 occurrances during test>,
+ ...
+ }
+'''
+def validate_weighted_route_rules(result_dict, test_id=None):
+ print('validate_weighted_route_rules: test id %s' % test_id)
+ svc_name = result_dict.get('service')
+ if not test_id:
+ rr_list = parse_route_rules(get_route_rules())
+ else:
+ rr_list = parse_route_rules(fetch_route_rules(test_id))
+ errors = []
+ ret = True
+ for rr in rr_list:
+ route_rules = rr.get('rules')
+ if not route_rules:
+ break
+ for rule in route_rules:
+ version = rule.get('labels').get('version')
+ weight = rule.get('weight')
+ if not weight: weight = 1
+ if abs(weight - result_dict[version]) > 10:
+ err = 'svc %s version %s expected to get %d, but got %d' % (svc_name, version, weight, result_dict[version])
+ ret = False
+ else:
+ err = 'svc %s version %s expected to get %d, got %d. Validation succeeded' % (svc_name, version, weight, result_dict[version])
+ errors.append(err)
+ return ret, errors
+
+
diff --git a/clover/tools/__init__.py b/clover/tools/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/tools/__init__.py
diff --git a/clover/tools/clover_validate_rr.py b/clover/tools/clover_validate_rr.py
new file mode 100644
index 0000000..ff1f8b4
--- /dev/null
+++ b/clover/tools/clover_validate_rr.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import getopt
+import sys
+
+sys.path.insert(0, '..')
+
+from orchestration import kube_client
+import servicemesh.route_rules as rr
+from tracing.tracing import Tracing
+from validate_rr import ValidateWRR
+
+def main(argv):
+ service_name = None
+ test_id = None
+ help_str = 'clover_validate_rr.py -t <test-id> -s <service name>'
+ try:
+ opts, args = getopt.getopt(argv,"hs:t:",["service-name", "test-id"])
+ except getopt.GetoptError:
+ print help_str
+ sys.exit(2)
+ for opt, arg in opts:
+ if opt == '-h':
+ print help_str
+ sys.exit()
+ elif opt in ("-t", "--test-id"):
+ test_id = str(arg)
+ elif opt in ("-s", "--service-name"):
+ service_name = str(arg)
+
+ if not service_name or not test_id:
+ print help_str
+ sys.exit(3)
+
+ validate_wrr = ValidateWRR(test_id)
+
+ istio_pods = ['istio-ingress', 'istio-mixer', 'istio-pilot']
+ jaeger_pods = ['jaeger-deployment']
+
+ if not validate_wrr.check_pods_up(istio_pods + jaeger_pods,
+ 'istio-system'):
+ sys.exit(4)
+
+ ret, errors = validate_wrr.validate(service_name)
+ for err in errors:
+ print err
+
+if __name__ == "__main__":
+ main(sys.argv[1:])
diff --git a/clover/tools/validate_rr.py b/clover/tools/validate_rr.py
new file mode 100644
index 0000000..0e7b9ed
--- /dev/null
+++ b/clover/tools/validate_rr.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+import sys
+
+sys.path.insert(0, '..')
+
+from orchestration import kube_client
+import servicemesh.route_rules as rr
+from tracing.tracing import Tracing
+
+class ValidateWRR(object):
+
+ def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298'):
+ self._k8s_client = kube_client.KubeClient()
+ self._test_id = test_id
+ self._tracing = Tracing(tracing_ip, tracing_port)
+
+ def check_pods_up(self, pod_list, namespace='default'):
+ for pod in pod_list:
+ up, name = self._k8s_client.check_pod_up(pod, namespace)
+ if not up:
+ print('pod %s in namespace %s not up' % (pod, namespace))
+ return False
+ return True
+
+ def set_test_id(self, test_id):
+ self._test_id = test_id
+
+ def validate(self, service_name):
+ total = 0
+ svc = self._k8s_client.find_svc_by_namespace(svc_name=service_name)
+ if not svc:
+ err_msg = 'Failed to locate service %s in default namespace' % service_name
+ print err_msg
+ return False, [err_msg]
+ pods = self._k8s_client.find_pod_by_namespace()
+ if not pods:
+ err_msg = 'No pod found in default namespace'
+ return False, [err_msg]
+ svc_pods = {}
+ for p,l in pods.items():
+ pod_labels = l.get('labels')
+ svc_selector_dict = svc[service_name].get('selector')
+ for svc_select_key in svc_selector_dict:
+ if svc_select_key in pod_labels:
+ if svc_selector_dict[svc_select_key] == pod_labels[svc_select_key]:
+ svc_pods[p] = l
+
+ trace_ids = self._tracing.getRedisTraceids(self._test_id)
+ rr_dict = {'service': service_name}
+ ver_count_dict = {}
+ for trace_id in trace_ids:
+ span_ids = self._tracing.getRedisSpanids(trace_id)
+ for span in span_ids:
+ # count only the received side --- i.e., messages sent TO
+ # service
+ node_id = self._tracing.getRedisTagsValue(span, trace_id, 'node_id')
+ direction = self._tracing.getRedisTagsValue(span, trace_id, 'upstream_cluster')
+ if direction.startswith('in.'):
+ for pod_name in svc_pods:
+ if pod_name in node_id:
+ total += 1
+ labels = svc_pods[pod_name]['labels']
+ print('node %s pod %s labels %s' % (node_id, pod_name, labels))
+ if 'version' in labels:
+ version = labels.get('version')
+ if version in ver_count_dict:
+ ver_count_dict[version] += 1
+ else:
+ ver_count_dict[version] = 1
+
+ print('total is %d, ver_count_dict is %s' % (total, ver_count_dict))
+ if ver_count_dict and total > 0:
+ for version in ver_count_dict:
+ rr_dict[version] = float(ver_count_dict[version]) / float(total) * 100
+
+ return(rr.validate_weighted_route_rules(rr_dict, self._test_id))
+ else:
+ err_msg = 'No version label found on any pod'
+ print(err_msg)
+ return False, [err_msg]
+