diff options
| -rw-r--r-- | clover/orchestration/Pipfile | 19 | ||||
| -rw-r--r-- | clover/orchestration/Pipfile.lock | 178 | ||||
| -rw-r--r-- | clover/orchestration/__init__.py | 0 | ||||
| -rw-r--r-- | clover/orchestration/kube_client.py | 103 | ||||
| -rw-r--r-- | clover/servicemesh/route_rules.py | 134 | ||||
| -rw-r--r-- | clover/tools/__init__.py | 0 | ||||
| -rw-r--r-- | clover/tools/clover_validate_rr.py | 56 | ||||
| -rw-r--r-- | clover/tools/validate_rr.py | 88 |
8 files changed, 578 insertions, 0 deletions
diff --git a/clover/orchestration/Pipfile b/clover/orchestration/Pipfile new file mode 100644 index 0000000..12f776c --- /dev/null +++ b/clover/orchestration/Pipfile @@ -0,0 +1,19 @@ +[[source]] + +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + + +[dev-packages] + + + +[packages] + +kubernetes = "*" + + +[requires] + +python_version = "2.7" diff --git a/clover/orchestration/Pipfile.lock b/clover/orchestration/Pipfile.lock new file mode 100644 index 0000000..4831a48 --- /dev/null +++ b/clover/orchestration/Pipfile.lock @@ -0,0 +1,178 @@ +{ + "_meta": { + "hash": { + "sha256": "97b3bd99fc2b3c80b1109170700a7d2a8e0b4330eaba769f7bc7aaa7f4a42925" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "2.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "cachetools": { + "hashes": [ + "sha256:4319bbb78172e7bcf99423e1ecd6914b32336ccfe97d2058ffe62e641a7f3abe", + "sha256:ede01f2d3cbd6ddc9e35e16c2b0ce011d8bb70ce0dbaf282f5b4df24b213bc5d" + ], + "version": "==2.0.1" + }, + "certifi": { + "hashes": [ + "sha256:14131608ad2fd56836d33a71ee60fa1c82bc9d2c8d98b7bdbc631fe1b3cd1296", + "sha256:edbc3f203427eef571f79a7692bb160a2b0f7ccaa31953e99bd17e307cf63f7d" + ], + "version": "==2018.1.18" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "google-auth": { + "hashes": [ + "sha256:34088434cb2a2409360b8f3cbc04195a465df1fb2aafad71ebbded77cbf08803", + "sha256:9051802d3dae256036cca9e34633a32c0ed1427730d4ebc513dff91ec8b6dd45" + ], + "version": "==1.4.1" + }, + "idna": { + "hashes": [ + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f", + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4" + ], + "version": "==2.6" + }, + "ipaddress": { + "hashes": [ + "sha256:200d8686011d470b5e4de207d803445deee427455cd0cb7c982b68cf82524f81" + ], + "version": "==1.0.19" + }, + "kubernetes": { + "hashes": [ + "sha256:2f1a05a9bb2549d6afb6d138b2767d61d8aeb735a7a12bf554440524205e2894", + "sha256:f81f145882471a1dd9d23360e99bd77027f07744729ef2728af4af7130cd19fd" + ], + "index": "pypi", + "version": "==5.0.0" + }, + "oauthlib": { + "hashes": [ + "sha256:ce57b501e906ff4f614e71c36a3ab9eacbb96d35c24d1970d2539bbc3ec70ce1" + ], + "version": "==2.0.6" + }, + "pyasn1": { + "hashes": [ + "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89", + "sha256:5a0db897b311d265cde49615cf783f1c78613138605cdd0f907ecfa5b2aba3ee", + "sha256:758cb50abddc03e4563fd9e7f03db56e3e87b58c0bd01247360326e5c0c7ffa5", + "sha256:7d626683e3d792cccc608da02498aff37ab4f3dafd8905d6bf755d11f9b26b43", + "sha256:a7efe807c4b83a859e2735c692b92ed7b567cfddc4163763412920041d876c2b", + "sha256:b5a9ca48055b9a20f6d1b3d68e38692e5431c86a0f99ea602e61294e891fee5b", + "sha256:c07d6e587b2f928366b1f67c09bda026a3e6fcc99e80a744dc67f8fca3895626", + "sha256:d258b0a71994f7770599835249cece1caef3c70def868c4915e6e5ca49b67d15", + "sha256:d5cd6ed995dba16fad0c521cfe31cd2d68400b53fcc2bce93326829be73ab6d1", + "sha256:d84c2aea3cf43780e9e6a19f4e4dddee9f6976519020e64e47c57e5c7a8c3dd2", + "sha256:e85895087905c65b5b594eb91f7522664c85545b147d5f4d4e7b1b07da8dcbdc", + "sha256:f81c96761fca60d64b1c9b79ec2e40cf9495a745cf570613079ef324aeb9672b" + ], + "version": "==0.4.2" + }, + "pyasn1-modules": { + "hashes": [ + "sha256:041e9fbafac548d095f5b6c3b328b80792f006196e15a232b731a83c93d59493", + "sha256:0cdca76a68dcb701fff58c397de0ef9922b472b1cb3ea9695ca19d03f1869787", + "sha256:0cea139045c38f84abaa803bcb4b5e8775ea12a42af10019d942f227acc426c3", + "sha256:0f2e50d20bc670be170966638fa0ae603f0bc9ed6ebe8e97a6d1d4cef30cc889", + "sha256:47fb6757ab78fe966e7c58b2030b546854f78416d653163f0ce9290cf2278e8b", + "sha256:598a6004ec26a8ab40a39ea955068cf2a3949ad9c0030da970f2e1ca4c9f1cc9", + "sha256:72fd8b0c11191da088147c6e4678ec53e573923ecf60b57eeac9e97433e09fc2", + "sha256:854700bbdd01394e2ada9c1bfbd0ed9f5d0c551350dbbd023e88b11d2771ae06", + "sha256:af00ea8f2022b6287dc375b2c70f31ab5af83989fc6fe9eacd4976ce26cd7ccc", + "sha256:b1f395cae2d669e0830cb023aa86f9f283b7a9aa32317d7f80d8e78aa2745812", + "sha256:c6747146e95d2b14cc2a8399b2b0bde3f93778f8f9ec704690d2b589c376c137", + "sha256:f53fe5bcebdf318f51399b250fe8325ef3a26d927f012cc0c8e0f9e9af7f9deb" + ], + "version": "==0.2.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:07009062406cffd554a9b4135cd2ff167c9bf6b7aac61fe946c93e69fad1bbd8", + "sha256:8f95bb7e6edbb2456a51a1fb58c8dca942024b4f5844cae62c90aa88afe6e300" + ], + "version": "==2.7.0" + }, + "pyyaml": { + "hashes": [ + "sha256:0c507b7f74b3d2dd4d1322ec8a94794927305ab4cebbe89cc47fe5e81541e6e8", + "sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736", + "sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f", + "sha256:326420cbb492172dec84b0f65c80942de6cedb5233c413dd824483989c000608", + "sha256:4474f8ea030b5127225b8894d626bb66c01cda098d47a2b0d3429b6700af9fd8", + "sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab", + "sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7", + "sha256:5f84523c076ad14ff5e6c037fe1c89a7f73a3e04cf0377cb4d017014976433f3", + "sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1", + "sha256:b4c423ab23291d3945ac61346feeb9a0dc4184999ede5e7c43e1ffb975130ae6", + "sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8", + "sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4", + "sha256:ca233c64c6e40eaa6c66ef97058cdc80e8d0157a443655baa1b2966e812807ca", + "sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269" + ], + "version": "==3.12" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, + "requests-oauthlib": { + "hashes": [ + "sha256:50a8ae2ce8273e384895972b56193c7409601a66d4975774c60c2aed869639ca", + "sha256:883ac416757eada6d3d07054ec7092ac21c7f35cb1d2cf82faf205637081f468" + ], + "version": "==0.8.0" + }, + "rsa": { + "hashes": [ + "sha256:25df4e10c263fb88b5ace923dd84bf9aa7f5019687b5e55382ffcdb8bede9db5", + "sha256:43f682fea81c452c98d09fc316aae12de6d30c4b5c84226642cf8f8fd1c93abd" + ], + "version": "==3.4.2" + }, + "six": { + "hashes": [ + "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", + "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + ], + "version": "==1.11.0" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "version": "==1.22" + }, + "websocket-client": { + "hashes": [ + "sha256:188b68b14fdb2d8eb1a111f21b9ffd2dbf1dbc4e4c1d28cf2c37cdbf1dd1cae6", + "sha256:a453dc4dfa6e0db3d8fd7738a308a88effe6240c59f3226eb93e8f020c216149" + ], + "version": "==0.47.0" + } + }, + "develop": {} +} diff --git a/clover/orchestration/__init__.py b/clover/orchestration/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/clover/orchestration/__init__.py diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py new file mode 100644 index 0000000..e5f1d89 --- /dev/null +++ b/clover/orchestration/kube_client.py @@ -0,0 +1,103 @@ +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +from os import path +import yaml + +from kubernetes import client, config + +class KubeClient(object): + + def __init__(self): + config.load_kube_config() + self.core_v1 = client.CoreV1Api() + self.extensions_v1beta1 = client.ExtensionsV1beta1Api() + + def find_svc_by_namespace(self, svc_name, namespace='default'): + ret_dict = {} + try: + svc = self.core_v1.read_namespaced_service(name=svc_name, + namespace=namespace) + except client.rest.ApiException: + svc = None + if not svc: + print('found no service %s in namespace %s' \ + % (svc_name, namespace)) + return None + ret_dict[svc.metadata.name] = {} + ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels + ret_dict[svc.metadata.name]['selector'] = svc.spec.selector + + return ret_dict + + def find_pod_by_namespace(self, namespace='default'): + ret_dict = {} + pods = self.core_v1.list_namespaced_pod(namespace=namespace) + if not pods: + print('found no pod') + return None + for pod in pods.items: + if pod.metadata.name not in ret_dict: + ret_dict[pod.metadata.name] = {} + ret_dict[pod.metadata.name]['labels'] = pod.metadata.labels + + return ret_dict + + def _check_pod(self, pod_name, namespace='defualt', container_name=None): + ret = self.core_v1.list_namespaced_pod(namespace=namespace) + ret_code = False + new_pod_name = None + for i in ret.items: + if pod_name in i.metadata.name: + if i.status.container_statuses and len(i.status.container_statuses) > 0: + container_up = False + for container in i.status.container_statuses: + check_state = True + if container_name: + if container_name != container.name: + check_state = False + if check_state and container.state.running is not None: + container_up = True + else: + if container_up: + container_up = False + break + if container_up: + ret_code = True + new_pod_name = i.metadata.name + return ret_code, new_pod_name + + def check_pod_up(self, pod_name, namespace='default'): + return self._check_pod(pod_name, namespace) + + def check_container_in_pods(self, container_name, pods, namespace='default'): + ret = False + for pod in pods: + ret, _ = self._check_pod(pod, namespace, container_name) + if not ret: + return ret + return ret + + def create_deployment_yaml(self, deployment_yaml_path, namespace='default'): + with open(deployment_yaml_path) as fp: + body = yaml.load(fp) + resp = self.extensions_v1beta1.create_namespaced_deployment( + body=body, namespace=namespace) + print('Deployment created. Status=%s' % str(resp.status)) + + dep_name = body.get('metadata').get('name') + return dep_name + + def create_service_yaml(self, service_yaml_path, namespace='default'): + with open(service_yaml_path) as fp: + body = yaml.load(fp) + resp = self.extensions_v1beta1.create_namespaced_service( + body=body, namespace=namespace) + print('Service created. Status=%s' % str(resp.status)) + + svc_name = body.get('metadata').get('name') + return svc_name diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py new file mode 100644 index 0000000..cc2ee0c --- /dev/null +++ b/clover/servicemesh/route_rules.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +import os +import redis +import subprocess +import sys +import yaml + +#istioctl='$HOME/istio-0.6.0/bin/istioctl' +# The assumption is that istioctl is already in the user's path +ISTIOCTL='istioctl' + +def cmd_exists(cmd): + return any( + os.access(os.path.join(path, cmd), os.X_OK) + for path in os.environ["PATH"].split(os.pathsep) + ) + +def load_route_rules(rr_yaml_path): + if not cmd_exists(ISTIOCTL): + print('%s does not exist in PATH, please export istioctl to PATH' % istioctl) + return False + + # TODO(s3wong): load yaml and verify it does indeed contain route rule + cmd = ISTIOCTL + ' create -f ' + rr_yaml_path + output = subprocess.check_output(cmd, shell=True) + if not output: + print('Route rule creation failed: %s' % output) + return False + return True + +def delete_route_rules(rr_yaml_path, namespace): + if not cmd_exists(ISTIOCTL): + print('%s does not exist in PATH, please export istioctl to PATH' % istioctl) + return False + + # TODO(s3wong): load yaml and verify it does indeed contain route rule + cmd = ISTIOCTL + ' delete -f ' + rr_yaml_path + ' -n ' + namespace + output = subprocess.check_output(cmd, shell=True) + if not output or not 'Deleted' in output: + print('Route rule deletion failed: %s' % output) + return False + return True + +def get_route_rules(): + if not cmd_exists(ISTIOCTL): + print('%s does not exist in PATH, please export istioctl to PATH' % istioctl) + return None + cmd = ISTIOCTL + ' get routerules -o yaml' + output = subprocess.check_output(cmd, shell=True) + if not output: + print('No route rule configured') + return None + docs = [] + for raw_doc in output.split('\n---'): + try: + docs.append(yaml.load(raw_doc)) + except SyntaxError: + print('syntax error: %s' % raw_doc) + return docs + +def parse_route_rules(routerules): + ret_list = [] + if not routerules: + print('No routerules') + return ret_list + for routerule in routerules: + if not routerule or routerule == 'None': continue + print('routerule is %s' % routerule) + if routerule.get('kind') != 'RouteRule': continue + ret_rr_dict = {} + spec = routerule.get('spec') + if not spec: continue + ret_rr_dict['service'] = spec.get('destination').get('name') + ret_rr_dict['rules'] = spec.get('route') + ret_list.append(ret_rr_dict) + return ret_list + +def _derive_key_from_test_id(test_id): + return 'route-rules-' + str(test_id) + +def set_route_rules(test_id): + r = redis.StrictRedis(host='localhost', port=6379, db=0) + key = _derive_key_from_test_id(test_id) + rr = get_route_rules() + r.set(key, rr) + +def fetch_route_rules(test_id): + r = redis.StrictRedis(host='localhost', port=6379, db=0) + key = _derive_key_from_test_id(test_id) + rr = r.get(key) + return yaml.load(rr) + +''' + The format of result_dict is expected to be: + { + 'service': <service name>, + <version string 1>: <integer representation of version string 1 occurrances during test>, + <version string 2>: <integer representation of version string 2 occurrances during test>, + ... + } +''' +def validate_weighted_route_rules(result_dict, test_id=None): + print('validate_weighted_route_rules: test id %s' % test_id) + svc_name = result_dict.get('service') + if not test_id: + rr_list = parse_route_rules(get_route_rules()) + else: + rr_list = parse_route_rules(fetch_route_rules(test_id)) + errors = [] + ret = True + for rr in rr_list: + route_rules = rr.get('rules') + if not route_rules: + break + for rule in route_rules: + version = rule.get('labels').get('version') + weight = rule.get('weight') + if not weight: weight = 1 + if abs(weight - result_dict[version]) > 10: + err = 'svc %s version %s expected to get %d, but got %d' % (svc_name, version, weight, result_dict[version]) + ret = False + else: + err = 'svc %s version %s expected to get %d, got %d. Validation succeeded' % (svc_name, version, weight, result_dict[version]) + errors.append(err) + return ret, errors + + diff --git a/clover/tools/__init__.py b/clover/tools/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/clover/tools/__init__.py diff --git a/clover/tools/clover_validate_rr.py b/clover/tools/clover_validate_rr.py new file mode 100644 index 0000000..ff1f8b4 --- /dev/null +++ b/clover/tools/clover_validate_rr.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +import getopt +import sys + +sys.path.insert(0, '..') + +from orchestration import kube_client +import servicemesh.route_rules as rr +from tracing.tracing import Tracing +from validate_rr import ValidateWRR + +def main(argv): + service_name = None + test_id = None + help_str = 'clover_validate_rr.py -t <test-id> -s <service name>' + try: + opts, args = getopt.getopt(argv,"hs:t:",["service-name", "test-id"]) + except getopt.GetoptError: + print help_str + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print help_str + sys.exit() + elif opt in ("-t", "--test-id"): + test_id = str(arg) + elif opt in ("-s", "--service-name"): + service_name = str(arg) + + if not service_name or not test_id: + print help_str + sys.exit(3) + + validate_wrr = ValidateWRR(test_id) + + istio_pods = ['istio-ingress', 'istio-mixer', 'istio-pilot'] + jaeger_pods = ['jaeger-deployment'] + + if not validate_wrr.check_pods_up(istio_pods + jaeger_pods, + 'istio-system'): + sys.exit(4) + + ret, errors = validate_wrr.validate(service_name) + for err in errors: + print err + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/clover/tools/validate_rr.py b/clover/tools/validate_rr.py new file mode 100644 index 0000000..0e7b9ed --- /dev/null +++ b/clover/tools/validate_rr.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +import sys + +sys.path.insert(0, '..') + +from orchestration import kube_client +import servicemesh.route_rules as rr +from tracing.tracing import Tracing + +class ValidateWRR(object): + + def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298'): + self._k8s_client = kube_client.KubeClient() + self._test_id = test_id + self._tracing = Tracing(tracing_ip, tracing_port) + + def check_pods_up(self, pod_list, namespace='default'): + for pod in pod_list: + up, name = self._k8s_client.check_pod_up(pod, namespace) + if not up: + print('pod %s in namespace %s not up' % (pod, namespace)) + return False + return True + + def set_test_id(self, test_id): + self._test_id = test_id + + def validate(self, service_name): + total = 0 + svc = self._k8s_client.find_svc_by_namespace(svc_name=service_name) + if not svc: + err_msg = 'Failed to locate service %s in default namespace' % service_name + print err_msg + return False, [err_msg] + pods = self._k8s_client.find_pod_by_namespace() + if not pods: + err_msg = 'No pod found in default namespace' + return False, [err_msg] + svc_pods = {} + for p,l in pods.items(): + pod_labels = l.get('labels') + svc_selector_dict = svc[service_name].get('selector') + for svc_select_key in svc_selector_dict: + if svc_select_key in pod_labels: + if svc_selector_dict[svc_select_key] == pod_labels[svc_select_key]: + svc_pods[p] = l + + trace_ids = self._tracing.getRedisTraceids(self._test_id) + rr_dict = {'service': service_name} + ver_count_dict = {} + for trace_id in trace_ids: + span_ids = self._tracing.getRedisSpanids(trace_id) + for span in span_ids: + # count only the received side --- i.e., messages sent TO + # service + node_id = self._tracing.getRedisTagsValue(span, trace_id, 'node_id') + direction = self._tracing.getRedisTagsValue(span, trace_id, 'upstream_cluster') + if direction.startswith('in.'): + for pod_name in svc_pods: + if pod_name in node_id: + total += 1 + labels = svc_pods[pod_name]['labels'] + print('node %s pod %s labels %s' % (node_id, pod_name, labels)) + if 'version' in labels: + version = labels.get('version') + if version in ver_count_dict: + ver_count_dict[version] += 1 + else: + ver_count_dict[version] = 1 + + print('total is %d, ver_count_dict is %s' % (total, ver_count_dict)) + if ver_count_dict and total > 0: + for version in ver_count_dict: + rr_dict[version] = float(ver_count_dict[version]) / float(total) * 100 + + return(rr.validate_weighted_route_rules(rr_dict, self._test_id)) + else: + err_msg = 'No version label found on any pod' + print(err_msg) + return False, [err_msg] + |