diff options
| -rw-r--r-- | clover/orchestration/kube_client.py | 19 | ||||
| -rw-r--r-- | clover/servicemesh/route_rules.py | 14 | ||||
| -rw-r--r-- | clover/test/app/sdc/clover-server4.yaml | 33 | ||||
| -rw-r--r-- | clover/test/app/sdc/clover-server5.yaml | 33 | ||||
| -rw-r--r-- | clover/test/app/sdc/lb-v2.yaml | 23 | ||||
| -rw-r--r-- | clover/test/fraser_a_b_test.py | 293 | ||||
| -rw-r--r-- | clover/test/istio/sdc/clover-server4-delay.yaml | 13 | ||||
| -rw-r--r-- | clover/test/istio/sdc/clover-server5-delay.yaml | 13 | ||||
| -rw-r--r-- | clover/test/istio/sdc/route-rule-lb-50-v2.yaml | 15 | ||||
| -rw-r--r-- | clover/test/istio/sdc/route-rule-lb-v1.yaml | 11 | ||||
| -rw-r--r-- | clover/test/istio/sdc/route-rule-lb-v2.yaml | 11 | ||||
| -rwxr-xr-x | clover/test/script/lb-test.sh | 7 | ||||
| -rw-r--r-- | clover/test/validate_success.py | 131 | ||||
| -rw-r--r-- | clover/test/yaml/fraser_a_b_test.yaml | 28 | ||||
| -rw-r--r-- | clover/tools/validate_rr.py | 6 |
15 files changed, 646 insertions, 4 deletions
diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py index e5f1d89..f7fa708 100644 --- a/clover/orchestration/kube_client.py +++ b/clover/orchestration/kube_client.py @@ -31,9 +31,28 @@ class KubeClient(object): ret_dict[svc.metadata.name] = {} ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels ret_dict[svc.metadata.name]['selector'] = svc.spec.selector + ret_dict[svc.metadata.name]['cluster_ip'] = svc.spec.cluster_ip return ret_dict + def find_pod_by_name(self, pod_name, namespace='default'): + ret_dict = {} + try: + pod = self.core_v1.read_namespaced_pod(name=pod_name, + namespace=namespace) + except client.rest.ApiException: + pod = None + if not pod: + print('found no pod %s in namespace %s' \ + % (pod_name, namespace)) + return None + ret_dict['name'] = pod_name + ret_dict['labels'] = pod.metadata.labels + ret_dict['pod_ip'] = pod.status.pod_ip + + return ret_dict + + def find_pod_by_namespace(self, namespace='default'): ret_dict = {} pods = self.core_v1.list_namespaced_pod(namespace=namespace) diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py index cc2ee0c..935940e 100644 --- a/clover/servicemesh/route_rules.py +++ b/clover/servicemesh/route_rules.py @@ -12,6 +12,8 @@ import subprocess import sys import yaml +from clover.orchestration.kube_client import KubeClient + #istioctl='$HOME/istio-0.6.0/bin/istioctl' # The assumption is that istioctl is already in the user's path ISTIOCTL='istioctl' @@ -85,14 +87,22 @@ def parse_route_rules(routerules): def _derive_key_from_test_id(test_id): return 'route-rules-' + str(test_id) +def _get_redis_ip(): + k8s_client = KubeClient() + redis_pod = k8s_client.find_pod_by_name('redis') + redis_ip = redis_pod.get('pod_ip') + return redis_ip + def set_route_rules(test_id): - r = redis.StrictRedis(host='localhost', port=6379, db=0) + redis_ip = _get_redis_ip() + r = redis.StrictRedis(host=redis_ip, port=6379, db=0) key = _derive_key_from_test_id(test_id) rr = get_route_rules() r.set(key, rr) def fetch_route_rules(test_id): - r = redis.StrictRedis(host='localhost', port=6379, db=0) + redis_ip = _get_redis_ip() + r = redis.StrictRedis(host=redis_ip, port=6379, db=0) key = _derive_key_from_test_id(test_id) rr = r.get(key) return yaml.load(rr) diff --git a/clover/test/app/sdc/clover-server4.yaml b/clover/test/app/sdc/clover-server4.yaml new file mode 100644 index 0000000..e7d8ebb --- /dev/null +++ b/clover/test/app/sdc/clover-server4.yaml @@ -0,0 +1,33 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-server4 + labels: + app: clover-server4 +spec: + template: + metadata: + labels: + app: clover-server4 + spec: + containers: + - name: clover-server4 + image: opnfv/clover-ns-nginx-server:latest + ports: + - containerPort: 50054 + - containerPort: 9180 +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-server4 + labels: + app: clover-server4 +spec: + ports: + - port: 50054 + name: grpc + - port: 9180 + name: http + selector: + app: clover-server4 diff --git a/clover/test/app/sdc/clover-server5.yaml b/clover/test/app/sdc/clover-server5.yaml new file mode 100644 index 0000000..911b81e --- /dev/null +++ b/clover/test/app/sdc/clover-server5.yaml @@ -0,0 +1,33 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: clover-server5 + labels: + app: clover-server5 +spec: + template: + metadata: + labels: + app: clover-server5 + spec: + containers: + - name: clover-server5 + image: opnfv/clover-ns-nginx-server:latest + ports: + - containerPort: 50054 + - containerPort: 9180 +--- +apiVersion: v1 +kind: Service +metadata: + name: clover-server5 + labels: + app: clover-server5 +spec: + ports: + - port: 50054 + name: grpc + - port: 9180 + name: http + selector: + app: clover-server5 diff --git a/clover/test/app/sdc/lb-v2.yaml b/clover/test/app/sdc/lb-v2.yaml new file mode 100644 index 0000000..22285c4 --- /dev/null +++ b/clover/test/app/sdc/lb-v2.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: http-lb-v2 + labels: + app: http-lb + version: v2 +spec: + template: + metadata: + labels: + app: http-lb + version: v2 + spec: + containers: + - name: http-lb + image: localhost:5000/http-lb2 + ports: + - containerPort: 50054 + - containerPort: 9188 +--- + diff --git a/clover/test/fraser_a_b_test.py b/clover/test/fraser_a_b_test.py new file mode 100644 index 0000000..cfbc79f --- /dev/null +++ b/clover/test/fraser_a_b_test.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +import getopt +import subprocess +import sys +import time +import uuid +import yaml + +#sys.path.insert(0, '..') + +from clover.orchestration.kube_client import KubeClient +import clover.servicemesh.route_rules as rr +from clover.tools.validate_rr import ValidateWRR +from clover.tracing.tracing import Tracing + +from validate_success import validate_perf + +def _format_perf_data(perf_dict, dep_name, svc): + in_pod= None + out_pod = None + out_pod_list = [] + for key, perf in perf_dict.items(): + if key == 'in': + continue + elif key == 'out': + if 'out_svc' in perf: + out_pod = perf.get('out_svc') + elif 'out_svc' in perf: + if perf.get('out_svc') == svc: + in_pod = key + + if out_pod: + out_pod_list = [key for key in perf_dict.keys() if out_pod in key.lower()] + if out_pod_list: + out_pod = out_pod_list[0] + print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name] + out_pod_list)) + print("{: >20} {: >20} {: >20}".format(*[perf_dict[in_pod].get('average'), + perf_dict['in'].get('average'), + perf_dict[out_pod].get('average')])) + return + + print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name, out_pod])) + print("{: >20} {: >20}".format(*[perf_dict[in_pod].get('average'), + perf_dict['in'].get('average')])) + + + +def main(argv): + test_yaml = None + namespace = 'default' + tracing_port = 0 + help_str = 'python fraser_a_b_test.py -t <test-yaml> -n <namespace> -p <tracing port>' + try: + opts, args = getopt.getopt(argv,"ht:n:p:",["test-yaml", "namespace", "tracing-port"]) + except getopt.GetoptError: + print help_str + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print help_str + sys.exit() + elif opt in ("-t", "--test-yaml"): + test_yaml = str(arg) + elif opt in ("-n", "--namespace"): + namespace = str(arg) + elif opt in ("-p", "--tracing-port"): + tracing_port = int(arg) + + if not test_yaml or tracing_port == 0: + print help_str + sys.exit(3) + + with open(test_yaml) as fp: + test_params = yaml.load(fp) + + ''' + Steps: + (1) get version one info + (2) get version two info + (3) start version two + (4) validate version two pod and sidecar all up + (5) load A-B testing route rules + (6) execute traffic test script + (7) validate route rules traffic distribution + (8) validate version two success criteria + (9) if (8) works, change to version 2 only + (10) execute traffic test script + (11) validate route rules traffic distribution + ''' + APP_BASE = 'test/app/' + POLICY_BASE = 'test/istio/' + SCRIPT_BASE = 'test/script/' + print('Current pods running at namespace %s' % namespace) + # as this is just for display purpose, we directly use kubectl get pods + cmd = 'kubectl get pods -n %s' % namespace + output = subprocess.check_output(cmd, shell=True) + print(output) + + print('Current services running at namespace %s' % namespace) + cmd = 'kubectl get svc -n %s' % namespace + output = subprocess.check_output(cmd, shell=True) + print(output) + + # service under test + test_svc = test_params.get('test-svc') + print('Service under test: %s' % test_svc) + + k8s_client = KubeClient() + on, _ = k8s_client.check_pod_up('istio-sidecar-injector', 'istio-system') + print('Istio automatic sidecar injection is %s' % on) + dep_a_name = test_params.get('deployment-A') + dep_b = test_params.get('deployment-B') + dep_b_name = dep_b.get('name') + dep_b_yaml = APP_BASE + dep_b.get('manifest') + additional_deps = test_params.get('additional-deployments') + + # TODO(s3wong): use istio-inject, then use kube_client to invoke + dep_list = [] + print('Deploying %s...' % dep_b_name) + if not on: + cmd_temp = 'istioctl kube-inject -f %s > app/__tmp.yaml; kubectl apply -f app/__tmp.yaml; rm -f app/__tmp.yaml' + else: + cmd_temp = 'kubectl apply -f %s' + + up, _ = k8s_client.check_pod_up(dep_b_name, namespace=namespace) + if up: + print('%s already has pod up, no need to spawn...' % dep_b_name) + else: + cmd = cmd_temp % dep_b_yaml + output = subprocess.check_output(cmd, shell=True) + print(output) + dep_list.append({'name': dep_b_name, 'up': False}) + if additional_deps: + for dep in additional_deps: + dep_name = dep.get('name') + dep_yaml = APP_BASE + dep.get('manifest') + up, _ = k8s_client.check_pod_up(dep_name, namespace=namespace) + if up: + print('%s already has pod up, no need to spawn...' % dep_name) + else: + cmd = cmd_temp % dep_yaml + output = subprocess.check_output(cmd, shell=True) + print(output) + dep_list.append({'name': dep_name, 'up': False}) + + time.sleep(3) + + wait_count = 0 + continue_waiting = False + while wait_count < 5: + continue_waiting = False + for dep in dep_list: + if not dep.get('up'): + dep['up'], _ = k8s_client.check_pod_up(dep.get('name'), namespace=namespace) + if not dep['up']: + continue_waiting = True + if continue_waiting: + wait_count += 1 + time.sleep(3) + else: + break + + if continue_waiting: + print('Some pods are still not up after 15 seconds: %s' % dep_list) + sys.exit(4) + + print('All pods are up') + cmd = 'kubectl get pods -n %s' % namespace + output = subprocess.check_output(cmd, shell=True) + print(output) + + time.sleep(3) + + a_b_test_rr_yaml = POLICY_BASE + test_params.get('ab-test-rr') + print('Loading route rules in %s' % a_b_test_rr_yaml) + ret = rr.load_route_rules(a_b_test_rr_yaml) + print('Route rules are now %s' % rr.get_route_rules()) + + time.sleep(5) + + redis_pod = k8s_client.find_pod_by_name('redis') + if not redis_pod: + print('redis not running in default namespace') + sys.exit(6) + redis_ip = redis_pod.get('pod_ip') + tracing = Tracing(tracing_ip='localhost', + tracing_port=str(tracing_port), + redis_ip=redis_ip) + # turn off tracing to redis for warm up run + tracing.use_redis = False + traffic_test_dict = test_params.get('traffic-test') + traffic_test_script = traffic_test_dict.get('name') + traffic_test_params = traffic_test_dict.get('params') + cmd = SCRIPT_BASE + traffic_test_script + if traffic_test_params: + for param in traffic_test_params: + cmd = cmd + ' ' + str(param) + print('Execute traffic test %s' % cmd) + ''' + print('Warming up for route rules to take place') + try: + output = subprocess.check_output(cmd, shell=True) + except subprocess.CalledProcessError, e: + print('%s returns error %s' % e.output) + print(output) + print('Running recorded traffic test...') + ''' + time.sleep(30) + tracing.use_redis = True + test_id = uuid.uuid4() + rr.set_route_rules(test_id) + tracing.setTest(test_id) + try: + output = subprocess.check_output(cmd, shell=True) + except subprocess.CalledProcessError, e: + print('non zero return value on traffic script: %s, ignoring...' % e.output) + print(output) + time.sleep(30) + traces = tracing.getTraces(test_svc, 0) + tracing.outTraces(traces) + + time.sleep(3) + print('Validating route rules...') + validate_wrr = ValidateWRR(test_id, redis_ip=redis_ip) + ret, errors = validate_wrr.validate(test_svc) + + # TODO(s3wong): for now, route rules failure seems more like a warning + if ret: + print('Route rules for service %s validated' % test_svc) + else: + print('Route rules for service %s validation failed' % test_svc) + for err in errors: + print err + + success_factors = test_params.get('success') + if success_factors: + criteria = success_factors.get('criteria') + success_check = True + for criterion in criteria: + c_type = criterion.get('type') + if c_type == 'performance': + condition = int(criterion.get('condition')) + ret_dict = validate_perf(tracing, test_id, test_svc, + dep_a_name, dep_b_name) + # print performance data + _format_perf_data(ret_dict.get(dep_a_name), dep_a_name, test_svc) + print('\n') + _format_perf_data(ret_dict.get(dep_b_name), dep_b_name, test_svc) + ret = (ret_dict.get(dep_b_name).get('in').get('average') <= \ + (ret_dict.get(dep_a_name).get('in').get('average') * condition / 100)) + if not ret: + print('Performance check failed') + success_check = False + break + else: + print('Performance check succeed') + ''' + elif c_type == 'services': + srv_list = criterion.get('services') + ret = check_services_traverse(tracing, test_id, test_svc, + dep_b_name, srv_list) + if not ret: + print('Additional services traversal test failed') + success_check = False + break + else: + print('Additional services traversal test succeed') + ''' + if success_check: + actions = success_factors.get('action') + else: + failed = success_factors.get('failed') + actions = failed.get('action') + for action in actions: + action_type = action.get('type') + if action_type == 'commit' or action_type == 'rollback': + rr.delete_route_rules(a_b_test_rr_yaml, namespace) + ret = rr.load_route_rules(POLICY_BASE + action.get('routerule')) + if ret: + print('loading route rule %s succeed' % action.get('routerule')) + + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/clover/test/istio/sdc/clover-server4-delay.yaml b/clover/test/istio/sdc/clover-server4-delay.yaml new file mode 100644 index 0000000..32a9f6a --- /dev/null +++ b/clover/test/istio/sdc/clover-server4-delay.yaml @@ -0,0 +1,13 @@ +apiVersion: config.istio.io/v1alpha2 +kind: RouteRule +metadata: + name: clover-server4-delay + namespace: default +spec: + destination: + name: clover-server4 + httpFault: + delay: + fixedDelay: 1.000s + percent: 100 + precedence: 2 diff --git a/clover/test/istio/sdc/clover-server5-delay.yaml b/clover/test/istio/sdc/clover-server5-delay.yaml new file mode 100644 index 0000000..0c8bcd8 --- /dev/null +++ b/clover/test/istio/sdc/clover-server5-delay.yaml @@ -0,0 +1,13 @@ +apiVersion: config.istio.io/v1alpha2 +kind: RouteRule +metadata: + name: clover-server5-delay + namespace: default +spec: + destination: + name: clover-server5 + httpFault: + delay: + fixedDelay: 1.000s + percent: 100 + precedence: 2 diff --git a/clover/test/istio/sdc/route-rule-lb-50-v2.yaml b/clover/test/istio/sdc/route-rule-lb-50-v2.yaml new file mode 100644 index 0000000..529da78 --- /dev/null +++ b/clover/test/istio/sdc/route-rule-lb-50-v2.yaml @@ -0,0 +1,15 @@ +apiVersion: config.istio.io/v1alpha2 +kind: RouteRule +metadata: + name: lb-default +spec: + destination: + name: http-lb + precedence: 1 + route: + - labels: + version: v1 + weight: 50 + - labels: + version: v2 + weight: 50 diff --git a/clover/test/istio/sdc/route-rule-lb-v1.yaml b/clover/test/istio/sdc/route-rule-lb-v1.yaml new file mode 100644 index 0000000..a3cac3a --- /dev/null +++ b/clover/test/istio/sdc/route-rule-lb-v1.yaml @@ -0,0 +1,11 @@ +apiVersion: config.istio.io/v1alpha2 +kind: RouteRule +metadata: + name: lb-default +spec: + destination: + name: http-lb + precedence: 1 + route: + - labels: + version: v1 diff --git a/clover/test/istio/sdc/route-rule-lb-v2.yaml b/clover/test/istio/sdc/route-rule-lb-v2.yaml new file mode 100644 index 0000000..23a3c8d --- /dev/null +++ b/clover/test/istio/sdc/route-rule-lb-v2.yaml @@ -0,0 +1,11 @@ +apiVersion: config.istio.io/v1alpha2 +kind: RouteRule +metadata: + name: lb-default +spec: + destination: + name: http-lb + precedence: 1 + route: + - labels: + version: v2 diff --git a/clover/test/script/lb-test.sh b/clover/test/script/lb-test.sh new file mode 100755 index 0000000..4d10386 --- /dev/null +++ b/clover/test/script/lb-test.sh @@ -0,0 +1,7 @@ +#!/bin/bash +for i in `seq 1 20`; +do + #wget http://10.244.0.1:32580/ + wget http://$1:$2/ + sleep 1 +done diff --git a/clover/test/validate_success.py b/clover/test/validate_success.py new file mode 100644 index 0000000..65cd579 --- /dev/null +++ b/clover/test/validate_success.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +from clover.orchestration import kube_client +from clover.tracing.tracing import Tracing + +def _get_svc_pods(svc_name, namespace): + k8s_client = kube_client.KubeClient() + svc = k8s_client.find_svc_by_namespace(svc_name=svc_name, + namespace=namespace) + if not svc: + err_msg = 'Failed to locate service %s in %s namespace' \ + % (svc_name, namespace) + print err_msg + return False, [err_msg] + pods = k8s_client.find_pod_by_namespace() + if not pods: + err_msg = 'No pod found in namespace %s' % namespace + return False, [err_msg] + svc_pods = {} + for p,l in pods.items(): + pod_labels = l.get('labels') + svc_selector_dict = svc[service_name].get('selector') + for svc_select_key in svc_selector_dict: + if svc_select_key in pod_labels: + if svc_selector_dict[svc_select_key] == pod_labels[svc_select_key]: + svc_pods[p] = l + return svc_pods + +def validate_perf(tracing, test_id, svc_name, control_svc, variant_svc): + ret_dict = {} + ret_dict[control_svc] = {} + ret_dict[control_svc]['in'] = {} + ret_dict[control_svc]['in']['total'] = 0 + ret_dict[control_svc]['in']['average'] = 0 + ret_dict[control_svc]['out'] = {} + ret_dict[control_svc]['out']['total'] = 0 + ret_dict[control_svc]['out']['average'] = 0 + + ret_dict[variant_svc] = {} + ret_dict[variant_svc]['in'] = {} + ret_dict[variant_svc]['in']['total'] = 0 + ret_dict[variant_svc]['in']['average'] = 0 + ret_dict[variant_svc]['out'] = {} + ret_dict[variant_svc]['out']['total'] = 0 + ret_dict[variant_svc]['out']['average'] = 0 + + req_id_dict = {} + def _fill_up_ret_dict(direction, svc, duration, out_svc=None): + sum = ret_dict[svc][direction]['average'] * \ + ret_dict[svc][direction]['total'] + \ + int(duration) + ret_dict[svc][direction]['total'] += 1 + ret_dict[svc][direction]['average'] = \ + float(sum) / float(ret_dict[svc][direction]['total']) + if direction == 'out' and out_svc: + # tracking the out service from svc + # TODO(s3wong): this assumes only ONE direction from + # service to another service, which may not be true + # in essence, the data structure should track (srv, out) + # pairs and calculate average that way + ret_dict[svc][direction]['out_svc'] = out_svc + + + def _check_req_id(req_id, svc=None, node_id=None, + duration=None, direction=None, + out_svc=None): + if req_id not in req_id_dict: + req_id_dict[req_id] = {} + + if svc: + req_id_dict[req_id]['svc'] = svc + else: + req_id_dict[req_id]['node_id'] = node_id + req_id_dict[req_id]['duration'] = int(duration) + req_id_dict[req_id]['direction'] = direction + if direction == 'out' and out_svc: + req_id_dict[req_id]['out_svc'] = out_svc + + trace_ids = tracing.getRedisTraceids(test_id) + for trace_id in trace_ids: + span_ids = tracing.getRedisSpanids(trace_id) + for span in span_ids: + out_svc = None + duration = tracing.getRedisSpanValue(span, trace_id, 'duration') + node_id = tracing.getRedisTagsValue(span, trace_id, 'node_id') + upstream_cluster = tracing.getRedisTagsValue(span, trace_id, 'upstream_cluster') + req_id = tracing.getRedisTagsValue(span, trace_id, 'guid:x-request-id') + if upstream_cluster.startswith('in.'): + direction = 'in' + else: + direction = 'out' + out_svc = upstream_cluster.split('.')[1] + if control_svc in node_id: + _fill_up_ret_dict(direction, control_svc, duration, out_svc=out_svc) + _check_req_id(req_id, svc=control_svc) + elif variant_svc in node_id: + _fill_up_ret_dict(direction, variant_svc, duration, out_svc=out_svc) + _check_req_id(req_id, svc=variant_svc) + else: + # client to svc or server from svc as client + if out_svc and out_svc == svc_name: + _check_req_id(req_id, node_id=node_id, direction=direction, + duration=duration, out_svc=out_svc) + + for req_id, svc_dict in req_id_dict.items(): + node_id = svc_dict.get('node_id') + if not node_id: + continue + pod_name = node_id.split('~')[2] + svc = svc_dict.get('svc') + if pod_name not in ret_dict.get(svc): + ret_dict[svc][pod_name] = {} + ret_dict[svc][pod_name]['total'] = 0 + ret_dict[svc][pod_name]['direction'] = svc_dict.get('direction') + if svc_dict.get('out_svc'): + ret_dict[svc][pod_name]['out_svc'] = svc_dict.get('out_svc') + ret_dict[svc][pod_name]['average'] = 0 + sum = ret_dict[svc][pod_name]['average'] * \ + ret_dict[svc][pod_name]['total'] + \ + svc_dict.get('duration') + ret_dict[svc][pod_name]['total'] += 1 + ret_dict[svc][pod_name]['average'] = \ + float(sum) / float(ret_dict[svc][pod_name]['total']) + + return ret_dict diff --git a/clover/test/yaml/fraser_a_b_test.yaml b/clover/test/yaml/fraser_a_b_test.yaml new file mode 100644 index 0000000..d8fbc0b --- /dev/null +++ b/clover/test/yaml/fraser_a_b_test.yaml @@ -0,0 +1,28 @@ +test-svc: http-lb +kind: A-B-Testing +deployment-A: http-lb-v1 +deployment-B: + name: http-lb-v2 + manifest: sdc/lb-v2.yaml +additional-deployments: + - name: clover-server4 + manifest: sdc/clover-server4.yaml + - name: clover-server5 + manifest: sdc/clover-server5.yaml +ab-test-rr: sdc/route-rule-lb-50-v2.yaml +traffic-test: + name: lb-test.sh + params: + - 10.244.0.1 + - 32580 +success: + criteria: + - type: performance + condition: 120 + action: + - type: commit + routerule: sdc/route-rule-lb-v2.yaml + failed: + action: + - type: rollback + routerule: sdc/route-rule-lb-v1.yaml diff --git a/clover/tools/validate_rr.py b/clover/tools/validate_rr.py index aa1b211..fbda20e 100644 --- a/clover/tools/validate_rr.py +++ b/clover/tools/validate_rr.py @@ -14,10 +14,12 @@ from clover.tracing.tracing import Tracing class ValidateWRR(object): - def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298'): + def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298', + redis_ip='localhost'): self._k8s_client = kube_client.KubeClient() self._test_id = test_id - self._tracing = Tracing(tracing_ip, tracing_port) + self._tracing = Tracing(tracing_ip, tracing_port, + redis_ip=redis_ip) def check_pods_up(self, pod_list, namespace='default'): for pod in pod_list: |