Merge "Experimental commit for A-B testing with Clover Fraser release and on the SDC application" into stable/fraser

15 files changed, 646 insertions, 4 deletions

diff --git a/clover/orchestration/kube_client.py b/clover/orchestration/kube_client.py
index e5f1d89..f7fa708 100644
--- a/clover/orchestration/kube_client.py
+++ b/clover/orchestration/kube_client.py
@@ -31,9 +31,28 @@ class KubeClient(object):
         ret_dict[svc.metadata.name] = {}
         ret_dict[svc.metadata.name]['labels'] = svc.metadata.labels
         ret_dict[svc.metadata.name]['selector'] = svc.spec.selector
+        ret_dict[svc.metadata.name]['cluster_ip'] = svc.spec.cluster_ip
 
         return ret_dict
 
+    def find_pod_by_name(self, pod_name, namespace='default'):
+        ret_dict = {}
+        try:
+            pod = self.core_v1.read_namespaced_pod(name=pod_name,
+                                                   namespace=namespace)
+        except client.rest.ApiException:
+            pod = None
+        if not pod:
+            print('found no pod %s in namespace %s' \
+                   % (pod_name, namespace))
+            return None
+        ret_dict['name'] = pod_name
+        ret_dict['labels'] = pod.metadata.labels
+        ret_dict['pod_ip'] = pod.status.pod_ip
+
+        return ret_dict
+
+
     def find_pod_by_namespace(self, namespace='default'):
         ret_dict = {}
         pods = self.core_v1.list_namespaced_pod(namespace=namespace)
diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py
index cc2ee0c..935940e 100644
--- a/clover/servicemesh/route_rules.py
+++ b/clover/servicemesh/route_rules.py
@@ -12,6 +12,8 @@ import subprocess
 import sys
 import yaml
 
+from clover.orchestration.kube_client import KubeClient
+
 #istioctl='$HOME/istio-0.6.0/bin/istioctl'
 # The assumption is that istioctl is already in the user's path
 ISTIOCTL='istioctl'
@@ -85,14 +87,22 @@ def parse_route_rules(routerules):
 def _derive_key_from_test_id(test_id):
     return 'route-rules-' + str(test_id)
 
+def _get_redis_ip():
+    k8s_client = KubeClient()
+    redis_pod = k8s_client.find_pod_by_name('redis')
+    redis_ip = redis_pod.get('pod_ip')
+    return redis_ip
+
 def set_route_rules(test_id):
-    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    redis_ip = _get_redis_ip()
+    r = redis.StrictRedis(host=redis_ip, port=6379, db=0)
     key = _derive_key_from_test_id(test_id)
     rr = get_route_rules()
     r.set(key, rr)
 
 def fetch_route_rules(test_id):
-    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    redis_ip = _get_redis_ip()
+    r = redis.StrictRedis(host=redis_ip, port=6379, db=0)
     key = _derive_key_from_test_id(test_id)
     rr = r.get(key)
     return yaml.load(rr)
diff --git a/clover/test/app/sdc/clover-server4.yaml b/clover/test/app/sdc/clover-server4.yaml
new file mode 100644
index 0000000..e7d8ebb
--- /dev/null
+++ b/clover/test/app/sdc/clover-server4.yaml
@@ -0,0 +1,33 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: clover-server4
+  labels:
+    app: clover-server4
+spec:
+  template:
+    metadata:
+      labels:
+        app: clover-server4
+    spec:
+      containers:
+        - name: clover-server4
+          image: opnfv/clover-ns-nginx-server:latest
+          ports:
+           - containerPort: 50054
+           - containerPort: 9180
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clover-server4
+  labels:
+    app: clover-server4
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  - port: 9180
+    name: http
+  selector:
+    app: clover-server4
diff --git a/clover/test/app/sdc/clover-server5.yaml b/clover/test/app/sdc/clover-server5.yaml
new file mode 100644
index 0000000..911b81e
--- /dev/null
+++ b/clover/test/app/sdc/clover-server5.yaml
@@ -0,0 +1,33 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: clover-server5
+  labels:
+    app: clover-server5
+spec:
+  template:
+    metadata:
+      labels:
+        app: clover-server5
+    spec:
+      containers:
+        - name: clover-server5
+          image: opnfv/clover-ns-nginx-server:latest
+          ports:
+           - containerPort: 50054
+           - containerPort: 9180
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clover-server5
+  labels:
+    app: clover-server5
+spec:
+  ports:
+  - port: 50054
+    name: grpc
+  - port: 9180
+    name: http
+  selector:
+    app: clover-server5
diff --git a/clover/test/app/sdc/lb-v2.yaml b/clover/test/app/sdc/lb-v2.yaml
new file mode 100644
index 0000000..22285c4
--- /dev/null
+++ b/clover/test/app/sdc/lb-v2.yaml
@@ -0,0 +1,23 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: http-lb-v2
+  labels:
+    app: http-lb
+    version: v2
+spec:
+  template:
+    metadata:
+      labels:
+        app: http-lb
+        version: v2
+    spec:
+      containers:
+        - name: http-lb
+          image: localhost:5000/http-lb2
+          ports:
+           - containerPort: 50054
+           - containerPort: 9188
+---
+
diff --git a/clover/test/fraser_a_b_test.py b/clover/test/fraser_a_b_test.py
new file mode 100644
index 0000000..cfbc79f
--- /dev/null
+++ b/clover/test/fraser_a_b_test.py
@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import getopt
+import subprocess
+import sys
+import time
+import uuid
+import yaml
+
+#sys.path.insert(0, '..')
+
+from clover.orchestration.kube_client import KubeClient
+import clover.servicemesh.route_rules as rr
+from clover.tools.validate_rr import ValidateWRR
+from clover.tracing.tracing import Tracing
+
+from validate_success import validate_perf
+
+def _format_perf_data(perf_dict, dep_name, svc):
+    in_pod= None
+    out_pod = None
+    out_pod_list = []
+    for key, perf in perf_dict.items():
+        if key == 'in':
+            continue
+        elif key == 'out':
+            if 'out_svc' in perf:
+                out_pod = perf.get('out_svc')
+        elif 'out_svc' in perf:
+            if perf.get('out_svc') == svc:
+                in_pod = key
+
+    if out_pod:
+        out_pod_list = [key for key in perf_dict.keys() if out_pod in key.lower()]
+        if out_pod_list:
+            out_pod = out_pod_list[0]
+            print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name] + out_pod_list))
+            print("{: >20} {: >20} {: >20}".format(*[perf_dict[in_pod].get('average'),
+                                                     perf_dict['in'].get('average'),
+                                                     perf_dict[out_pod].get('average')]))
+            return
+
+    print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name, out_pod]))
+    print("{: >20} {: >20}".format(*[perf_dict[in_pod].get('average'),
+                                     perf_dict['in'].get('average')]))
+
+
+
+def main(argv):
+    test_yaml = None
+    namespace = 'default'
+    tracing_port = 0
+    help_str = 'python fraser_a_b_test.py -t <test-yaml> -n <namespace> -p <tracing port>'
+    try:
+        opts, args = getopt.getopt(argv,"ht:n:p:",["test-yaml", "namespace", "tracing-port"])
+    except getopt.GetoptError:
+        print help_str
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            print help_str
+            sys.exit()
+        elif opt in ("-t", "--test-yaml"):
+            test_yaml = str(arg)
+        elif opt in ("-n", "--namespace"):
+            namespace = str(arg)
+        elif opt in ("-p", "--tracing-port"):
+            tracing_port = int(arg)
+
+    if not test_yaml or tracing_port == 0:
+        print help_str
+        sys.exit(3)
+
+    with open(test_yaml) as fp:
+        test_params = yaml.load(fp)
+
+    '''
+    Steps:
+    (1) get version one info
+    (2) get version two info
+    (3) start version two
+    (4) validate version two pod and sidecar all up
+    (5) load A-B testing route rules
+    (6) execute traffic test script
+    (7) validate route rules traffic distribution
+    (8) validate version two success criteria
+    (9) if (8) works, change to version 2 only
+    (10) execute traffic test script
+    (11) validate route rules traffic distribution
+    '''
+    APP_BASE = 'test/app/'
+    POLICY_BASE = 'test/istio/'
+    SCRIPT_BASE = 'test/script/'
+    print('Current pods running at namespace %s' % namespace)
+    # as this is just for display purpose, we directly use kubectl get pods
+    cmd = 'kubectl get pods -n %s' % namespace
+    output = subprocess.check_output(cmd, shell=True)
+    print(output)
+
+    print('Current services running at namespace %s' % namespace)
+    cmd = 'kubectl get svc -n %s' % namespace
+    output = subprocess.check_output(cmd, shell=True)
+    print(output)
+
+    # service under test
+    test_svc = test_params.get('test-svc')
+    print('Service under test: %s' % test_svc)
+
+    k8s_client = KubeClient()
+    on, _ = k8s_client.check_pod_up('istio-sidecar-injector', 'istio-system')
+    print('Istio automatic sidecar injection is %s' % on)
+    dep_a_name = test_params.get('deployment-A')
+    dep_b = test_params.get('deployment-B')
+    dep_b_name = dep_b.get('name')
+    dep_b_yaml = APP_BASE + dep_b.get('manifest')
+    additional_deps = test_params.get('additional-deployments')
+
+    # TODO(s3wong): use istio-inject, then use kube_client to invoke
+    dep_list = []
+    print('Deploying %s...' % dep_b_name)
+    if not on:
+        cmd_temp = 'istioctl kube-inject -f %s > app/__tmp.yaml; kubectl apply -f app/__tmp.yaml; rm -f app/__tmp.yaml'
+    else:
+        cmd_temp = 'kubectl apply -f %s'
+
+    up, _ = k8s_client.check_pod_up(dep_b_name, namespace=namespace)
+    if up:
+        print('%s already has pod up, no need to spawn...' % dep_b_name)
+    else:
+        cmd = cmd_temp % dep_b_yaml
+        output = subprocess.check_output(cmd, shell=True)
+        print(output)
+        dep_list.append({'name': dep_b_name, 'up': False})
+    if additional_deps:
+        for dep in additional_deps:
+            dep_name = dep.get('name')
+            dep_yaml = APP_BASE + dep.get('manifest')
+            up, _ = k8s_client.check_pod_up(dep_name, namespace=namespace)
+            if up:
+                print('%s already has pod up, no need to spawn...' % dep_name)
+            else:
+                cmd = cmd_temp % dep_yaml
+                output = subprocess.check_output(cmd, shell=True)
+                print(output)
+                dep_list.append({'name': dep_name, 'up': False})
+
+    time.sleep(3)
+
+    wait_count = 0
+    continue_waiting = False
+    while wait_count < 5:
+        continue_waiting = False
+        for dep in dep_list:
+            if not dep.get('up'):
+                dep['up'], _ = k8s_client.check_pod_up(dep.get('name'), namespace=namespace)
+                if not dep['up']:
+                    continue_waiting = True
+        if continue_waiting:
+            wait_count += 1
+            time.sleep(3)
+        else:
+            break
+
+    if continue_waiting:
+        print('Some pods are still not up after 15 seconds: %s' % dep_list)
+        sys.exit(4)
+
+    print('All pods are up')
+    cmd = 'kubectl get pods -n %s' % namespace
+    output = subprocess.check_output(cmd, shell=True)
+    print(output)
+
+    time.sleep(3)
+
+    a_b_test_rr_yaml = POLICY_BASE + test_params.get('ab-test-rr')
+    print('Loading route rules in %s' % a_b_test_rr_yaml)
+    ret = rr.load_route_rules(a_b_test_rr_yaml)
+    print('Route rules are now %s' % rr.get_route_rules())
+
+    time.sleep(5)
+
+    redis_pod = k8s_client.find_pod_by_name('redis')
+    if not redis_pod:
+        print('redis not running in default namespace')
+        sys.exit(6)
+    redis_ip = redis_pod.get('pod_ip')
+    tracing = Tracing(tracing_ip='localhost',
+                      tracing_port=str(tracing_port),
+                      redis_ip=redis_ip)
+    # turn off tracing to redis for warm up run
+    tracing.use_redis = False
+    traffic_test_dict = test_params.get('traffic-test')
+    traffic_test_script = traffic_test_dict.get('name')
+    traffic_test_params = traffic_test_dict.get('params')
+    cmd = SCRIPT_BASE + traffic_test_script
+    if traffic_test_params:
+        for param in traffic_test_params:
+            cmd = cmd + ' ' + str(param)
+    print('Execute traffic test %s' % cmd)
+    '''
+    print('Warming up for route rules to take place')
+    try:
+        output = subprocess.check_output(cmd, shell=True)
+    except subprocess.CalledProcessError, e:
+        print('%s returns error %s' % e.output)
+    print(output)
+    print('Running recorded traffic test...')
+    '''
+    time.sleep(30)
+    tracing.use_redis = True
+    test_id = uuid.uuid4()
+    rr.set_route_rules(test_id)
+    tracing.setTest(test_id)
+    try:
+        output = subprocess.check_output(cmd, shell=True)
+    except subprocess.CalledProcessError, e:
+        print('non zero return value on traffic script: %s, ignoring...' % e.output)
+    print(output)
+    time.sleep(30)
+    traces = tracing.getTraces(test_svc, 0)
+    tracing.outTraces(traces)
+
+    time.sleep(3)
+    print('Validating route rules...')
+    validate_wrr = ValidateWRR(test_id, redis_ip=redis_ip)
+    ret, errors = validate_wrr.validate(test_svc)
+
+    # TODO(s3wong): for now, route rules failure seems more like a warning
+    if ret:
+        print('Route rules for service %s validated' % test_svc)
+    else:
+        print('Route rules for service %s validation failed' % test_svc)
+        for err in errors:
+            print err
+
+    success_factors = test_params.get('success')
+    if success_factors:
+        criteria = success_factors.get('criteria')
+        success_check = True
+        for criterion in criteria:
+            c_type = criterion.get('type')
+            if c_type == 'performance':
+                condition = int(criterion.get('condition'))
+                ret_dict = validate_perf(tracing, test_id, test_svc,
+                        dep_a_name, dep_b_name)
+                # print performance data
+                _format_perf_data(ret_dict.get(dep_a_name), dep_a_name, test_svc)
+                print('\n')
+                _format_perf_data(ret_dict.get(dep_b_name), dep_b_name, test_svc)
+                ret = (ret_dict.get(dep_b_name).get('in').get('average') <= \
+                      (ret_dict.get(dep_a_name).get('in').get('average') * condition / 100))
+                if not ret:
+                    print('Performance check failed')
+                    success_check = False
+                    break
+                else:
+                    print('Performance check succeed')
+            '''
+            elif c_type == 'services':
+                srv_list = criterion.get('services')
+                ret = check_services_traverse(tracing, test_id, test_svc,
+                        dep_b_name, srv_list)
+                if not ret:
+                    print('Additional services traversal test failed')
+                    success_check = False
+                    break
+                else:
+                    print('Additional services traversal test succeed')
+            '''
+        if success_check:
+            actions = success_factors.get('action')
+        else:
+            failed = success_factors.get('failed')
+            actions = failed.get('action')
+        for action in actions:
+            action_type = action.get('type')
+            if action_type == 'commit' or action_type == 'rollback':
+                rr.delete_route_rules(a_b_test_rr_yaml, namespace)
+                ret = rr.load_route_rules(POLICY_BASE + action.get('routerule'))
+                if ret:
+                    print('loading route rule %s succeed' % action.get('routerule'))
+
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/clover/test/istio/sdc/clover-server4-delay.yaml b/clover/test/istio/sdc/clover-server4-delay.yaml
new file mode 100644
index 0000000..32a9f6a
--- /dev/null
+++ b/clover/test/istio/sdc/clover-server4-delay.yaml
@@ -0,0 +1,13 @@
+apiVersion: config.istio.io/v1alpha2
+kind: RouteRule
+metadata:
+  name: clover-server4-delay
+  namespace: default
+spec:
+  destination:
+    name: clover-server4
+  httpFault:
+    delay:
+      fixedDelay: 1.000s
+      percent: 100
+  precedence: 2
diff --git a/clover/test/istio/sdc/clover-server5-delay.yaml b/clover/test/istio/sdc/clover-server5-delay.yaml
new file mode 100644
index 0000000..0c8bcd8
--- /dev/null
+++ b/clover/test/istio/sdc/clover-server5-delay.yaml
@@ -0,0 +1,13 @@
+apiVersion: config.istio.io/v1alpha2
+kind: RouteRule
+metadata:
+  name: clover-server5-delay
+  namespace: default
+spec:
+  destination:
+    name: clover-server5
+  httpFault:
+    delay:
+      fixedDelay: 1.000s
+      percent: 100
+  precedence: 2
diff --git a/clover/test/istio/sdc/route-rule-lb-50-v2.yaml b/clover/test/istio/sdc/route-rule-lb-50-v2.yaml
new file mode 100644
index 0000000..529da78
--- /dev/null
+++ b/clover/test/istio/sdc/route-rule-lb-50-v2.yaml
@@ -0,0 +1,15 @@
+apiVersion: config.istio.io/v1alpha2
+kind: RouteRule
+metadata:
+  name: lb-default
+spec:
+  destination:
+    name: http-lb
+  precedence: 1
+  route:
+  - labels:
+      version: v1
+    weight: 50
+  - labels:
+      version: v2
+    weight: 50
diff --git a/clover/test/istio/sdc/route-rule-lb-v1.yaml b/clover/test/istio/sdc/route-rule-lb-v1.yaml
new file mode 100644
index 0000000..a3cac3a
--- /dev/null
+++ b/clover/test/istio/sdc/route-rule-lb-v1.yaml
@@ -0,0 +1,11 @@
+apiVersion: config.istio.io/v1alpha2
+kind: RouteRule
+metadata:
+  name: lb-default
+spec:
+  destination:
+    name: http-lb
+  precedence: 1
+  route:
+  - labels:
+      version: v1
diff --git a/clover/test/istio/sdc/route-rule-lb-v2.yaml b/clover/test/istio/sdc/route-rule-lb-v2.yaml
new file mode 100644
index 0000000..23a3c8d
--- /dev/null
+++ b/clover/test/istio/sdc/route-rule-lb-v2.yaml
@@ -0,0 +1,11 @@
+apiVersion: config.istio.io/v1alpha2
+kind: RouteRule
+metadata:
+  name: lb-default
+spec:
+  destination:
+    name: http-lb
+  precedence: 1
+  route:
+  - labels:
+      version: v2
diff --git a/clover/test/script/lb-test.sh b/clover/test/script/lb-test.sh
new file mode 100755
index 0000000..4d10386
--- /dev/null
+++ b/clover/test/script/lb-test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+for i in `seq 1 20`;
+do
+    #wget http://10.244.0.1:32580/
+    wget http://$1:$2/
+    sleep 1
+done
diff --git a/clover/test/validate_success.py b/clover/test/validate_success.py
new file mode 100644
index 0000000..65cd579
--- /dev/null
+++ b/clover/test/validate_success.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+from clover.orchestration import kube_client
+from clover.tracing.tracing import Tracing
+
+def _get_svc_pods(svc_name, namespace):
+    k8s_client = kube_client.KubeClient()
+    svc = k8s_client.find_svc_by_namespace(svc_name=svc_name,
+                                           namespace=namespace)
+    if not svc:
+        err_msg = 'Failed to locate service %s in %s namespace' \
+                  % (svc_name, namespace)
+        print err_msg
+        return False, [err_msg]
+    pods = k8s_client.find_pod_by_namespace()
+    if not pods:
+        err_msg = 'No pod found in namespace %s' % namespace
+        return False, [err_msg]
+    svc_pods = {}
+    for p,l in pods.items():
+        pod_labels = l.get('labels')
+        svc_selector_dict = svc[service_name].get('selector')
+        for svc_select_key in svc_selector_dict:
+            if svc_select_key in pod_labels:
+                if svc_selector_dict[svc_select_key] == pod_labels[svc_select_key]:
+                    svc_pods[p] = l
+    return svc_pods
+
+def validate_perf(tracing, test_id, svc_name, control_svc, variant_svc):
+    ret_dict = {}
+    ret_dict[control_svc] = {}
+    ret_dict[control_svc]['in'] = {}
+    ret_dict[control_svc]['in']['total'] = 0
+    ret_dict[control_svc]['in']['average'] = 0
+    ret_dict[control_svc]['out'] = {}
+    ret_dict[control_svc]['out']['total'] = 0
+    ret_dict[control_svc]['out']['average'] = 0
+
+    ret_dict[variant_svc] = {}
+    ret_dict[variant_svc]['in'] = {}
+    ret_dict[variant_svc]['in']['total'] = 0
+    ret_dict[variant_svc]['in']['average'] = 0
+    ret_dict[variant_svc]['out'] = {}
+    ret_dict[variant_svc]['out']['total'] = 0
+    ret_dict[variant_svc]['out']['average'] = 0
+
+    req_id_dict = {}
+    def _fill_up_ret_dict(direction, svc, duration, out_svc=None):
+        sum = ret_dict[svc][direction]['average'] * \
+              ret_dict[svc][direction]['total'] + \
+              int(duration)
+        ret_dict[svc][direction]['total'] += 1
+        ret_dict[svc][direction]['average'] = \
+            float(sum) / float(ret_dict[svc][direction]['total'])
+        if direction == 'out' and out_svc:
+            # tracking the out service from svc
+            # TODO(s3wong): this assumes only ONE direction from
+            # service to another service, which may not be true
+            # in essence, the data structure should track (srv, out)
+            # pairs and calculate average that way
+            ret_dict[svc][direction]['out_svc'] = out_svc
+
+
+    def _check_req_id(req_id, svc=None, node_id=None,
+                      duration=None, direction=None,
+                      out_svc=None):
+        if req_id not in req_id_dict:
+            req_id_dict[req_id] = {}
+
+        if svc:
+            req_id_dict[req_id]['svc'] = svc
+        else:
+            req_id_dict[req_id]['node_id'] = node_id
+            req_id_dict[req_id]['duration'] = int(duration)
+            req_id_dict[req_id]['direction'] = direction
+            if direction == 'out' and out_svc:
+                req_id_dict[req_id]['out_svc'] = out_svc
+
+    trace_ids = tracing.getRedisTraceids(test_id)
+    for trace_id in trace_ids:
+        span_ids = tracing.getRedisSpanids(trace_id)
+        for span in span_ids:
+            out_svc = None
+            duration = tracing.getRedisSpanValue(span, trace_id, 'duration')
+            node_id = tracing.getRedisTagsValue(span, trace_id, 'node_id')
+            upstream_cluster = tracing.getRedisTagsValue(span, trace_id, 'upstream_cluster')
+            req_id = tracing.getRedisTagsValue(span, trace_id, 'guid:x-request-id')
+            if upstream_cluster.startswith('in.'):
+                direction = 'in'
+            else:
+                direction = 'out'
+                out_svc = upstream_cluster.split('.')[1]
+            if control_svc in node_id:
+                _fill_up_ret_dict(direction, control_svc, duration, out_svc=out_svc)
+                _check_req_id(req_id, svc=control_svc)
+            elif variant_svc in node_id:
+                _fill_up_ret_dict(direction, variant_svc, duration, out_svc=out_svc)
+                _check_req_id(req_id, svc=variant_svc)
+            else:
+                # client to svc or server from svc as client
+                if out_svc and out_svc == svc_name:
+                    _check_req_id(req_id, node_id=node_id, direction=direction,
+                                  duration=duration, out_svc=out_svc)
+
+    for req_id, svc_dict in req_id_dict.items():
+        node_id = svc_dict.get('node_id')
+        if not node_id:
+            continue
+        pod_name = node_id.split('~')[2]
+        svc = svc_dict.get('svc')
+        if pod_name not in ret_dict.get(svc):
+            ret_dict[svc][pod_name] = {}
+            ret_dict[svc][pod_name]['total'] = 0
+            ret_dict[svc][pod_name]['direction'] = svc_dict.get('direction')
+            if svc_dict.get('out_svc'):
+                ret_dict[svc][pod_name]['out_svc'] = svc_dict.get('out_svc')
+            ret_dict[svc][pod_name]['average'] = 0
+        sum = ret_dict[svc][pod_name]['average'] * \
+              ret_dict[svc][pod_name]['total'] + \
+              svc_dict.get('duration')
+        ret_dict[svc][pod_name]['total'] += 1
+        ret_dict[svc][pod_name]['average'] = \
+            float(sum) / float(ret_dict[svc][pod_name]['total'])
+
+    return ret_dict
diff --git a/clover/test/yaml/fraser_a_b_test.yaml b/clover/test/yaml/fraser_a_b_test.yaml
new file mode 100644
index 0000000..d8fbc0b
--- /dev/null
+++ b/clover/test/yaml/fraser_a_b_test.yaml
@@ -0,0 +1,28 @@
+test-svc: http-lb
+kind: A-B-Testing
+deployment-A: http-lb-v1
+deployment-B:
+  name: http-lb-v2
+  manifest: sdc/lb-v2.yaml
+additional-deployments:
+  - name: clover-server4
+    manifest: sdc/clover-server4.yaml
+  - name: clover-server5
+    manifest: sdc/clover-server5.yaml
+ab-test-rr: sdc/route-rule-lb-50-v2.yaml
+traffic-test:
+  name: lb-test.sh
+  params:
+    - 10.244.0.1
+    - 32580
+success:
+  criteria:
+  - type: performance
+    condition: 120
+  action:
+  - type: commit
+    routerule: sdc/route-rule-lb-v2.yaml
+  failed:
+    action:
+    - type: rollback
+      routerule: sdc/route-rule-lb-v1.yaml
diff --git a/clover/tools/validate_rr.py b/clover/tools/validate_rr.py
index aa1b211..fbda20e 100644
--- a/clover/tools/validate_rr.py
+++ b/clover/tools/validate_rr.py
@@ -14,10 +14,12 @@ from clover.tracing.tracing import Tracing
 
 class ValidateWRR(object):
 
-    def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298'):
+    def __init__(self, test_id, tracing_ip='localhost', tracing_port='31298',
+                 redis_ip='localhost'):
         self._k8s_client = kube_client.KubeClient()
         self._test_id = test_id
-        self._tracing = Tracing(tracing_ip, tracing_port)
+        self._tracing = Tracing(tracing_ip, tracing_port,
+                                redis_ip=redis_ip)
 
     def check_pods_up(self, pod_list, namespace='default'):
         for pod in pod_list: