Clover initial commit for servicemesh/route_rules,

orchestration/kube_client, and tools/clover_validate_rr

Add an 'orchestration' directory. Please note that
'orchestration' does NOT mean Clover does any orchestration ---
similar to how Clover doesn't by itself implement tracing or
logging, orchestration is a directory for code related to Docker
orchestration client --- such as k8s client

kube_client utilizes the Kubernetes python client (a dependency)
to perform tasks against Kubernetes API server. For this commit,
it is only tested for weighted route rule verification, it does
three tasks:
(1) get a list of pods under a namespace --- pod dictionary now
    only contains pod name and label dictionary: used to match
    pod name with the node name in traces from OpenTracing
(2) check to see if a particular pod is up in a particular
    namespace: used to check if Istio pods are running in
    istio-system namespace
(3) check if a container exists in a list of pods under a
    namespace: used to check if application pods have
    istio-proxy container running

route_rule directly invokes istioctl as there isn't any Istio
Python client yet.  Currently it reads and parses routerules
from Istio, and validates if a particular trace result matches
the routerules

Finally, a sample tool clover_validate_rr is provided. This
tool assumes a previous test has been ran (with an id with
both the route-rule-under-test and corresponding traces are
stored --- currently the assumption is tests were ran with
redis-master running on system). The tool can be invoked:

python clover_validate_rr.py -t <test-id> -s <service name>

where test-id is the ID of the test (most likely uuid) and
service name is the name of the service running in the
Kubernetes cluster upon which test traces should be fetched
against

Change-Id: Ic8ab6efc23c71ac4643bee796ef986a86f6fc7dd
Signed-off-by: Stephen Wong <stephen.kf.wong@gmail.com>

1 files changed, 134 insertions, 0 deletions

diff --git a/clover/servicemesh/route_rules.py b/clover/servicemesh/route_rules.py
new file mode 100644
index 0000000..cc2ee0c
--- /dev/null
+++ b/clover/servicemesh/route_rules.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+import os
+import redis
+import subprocess
+import sys
+import yaml
+
+#istioctl='$HOME/istio-0.6.0/bin/istioctl'
+# The assumption is that istioctl is already in the user's path
+ISTIOCTL='istioctl'
+
+def cmd_exists(cmd):
+    return any(
+        os.access(os.path.join(path, cmd), os.X_OK)
+        for path in os.environ["PATH"].split(os.pathsep)
+    )
+
+def load_route_rules(rr_yaml_path):
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return False
+
+    # TODO(s3wong): load yaml and verify it does indeed contain route rule
+    cmd = ISTIOCTL + ' create -f ' + rr_yaml_path
+    output = subprocess.check_output(cmd, shell=True)
+    if not output:
+        print('Route rule creation failed: %s' % output)
+        return False
+    return True
+
+def delete_route_rules(rr_yaml_path, namespace):
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return False
+
+    # TODO(s3wong): load yaml and verify it does indeed contain route rule
+    cmd = ISTIOCTL + ' delete -f ' + rr_yaml_path + ' -n ' + namespace
+    output = subprocess.check_output(cmd, shell=True)
+    if not output or not 'Deleted' in output:
+        print('Route rule deletion failed: %s' % output)
+        return False
+    return True
+
+def get_route_rules():
+    if not cmd_exists(ISTIOCTL):
+        print('%s does not exist in PATH, please export istioctl to PATH' % istioctl)
+        return None
+    cmd = ISTIOCTL + ' get routerules -o yaml'
+    output = subprocess.check_output(cmd, shell=True)
+    if not output:
+        print('No route rule configured')
+        return None
+    docs = []
+    for raw_doc in output.split('\n---'):
+        try:
+            docs.append(yaml.load(raw_doc))
+        except SyntaxError:
+            print('syntax error: %s' % raw_doc)
+    return docs
+
+def parse_route_rules(routerules):
+    ret_list = []
+    if not routerules:
+        print('No routerules')
+        return ret_list
+    for routerule in routerules:
+        if not routerule or routerule == 'None':  continue
+        print('routerule is %s' % routerule)
+        if routerule.get('kind') != 'RouteRule': continue
+        ret_rr_dict = {}
+        spec = routerule.get('spec')
+        if not spec: continue
+        ret_rr_dict['service'] = spec.get('destination').get('name')
+        ret_rr_dict['rules'] = spec.get('route')
+        ret_list.append(ret_rr_dict)
+    return ret_list
+
+def _derive_key_from_test_id(test_id):
+    return 'route-rules-' + str(test_id)
+
+def set_route_rules(test_id):
+    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    key = _derive_key_from_test_id(test_id)
+    rr = get_route_rules()
+    r.set(key, rr)
+
+def fetch_route_rules(test_id):
+    r = redis.StrictRedis(host='localhost', port=6379, db=0)
+    key = _derive_key_from_test_id(test_id)
+    rr = r.get(key)
+    return yaml.load(rr)
+
+'''
+    The format of result_dict is expected to be:
+    {
+        'service':  <service name>,
+        <version string 1>: <integer representation of version string 1 occurrances during test>,
+        <version string 2>: <integer representation of version string 2 occurrances during test>,
+        ...
+    }
+'''
+def validate_weighted_route_rules(result_dict, test_id=None):
+    print('validate_weighted_route_rules: test id %s' % test_id)
+    svc_name = result_dict.get('service')
+    if not test_id:
+        rr_list = parse_route_rules(get_route_rules())
+    else:
+        rr_list = parse_route_rules(fetch_route_rules(test_id))
+    errors = []
+    ret = True
+    for rr in rr_list:
+        route_rules = rr.get('rules')
+        if not route_rules:
+            break
+        for rule in route_rules:
+            version = rule.get('labels').get('version')
+            weight = rule.get('weight')
+            if not weight: weight = 1
+            if abs(weight - result_dict[version]) > 10:
+                err = 'svc %s version %s expected to get %d, but got %d' % (svc_name, version, weight, result_dict[version])
+                ret = False
+            else:
+                err = 'svc %s version %s expected to get %d, got %d. Validation succeeded' % (svc_name, version, weight, result_dict[version])
+            errors.append(err)
+    return ret, errors
+
+