24 files changed, 929 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..988165b
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,37 @@
+*.DS_Store
+*.log
+*.pyc
+.vimrc
+.ropeproject
+.settings/
+.eggs/
+*.orig
+*.BACKUP.*
+*.BASE.*
+*.LOCAL.*
+*.REMOTE.*
+*.egg
+*.egg-info
+build
+htmlcov
+.agignore
+.coverage
+*.retry
+Session*.vim
+.tags*
+.coverage.*
+*~
+setuptools*zip
+dist/
+.testrepository/
+cover/
+.*.sw?
+/docs_build/
+/docs_output/
+/docs/apidocs/yardstick*.rst
+#PyCharm IDE project configuration files
+.idea/
+# tox virtualenvs
+.tox/
+# work env
+work/
diff --git a/ci/deploy.sh b/ci/deploy.sh
index 6280fd2..d1f2961 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -9,4 +9,36 @@
 #
 set -ex
 
+CLOVER_BASE_DIR=`cd ${BASH_SOURCE[0]%/*}/..;pwd`
+CLOVER_WORK_DIR=$CLOVER_BASE_DIR/work
+MASTER_NODE_NAME='master'
+SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+K8S_ISTIO_DEPLOY_TIMEOUT=3600
+
+mkdir -p $CLOVER_WORK_DIR
+cd $CLOVER_WORK_DIR
+
+# Fetch container4nfv source code
+if [ -d container4nfv ]; then
+    rm -rf container4nfv
+fi
+git clone https://git.opnfv.org/container4nfv/
+cd container4nfv
+
+# Create kubernetes + istio env
+timeout $K8S_ISTIO_DEPLOY_TIMEOUT ./src/vagrant/kubeadm_istio/deploy.sh
+
+# Fetch kube-master node info
+cd src/vagrant/kubeadm_istio
+MASTER_NODE_HOST=$(vagrant ssh-config $MASTER_NODE_NAME | awk '/HostName /{print $2}')
+MASTER_NODE_USER=$(vagrant ssh-config $MASTER_NODE_NAME | awk '/User /{print $2}')
+MASTER_NODE_KEY=$(vagrant ssh-config $MASTER_NODE_NAME | awk '/IdentityFile /{print $2}')
+
+# Push clover source code to kube-master node
+ssh $SSH_OPTIONS -i $MASTER_NODE_KEY ${MASTER_NODE_USER}@${MASTER_NODE_HOST} rm -rf clover
+scp $SSH_OPTIONS -i $MASTER_NODE_KEY -r $CLOVER_BASE_DIR ${MASTER_NODE_USER}@${MASTER_NODE_HOST}:clover
+
+# Run test
+ssh $SSH_OPTIONS -i $MASTER_NODE_KEY ${MASTER_NODE_USER}@${MASTER_NODE_HOST} ./clover/ci/test.sh
+
 echo "Clover deploy complete!"
diff --git a/ci/test.sh b/ci/test.sh
new file mode 100755
index 0000000..4e0ccc5
--- /dev/null
+++ b/ci/test.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+set -ex
+
+echo "Clover test complete!"
diff --git a/docs/.keep b/clover/__init__.py
index e69de29..e69de29 100644
--- a/docs/.keep
+++ b/clover/__init__.py
diff --git a/clover/functest/__init__.py b/clover/functest/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/clover/functest/__init__.py
diff --git a/clover/functest/clover_k8s.py b/clover/functest/clover_k8s.py
new file mode 100644
index 0000000..654c8e5
--- /dev/null
+++ b/clover/functest/clover_k8s.py
@@ -0,0 +1,26 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import functest_kubernetes.k8stest as k8stest
+
+
+class K8sCloverTest(k8stest.K8sTesting):
+    """Clover test suite"""
+
+    def __init__(self, **kwargs):
+        if "case_name" not in kwargs:
+            kwargs.get("case_name", 'clover_k8s')
+        super(K8sCloverTest, self).__init__(**kwargs)
+        self.check_envs()
+
+    def run_kubetest(self):
+        success = True
+        if success:
+            self.result = 100
+        elif failure:
+            self.result = 0
+
diff --git a/clover/monitoring/monitoring.py b/clover/monitoring/monitoring.py
new file mode 100644
index 0000000..9726fd1
--- /dev/null
+++ b/clover/monitoring/monitoring.py
@@ -0,0 +1,140 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from datetime import timedelta
+import pprint
+import requests
+import time
+
+PROMETHEUS_URL = "http://127.0.0.1:9090"
+
+
+class Monitoring(object):
+    PROMETHEUS_HEALTH_UP = "up"
+    PROMETHEUS_ISTIO_TARGETS = {"envoy",
+        "istio-mesh",
+        "kubernetes-apiservers",
+        "kubernetes-cadvisor",
+        "kubernetes-nodes",
+        "kubernetes-service-endpoints",
+        "mixer",
+        "pilot"}
+    PROMETHEUS_API_TARGETS = "/api/v1/targets"
+    PROMETHEUS_API_QUERY = "/api/v1/query"
+    PROMETHEUS_API_QUERY_RANGE = "/api/v1/query_range"
+
+    def __init__(self, host):
+        self.host = host
+
+    def get_targets(self):
+        try:
+            # Reference api: https://prometheus.io/docs/prometheus/latest/querying/api/#targets
+            response = requests.get('%s%s' % (self.host, Monitoring.PROMETHEUS_API_TARGETS))
+            if response.status_code != 200:
+                print("ERROR: get targets status code: %r" % response.status_code)
+                return False
+        except Exception as e:
+            print("ERROR: Cannot connect to prometheus\n%s" % e)
+            return False
+
+        return response.json()
+
+    def is_targets_healthy(self):
+        targets = set()
+
+        raw_targets = self.get_targets()
+        if raw_targets == False:
+            return False
+
+        for target in raw_targets["data"]["activeTargets"]:
+            if target["health"] != Monitoring.PROMETHEUS_HEALTH_UP:
+                print("ERROR: target unhealth job: %s, health: %s" % \
+                    (target["labels"]["job"], target["health"]))
+                return False
+            targets.add(target["labels"]["job"])
+
+        diff = Monitoring.PROMETHEUS_ISTIO_TARGETS - targets
+        if len(diff):
+            print("ERROR: targets %r not found!" % diff)
+            return False
+
+        return True
+
+    # Reference links:
+    #     - https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
+    #     - https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries
+    #     - https://github.com/prometheus/prombench/blob/master/apps/load-generator/main.py
+    def query(self, query_params):
+        try:
+            start = time.time()
+
+            query_type = query_params.get("type", "instant")
+            params = {"query": query_params["query"]}
+            if query_type == "instant":
+                url = "%s%s" % (self.host, Monitoring.PROMETHEUS_API_QUERY)
+            elif query_type == "range":
+                url = "%s%s" % (self.host, Monitoring.PROMETHEUS_API_QUERY_RANGE)
+                params["start"] = start - duration_seconds(query_params.get("start", "0h"))
+                params["end"] = start - duration_seconds(query_params.get("end", "0h"))
+                params["step"] = query_params.get("step", "15s")
+            else:
+                print("ERROR: invalidate query type")
+                return
+
+            resp = requests.get(url, params)
+            dur = time.time() - start
+
+            print("query %s %s, status=%s, size=%d, dur=%.3f" % \
+                (self.host, query_params["query"], resp.status_code, len(resp.text), dur))
+            pp = pprint.PrettyPrinter(indent=2)
+            pp.pprint(resp.json())
+
+        except Exception as e:
+            print("ERROR: Could not query prometheus instance %s. \n %s" % (url, e))
+
+
+def duration_seconds(s):
+    num = int(s[:-1])
+
+    if s.endswith('s'):
+        return timedelta(seconds=num).total_seconds()
+    elif s.endswith('m'):
+        return timedelta(minutes=num).total_seconds()
+    elif s.endswith('h'):
+        return timedelta(hours=num).total_seconds()
+
+    raise "ERROR: unknown duration %s" % s
+
+
+def main():
+    m = Monitoring(PROMETHEUS_URL)
+    if not m.is_targets_healthy():
+        print("ERROR: Prometheus targets is unhealthy!")
+    else:
+        print("Prometheus targets are all healthy!")
+
+    print "\n### query instant"
+    query_params = {
+        "type": "instant",
+        "query": "istio_double_request_count{destination='details.default.svc.cluster.local'}"
+    }
+    m.query(query_params)
+
+    print "\n### query range"
+    query_range_param = {
+        "type": "range",
+        "query": "istio_double_request_count{destination='details.default.svc.cluster.local'}",
+        "start": "5m",
+        "end": "3m",
+        "step": "30s"
+     }
+    m.query(query_range_param)
+
+
+if __name__ == '__main__':
+    main()
+
diff --git a/clover/monitoring/validate.py b/clover/monitoring/validate.py
new file mode 100644
index 0000000..fafe5df
--- /dev/null
+++ b/clover/monitoring/validate.py
@@ -0,0 +1,70 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from monitoring import Monitoring
+from kubernetes import client, config
+
+PROMETHEUS_URL = "http://127.0.0.1:9090"
+PROMETHEUS_DEPLOYMENT = "prometheus"
+PROMETHEUS_LABELS = "app=prometheus"
+ISTIO_NAMESPACE = "istio-system"
+
+
+def validateDeploy():
+    config.load_kube_config()
+    appsv1 = client.AppsV1Api()
+    corev1 = client.CoreV1Api()
+    find_flag = False
+    prom_pod_name = None
+
+    # check prometheus deploytment
+    ret = appsv1.list_deployment_for_all_namespaces(watch=False)
+    for i in ret.items:
+        if PROMETHEUS_DEPLOYMENT == i.metadata.name and \
+           ISTIO_NAMESPACE == i.metadata.namespace:
+           find_flag = True
+           break
+    if find_flag == False:
+        print("ERROR: Deployment: {} doesn't present in {} namespace".format(
+                        PROMETHEUS_DEPLOYMENT, ISTIO_NAMESPACE))
+        return False
+
+    # find prometheus pod by label selector
+    ret = corev1.list_namespaced_pod(ISTIO_NAMESPACE, label_selector=PROMETHEUS_LABELS)
+    for i in ret.items:
+        prom_pod_name = i.metadata.name
+    if prom_pod_name == None:
+        print("ERROR: prometheus pod not found")
+        return False
+
+    # check prometheus pod status
+    ret = corev1.read_namespaced_pod_status(prom_pod_name, ISTIO_NAMESPACE)
+    if ret.status.phase != "Running":
+        print("ERROR: prometheus pod %s is under %s state" % (prom_pod_name, ret.status.phase))
+        return False
+
+    return True
+
+
+def validateService():
+    m = Monitoring(PROMETHEUS_URL)
+
+    return m.is_targets_healthy()
+
+
+def main():
+    if validateDeploy() and validateService():
+        print"Prometheus monitoring validation has passed"
+        return True
+    else:
+        print"ERROR: Prometheus monitoring validation has failed"
+        return False
+
+
+if __name__ == '__main__':
+    main()
+
diff --git a/clover/tracing/tracing.py b/clover/tracing/tracing.py
new file mode 100644
index 0000000..16b952c
--- /dev/null
+++ b/clover/tracing/tracing.py
@@ -0,0 +1,201 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import requests
+import time
+import redis
+
+TRACING_IP = "localhost"
+TRACING_PORT = "30888"
+
+
+class Tracing:
+
+    def __init__(
+      self, tracing_ip, tracing_port, redis_ip='localhost', use_redis=True):
+        self.tracing_ip = tracing_ip
+        self.tracing_port = tracing_port
+        self.testid = '0'
+        self.test_start_time = 0
+        self.use_redis = use_redis
+        if use_redis:
+            try:
+                self.r = redis.StrictRedis(host=redis_ip, port=6379, db=0)
+            except Exception:
+                print("Failed to connect to redis")
+
+    def setRedisSet(self, rkey, rvalue):
+        if self.use_redis:
+            self.r.sadd(rkey, rvalue)
+
+    def setRedisList(self, rkey, rvalue):
+        if self.use_redis:
+            self.r.lpush(rkey, rvalue)
+
+    def setRedisHash(self, rkey, rvalue):
+        if self.use_redis:
+            self.r.hmset(rkey, rvalue)
+
+    def getRedisTestid(self, index):
+        testid = self.r.lrange("testids", index, index)
+        return testid[0]
+
+    def getRedisTraceids(self, testid):
+        rkey = "traceids:" + str(testid)
+        traceids = self.r.smembers(rkey)
+        return traceids
+
+    def getRedisSpanids(self, traceid):
+        rkey = "spanids:" + str(traceid)
+        spanids = self.r.smembers(rkey)
+        return spanids
+
+    def getRedisSpan(self, spanid, traceid):
+        rkey = "spans:" + str(traceid) + ':' + str(spanid)
+        span = self.r.hgetall(rkey)
+        return span
+
+    def getRedisSpanValue(self, spanid, traceid, span_key):
+        rkey = "spans:" + str(traceid) + ':' + str(spanid)
+        span_value = self.r.hget(rkey, span_key)
+        return span_value
+
+    def getRedisTags(self, spanid, traceid):
+        rkey = "tags:" + str(spanid) + ':' + str(traceid)
+        tags = self.r.hgetall(rkey)
+        return tags
+
+    def getRedisTagsValue(self, spanid, traceid, tag_key):
+        rkey = "tags:" + str(spanid) + ':' + str(traceid)
+        tag_value = self.r.hget(rkey, tag_key)
+        return tag_value
+
+    def getRedisTestAll(self, testid):
+        traceids = self.getRedisTraceids(testid)
+        for trace in traceids:
+            spanids = self.getRedisSpanids(trace)
+            for span in spanids:
+                # print(self.getRedisSpan(span, trace))
+                print(self.getRedisSpanValue(span, trace, 'duration'))
+                # print(self.getRedisTags(span, trace))
+                print(self.getRedisTagsValue(span, trace, 'node_id'))
+
+    def setTest(self, testid):
+        self.testid = testid
+        self.setRedisList("testids", testid)
+        self.test_start_time = int(time.time())
+
+    def getServices(self):
+        req_url = 'http://' + self.tracing_ip + ':' + self.tracing_port + \
+                                                        '/api/services'
+        try:
+            response = requests.get(req_url)
+            if response.status_code != 200:
+                print("ERROR: Cannot connect to tracing: {}".format(
+                                        response.status_code))
+                return False
+        except Exception as e:
+            print("ERROR: Cannot connect to tracing")
+            print(e)
+            return False
+
+        data = response.json()
+        services = data['data']
+        return services
+
+    def getTraces(self, service, time_back=3600, limit='1000'):
+        ref_time = int(time.time())
+        pad_time = '757000'
+        end_time = 'end=' + str(ref_time) + pad_time + '&'
+        if time_back == 0:
+            delta = self.test_start_time
+        else:
+            delta = ref_time - time_back
+        start_time = 'start=' + str(delta) + pad_time
+        limit = 'limit=' + limit + '&'
+        loopback = 'loopback=1h&'
+        max_dur = 'maxDuration&'
+        min_dur = 'minDuration&'
+        service = 'service=' + service + '&'
+        url_prefix = 'http://' + self.tracing_ip + ':' + self.tracing_port + \
+            '/api/traces?'
+        req_url = url_prefix + end_time + limit + loopback + max_dur + \
+            min_dur + service + start_time
+
+        try:
+            response = requests.get(req_url)
+            if response.status_code != 200:
+                print("ERROR: Cannot connect to tracing: {}".format(
+                                        response.status_code))
+                return False
+        except Exception as e:
+            print("ERROR: Cannot connect to tracing")
+            print(e)
+            return False
+
+        traces = response.json()
+        return traces
+
+    def numTraces(self, trace):
+        num_traces = len(trace['data'])
+        return str(num_traces)
+
+    def outProcesses(self, trace):
+        processes = []
+        if trace['data']:
+            first_trace = trace['data'][0]
+            for process in first_trace['processes']:
+                processes.append(process)
+            print(processes)
+        return processes
+
+    def outTraces(self, trace):
+        for traces in trace['data']:
+            print("TraceID: {}".format(traces['traceID']))
+            self.setRedisSet(
+              "traceids:{}".format(str(self.testid)), traces['traceID'])
+            for spans in traces['spans']:
+                    print("SpanID: {}".format(spans['spanID']))
+                    self.setRedisSet(
+                       "spanids:{}".format(traces['traceID']), spans['spanID'])
+                    print("Duration: {} usec".format(spans['duration']))
+                    span = {}
+                    span['spanID'] = spans['spanID']
+                    span['duration'] = spans['duration']
+                    span['startTime'] = spans['startTime']
+                    span['operationName'] = spans['operationName']
+                    # print("Tags:\n {} \n".format(spans['tags']))
+                    self.setRedisHash(
+                        "spans:{}:{}".format(
+                           traces['traceID'], spans['spanID']), span)
+                    tag = {}
+                    for tags in spans['tags']:
+                        print("Tag key: {}, value: {}".format(
+                                tags['key'], tags['value']))
+                        tag[tags['key']] = tags['value']
+                    self.setRedisHash("tags:{}:{}".format(
+                                spans['spanID'], traces['traceID']), tag)
+
+    def monitorTraces(self, sample_interval, service='istio-ingress'):
+        loop = True
+        while loop:
+            try:
+                t = self.getTraces(service, 10)
+                num_traces = self.numTraces(t)
+                print("Number of traces: " + num_traces)
+                self.outTraces(t)
+                time.sleep(sample_interval)
+            except KeyboardInterrupt:
+                print("Test Start: {}".format(self.test_start_time))
+                loop = False
+
+    def main(self):
+        self.monitorTraces(1)
+
+
+if __name__ == '__main__':
+    Tracing(TRACING_IP, TRACING_PORT).main()
diff --git a/clover/tracing/tracing_sample.py b/clover/tracing/tracing_sample.py
new file mode 100644
index 0000000..f0234bf
--- /dev/null
+++ b/clover/tracing/tracing_sample.py
@@ -0,0 +1,47 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+import uuid
+import time
+from tracing import Tracing
+
+t = Tracing('localhost', '30888')
+
+# Get toplevel services stored in tracing
+services = t.getServices()
+print(services)
+
+# Get traces from the last hour for istio-ingress service
+service = 'istio-ingress'
+traces = t.getTraces(service, 3600)
+# Get process names for first trace service
+t.outProcesses(traces)
+
+# Turn off redis tracing store and output basic trace info
+t.use_redis = False
+t.outTraces(traces)
+
+# Setup basic test and store in redis
+t.use_redis = True
+t.setTest(uuid.uuid4())
+time.sleep(20)
+# Get all traces from test start time when time_back=0
+traces = t.getTraces(service, 0)
+# Store traces in redis
+t.outTraces(traces)
+
+# Get test id for some number of tests back
+testid = t.getRedisTestid('0')
+print(testid)
+traceids = t.getRedisTraceids(testid)
+print(traceids)
+
+# Print out span and tag info for all traces in test
+# Will continue to consider what to extract from hashes for e2e validation
+t.getRedisTestAll(testid)
+
+# t.monitorTraces(1)
diff --git a/clover/tracing/validate.py b/clover/tracing/validate.py
new file mode 100644
index 0000000..eed6f9a
--- /dev/null
+++ b/clover/tracing/validate.py
@@ -0,0 +1,66 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from tracing import Tracing
+from kubernetes import client, config
+
+
+JAEGER_IP = "localhost"
+# JAEGER_IP = "1.1.1.1"
+JAEGER_PORT = "30888"
+JAEGER_DEPLOYMENT = "jaeger-deployment"
+ISTIO_NAMESPACE = "istio-system"
+ISTIO_SERVICES = ["istio-ingress", "istio-mixer"]
+
+
+def validateDeploy():
+    config.load_kube_config()
+    v1 = client.AppsV1Api()
+
+    deployments = []
+    namespaces = []
+    validate = False
+    ret = v1.list_deployment_for_all_namespaces(watch=False)
+    for i in ret.items:
+        deployments.append(i.metadata.name)
+        namespaces.append(i.metadata.namespace)
+    if JAEGER_DEPLOYMENT in deployments:
+        d_index = deployments.index(JAEGER_DEPLOYMENT)
+        if ISTIO_NAMESPACE in namespaces[d_index]:
+            print("Deployment: {} present in {} namespace".format(
+                          JAEGER_DEPLOYMENT, ISTIO_NAMESPACE))
+            validate = True
+    return validate
+
+# Services in Jaeger will only be present when traffic passes through Istio
+# Requires a deployment in Istio service mesh with some traffic targeting nodes
+def validateServices():
+    t = Tracing(JAEGER_IP, JAEGER_PORT)
+    services = t.getServices()
+    validate = True
+    if services:
+        for s in ISTIO_SERVICES:
+            if s in services:
+                print("Service in tracing: {} present".format(s))
+            else:
+                validate = False
+    else:
+        validate = False
+    return validate
+
+
+def main():
+    if validateDeploy() and validateServices():
+        print"Jaeger tracing validation has passed"
+        return True
+    else:
+        print"Jaeger tracing validation has failed"
+        return False
+
+
+if __name__ == '__main__':
+    main()
diff --git a/docs/development/design/.gitkeep b/docs/development/design/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/development/design/.gitkeep
diff --git a/docs/development/overview/.gitkeep b/docs/development/overview/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/development/overview/.gitkeep
diff --git a/docs/development/requirements/.gitkeep b/docs/development/requirements/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/development/requirements/.gitkeep
diff --git a/docs/monitoring.rst b/docs/monitoring.rst
new file mode 100644
index 0000000..44b01e3
--- /dev/null
+++ b/docs/monitoring.rst
@@ -0,0 +1,31 @@
+##########
+Monitoring
+##########
+
+************
+Installation
+************
+
+Currently, we use the Istio build-in prometheus addon to install prometheus::
+
+    cd <istio-release-path>
+    kubectl apply -f install/kubernetes/addons/prometheus.yaml
+
+********
+Validate
+********
+
+Setup port-forwarding for prometheus by executing the following command::
+
+    kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=prometheus -o jsonpath='{.items[0].metadata.name}') 9090:9090 &
+
+Run the scripts in ``clover/monitoring`` validates prometheus installation::
+
+    python clover/monitoring/validate.py
+
+It validates the installation with the following criterias
+
+#. [DONE] prometheus pod is in Running state
+#. [DONE] prometheus is conneted to monitoring targets
+#. [TODO] test collecting telemetry data from istio
+#. [TODO] TBD
diff --git a/docs/release/configguide/.gitkeep b/docs/release/configguide/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/release/configguide/.gitkeep
diff --git a/docs/release/release-notes/.gitkeep b/docs/release/release-notes/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/release/release-notes/.gitkeep
diff --git a/docs/release/release-notes/Fraser-release-notes.rst b/docs/release/release-notes/Fraser-release-notes.rst
new file mode 100644
index 0000000..3e864fb
--- /dev/null
+++ b/docs/release/release-notes/Fraser-release-notes.rst
@@ -0,0 +1,100 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. SPDX-License-Identifier CC-BY-4.0
+.. (c) optionally add copywriters name
+
+
+This document provides the release notes for Fraser of OPNFV Clover.
+
+.. contents::
+   :depth: 3
+   :local:
+
+
+Version history
+---------------
+
++--------------------+--------------------+--------------------+--------------------+
+| **Date**           | **Ver.**           | **Author**         | **Comment**        |
+|                    |                    |                    |                    |
++--------------------+--------------------+--------------------+--------------------+
+| 2018-03-14         | Fraser 1.0         | Stephen Wong       | First draft        |
+|                    |                    |                    |                    |
++--------------------+--------------------+--------------------+--------------------+
+
+Important notes
+===============
+
+The OPNFV Clover project for Fraser can ONLY be run on Kubernetes version 1.9.3 or
+above
+
+Summary
+=======
+
+Clover provides tools to help run cloud native virtual network functions. These
+tools include service-mesh and associated policy-based-routing config (via
+Istio), logging (via fluentd), monitoring (via Prometheus), and tracing (via
+OpenTracing and Jaeger).
+
+Release Data
+============
+
++--------------------------------------+--------------------------------------+
+| **Project**                          | Clover                               |
+|                                      |                                      |
++--------------------------------------+--------------------------------------+
+| **Repo/commit-ID**                   |                                      |
+|                                      |                                      |
++--------------------------------------+--------------------------------------+
+| **Release designation**              | Fraser                               |
+|                                      |                                      |
++--------------------------------------+--------------------------------------+
+| **Release date**                     | 2018-04-xx                           |
+|                                      |                                      |
++--------------------------------------+--------------------------------------+
+| **Purpose of the delivery**          | OPNFV Fraser release                 |
+|                                      |                                      |
++--------------------------------------+--------------------------------------+
+
+Version change
+^^^^^^^^^^^^^^^^
+
+Module version changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Fraser marks the first release of OPNFV Clover
+
+Document version changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Fraser marks the first release of OPNFV Clover
+
+Reason for version
+^^^^^^^^^^^^^^^^^^^^
+
+Feature additions
+~~~~~~~~~~~~~~~~~~~~~~~
+<None> (no backlog)
+
+Bug corrections
+~~~~~~~~~~~~~~~~~~~~~
+<None>
+
+Known Limitations, Issues and Workarounds
+=========================================
+
+System Limitations
+^^^^^^^^^^^^^^^^^^^^
+TBD
+
+Known issues
+^^^^^^^^^^^^^^^
+TBD
+
+Workarounds
+^^^^^^^^^^^^^^^^^
+
+Test Result
+===========
+
+
+References
+==========
diff --git a/docs/release/userguide/.gitkeep b/docs/release/userguide/.gitkeep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/docs/release/userguide/.gitkeep
diff --git a/docs/release/userguide/Fraser-userguide.rst b/docs/release/userguide/Fraser-userguide.rst
new file mode 100644
index 0000000..243c4e1
--- /dev/null
+++ b/docs/release/userguide/Fraser-userguide.rst
@@ -0,0 +1,81 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International License.
+.. http://creativecommons.org/licenses/by/4.0
+.. SPDX-License-Identifier CC-BY-4.0
+.. (c) optionally add copywriters name
+
+
+================================================================
+Clover User Guide (Fraser Release)
+================================================================
+
+This document provides the user guide for Fraser release of Clover.
+
+.. contents::
+   :depth: 3
+   :local:
+
+
+Description
+===========
+
+Project Clover was established to investigate best practice to implement,
+build, deploy, and operate virtual network functions as cloud native
+applications. "Cloud native" has a ever evolving and expanding definition,
+and in Clover, the focus is effectively running and operating VNFs built
+in a micro-service design pattern running on Docker containers and
+orchestrated by Kubernetes.
+
+The strength of cloud native applications is their operablity and
+scalability. Essential to achieve these qualities is the use of service
+mesh. As such, in Fraser release, Clover's emphasis is on demonstrating
+running a sample micro-service composed VNF on Istio, the service mesh
+platform of Clover's choice in Fraser, and how to maximize visibility
+of this sample running in a service mesh.
+
+What is in Fraser?
+==================
+
+ * a sample micro-service composed VNF
+
+ * logging module: fluentd and elasticsearch Kubernetes manifests,
+   installation validation, log data correlation in datastore
+
+ * tracing module: jaeger Kubernetes manifest, installation validation,
+   jaegar tracing query tools, trace data correlation in datastore
+
+ * monitoring module: prometheus Kubernetes manifest, installation
+   validation, prometheous query tools for Istio related metrics,
+   metrics correlation in datastore
+
+ * Istio route-rules and circuit breaking sample yaml and validation
+   tools
+
+ * Test scripts
+
+ * Reference for a demo shown during ONS
+
+Usage
+=====
+
+ * each modules (service mesh, logging, tracing, monitoring) are Python
+   modules with their own set of library calls / API exposed. The descriptions
+   of these library calls are under doc/developer (TBD)
+
+ * tools directory contains Python tools for generic use
+   python clover_validate_route_rules.py -s <service name> -n <number of tests>
+   [more TBD]
+
+ * an example scenario:
+   - version 2 (v2) of a micro-service component is deployed
+   - Istio route rule is applied to send 50% traffic to v2
+   - Clover tool validates traffic conformance with route rules
+   - user specify via yaml the "success" expectation of v2 (latency,
+     performance, session loss...etc)
+   - Clover tool validates sessions conformance with user defined expectations
+   - The "commit" action is invoked to move 100% traffic to v2
+   - Clover tool validates traffic conformance with route rules
+   - A fault is injected for the path to the extra service of v2 which adds
+     a one second delay onto the path
+   - The same A-B testing script is invoked, this time, performance
+     test now fails
+   - The "rollback" action is invoked to move 100% traffic back to v1
diff --git a/docs/tracing.rst b/docs/tracing.rst
new file mode 100644
index 0000000..79d686c
--- /dev/null
+++ b/docs/tracing.rst
@@ -0,0 +1,44 @@
+#######
+Tracing
+#######
+
+************
+Installation
+************
+
+Currently, we use the Jaeger tracing all-in-one Kubernetes template for development and testing,
+which uses in-memory storage. It can be deployed to the istio-system namespace with the
+following command::
+
+    kubectl apply -n istio-system -f https://raw.githubusercontent.com/jaegertracing/jaeger-kubernetes/master/all-in-one/jaeger-all-in-one-template.yml
+
+The standard Jaeger REST port is at 16686. To make this service available outside of the
+Kubernetes cluster, use the following command::
+
+    kubectl expose -n istio-system deployment jaeger-deployment --port=16686 --type=NodePort
+
+Kubernetes will expose the Jaeger service on another port, which can be found with::
+
+    kubectl get svc -n istio-system
+
+An example listing from the command above is shown below where the Jaeger service is exposed
+externally on port 30888::
+
+    istio-system   jaeger-deployment   NodePort  10.104.113.94  <none> 16686:30888/TCP
+
+Jaeger will be accessible using the host IP of the Kubernetes cluster and port provided.
+
+********
+Validate
+********
+
+The script in ``clover/tracing`` validates Jaeger installation::
+
+    python clover/tracing/validate.py
+
+It validates the installation with the following criteria:
+
+#. Existence of Jaeger all-in-one deployment using Kubernetes
+#. Jaeger service is accessible using IP address and port configured in installation steps
+#. Jaeger can retrieve default service listing for default Istio components
+#. TBD - consider installation of production setup with cassandra or elastic search
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..ac3fdd2
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,5 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+pbr!=2.1.0,>=2.0.0 # Apache-2.0
+xtesting # Apache-2.0
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..8453c4e
--- /dev/null
+++ b/setup.cfg
@@ -0,0 +1,8 @@
+[metadata]
+name = clover
+version = 2018.3.28
+home-page = https://wiki.opnfv.org/display/PROJ/Clover
+
+[files]
+packages = clover
+
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..566d844
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,29 @@
+# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
+import setuptools
+
+# In python < 2.7.4, a lazy loading of package `pbr` will break
+# setuptools if some other modules registered functions in `atexit`.
+# solution from: http://bugs.python.org/issue15881#msg170215
+try:
+    import multiprocessing  # noqa
+except ImportError:
+    pass
+
+setuptools.setup(
+    setup_requires=['pbr>=2.0.0'],
+    pbr=True)