Add visibility API in controller and CLI

- Modify get visibility to retrieve config and stats
- Add visibility REST API in controller to clear, set, and
get from redis
- Add example yaml to set visibility (service list by name,
metric suffixes/prefixes, and custom metrics) from CLI
- Modify example yaml to start visibility (collector) for
Istio 1.0 from CLI

Change-Id: I43304ff6d70bb4b817b345b9c383ce3621fb06c7
Signed-off-by: earrage <eddie.arrage@huawei.com>

9 files changed, 231 insertions, 37 deletions

diff --git a/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go b/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go
index 9468714..2ad43f1 100644
--- a/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go
+++ b/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go
@@ -28,7 +28,7 @@ func init() {
 }
 
 func clearCollector() {
-    url := controllerIP + "/collector/truncate"
+    url := controllerIP + "/visibility/clear"
 
     resp, err := resty.R().
     Get(url)
diff --git a/clover/cloverctl/src/cloverctl/cmd/get_visibility.go b/clover/cloverctl/src/cloverctl/cmd/get_visibility.go
index d987412..820b25a 100644
--- a/clover/cloverctl/src/cloverctl/cmd/get_visibility.go
+++ b/clover/cloverctl/src/cloverctl/cmd/get_visibility.go
@@ -13,23 +13,38 @@ import (
     "github.com/spf13/cobra"
 )
 
+var VisibilityStat string
+var VisibilityConfig string
 
-var visibilitystatsCmd = &cobra.Command{
+var visibilitygetCmd = &cobra.Command{
     Use:   "visibility",
-    Short: "Get toplevel visibility stats",
+    Short: "Get visibility config & stats",
     Long: ``,
     Run: func(cmd *cobra.Command, args []string) {
-        statsCollector()
+        getVisibility()
     },
 }
 
 func init() {
-    getCmd.AddCommand(visibilitystatsCmd)
-    //visibilitystartCmd.PersistentFlags().StringVarP(&cloverFile, "f", "f", "", "Input yaml file with test plan params")
+    getCmd.AddCommand(visibilitygetCmd)
+    visibilitygetCmd.PersistentFlags().StringVarP(&VisibilityStat, "stat", "s", "", "Visibility stats type to get")
+    visibilitygetCmd.PersistentFlags().StringVarP(&VisibilityConfig, "conf", "c", "", "Visibility config type to get")
 }
 
-func statsCollector() {
-    url := controllerIP + "/collector/stats"
+func getVisibility() {
+
+    url_prefix := "/visibility/get/"
+    get_data := "all"
+    response_prefix := "Config"
+    if VisibilityStat != "" {
+        url_prefix = "/visibility/get/stats/"
+        get_data =  VisibilityStat
+        response_prefix = "Stat"
+    } else if VisibilityConfig != "" {
+        get_data =  VisibilityConfig
+    }
+
+    url := controllerIP + url_prefix + get_data
 
     resp, err := resty.R().
     SetHeader("Accept", "application/json").
@@ -37,5 +52,5 @@ func statsCollector() {
     if err != nil {
         panic(err.Error())
     }
-    fmt.Printf("\nProxy Response Time: %v\n", resp)
+    fmt.Printf("\n%s %s: %v\n", response_prefix, get_data, resp)
 }
diff --git a/clover/cloverctl/src/cloverctl/cmd/set_visibility.go b/clover/cloverctl/src/cloverctl/cmd/set_visibility.go
new file mode 100644
index 0000000..685b250
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/cmd/set_visibility.go
@@ -0,0 +1,57 @@
+// Copyright (c) Authors of Clover
+//
+// All rights reserved. This program and the accompanying materials
+// are made available under the terms of the Apache License, Version 2.0
+// which accompanies this distribution, and is available at
+// http://www.apache.org/licenses/LICENSE-2.0
+
+package cmd
+
+import (
+    "fmt"
+    "io/ioutil"
+    "gopkg.in/resty.v1"
+    "github.com/spf13/cobra"
+    "github.com/ghodss/yaml"
+)
+
+
+var setvisibilitystatsCmd = &cobra.Command{
+    Use:   "visibility",
+    Short: "Set visibility config for services and metrics to track",
+    Long: ``,
+    Run: func(cmd *cobra.Command, args []string) {
+        setCollector()
+    },
+}
+
+func init() {
+    setCmd.AddCommand(setvisibilitystatsCmd)
+    setvisibilitystatsCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file to set visibility config")
+    setvisibilitystatsCmd.MarkFlagRequired("file")
+
+}
+
+func setCollector() {
+    url := controllerIP + "/visibility/set"
+
+    in, err := ioutil.ReadFile(cloverFile)
+    if err != nil {
+        fmt.Println("Please specify a valid yaml file")
+        return
+    }
+    out_json, err := yaml.YAMLToJSON(in)
+    if err != nil {
+        panic(err.Error())
+    }
+    resp, err := resty.R().
+    SetHeader("Content-Type", "application/json").
+    SetBody(out_json).
+    Post(url)
+    if err != nil {
+        panic(err.Error())
+    }
+    fmt.Printf("\n%v\n", resp)
+
+
+}
diff --git a/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml
new file mode 100644
index 0000000..f88c673
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml
@@ -0,0 +1,13 @@
+services:
+    - name: proxy_access_control
+    - name: clover_server1
+    - name: clover_server2
+    - name: clover_server3
+metric_prefixes:
+    - prefix: envoy_cluster_outbound_9180__
+    - prefix: envoy_cluster_inbound_9180__
+metric_suffixes:
+    - suffix: _default_svc_cluster_local_upstream_rq_2xx
+    - suffix: _default_svc_cluster_local_upstream_cx_active
+custom_metrics:
+    - metric: envoy_tracing_zipkin_spans_sent
diff --git a/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml
new file mode 100644
index 0000000..f27f0c6
--- /dev/null
+++ b/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml
@@ -0,0 +1,5 @@
+sample_interval: "2"
+t_host: tracing.istio-system
+t_port: "80"
+m_port: "9090"
+m_host: prometheus.istio-system
diff --git a/clover/cloverctl/src/cloverctl/yaml/visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/visibility.yaml
deleted file mode 100644
index 20264d2..0000000
--- a/clover/cloverctl/src/cloverctl/yaml/visibility.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-sample_interval: "10"
-#t_host: jaeger-deployment.istio-system
-#t_port: "16686"
-t_host: jaeger-query.istio-system
-t_port: "80"
-m_port: "9090"
-m_host: prometheus.istio-system
diff --git a/clover/controller/control/api/collector.py b/clover/controller/control/api/collector.py
index c82c543..3abcba7 100644
--- a/clover/controller/control/api/collector.py
+++ b/clover/controller/control/api/collector.py
@@ -5,7 +5,7 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 
-from flask import Blueprint, request, jsonify, Response
+from flask import Blueprint, request, Response
 import grpc
 import pickle
 import collector_pb2
@@ -17,13 +17,13 @@ import logging
 collector = Blueprint('collector', __name__)
 
 grpc_port = '50054'
-pod_name = 'clover-collector'
+pod_name = 'clover-collector.clover-system'
 collector_grpc = pod_name + ':' + grpc_port
 channel = grpc.insecure_channel(collector_grpc)
 stub = collector_pb2_grpc.ControllerStub(channel)
-CASSANDRA_HOSTS = pickle.dumps(['cassandra.default'])
+CASSANDRA_HOSTS = pickle.dumps(['cassandra.clover-system'])
 
-HOST_IP = 'redis'
+HOST_IP = 'redis.default'
 
 
 @collector.route("/collector/init")
@@ -62,7 +62,7 @@ def start():
         p = request.json
         if not p:
             sample_interval = '5'
-            t_host = 'jaeger-deployment.istio-system'
+            t_host = 'tracing.istio-system'
             t_port = '16686'
             m_host = 'prometheus.istio-system'
             m_port = '9090'
@@ -103,27 +103,36 @@ def stop():
     return response.message
 
 
-@collector.route("/collector/stats", methods=['GET', 'POST'])
-def stats():
+@collector.route("/collector/set", methods=['GET', 'POST'])
+def set_collector():
     try:
         p = request.json
-        if not p:
-            stat_type = 'toplevel'
-        else:
-            stat_type = p['stat_type']
         r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
-        content = {}
-        content['proxy_rt'] = r.get('proxy_rt')
-        content['trace_count'] = r.get('trace_count')
-        content['span_urls'] = list(r.smembers('span_urls'))
-        response = jsonify(content)
+        del_keys = ['visibility_services', 'metric_prefixes',
+                    'metric_suffixes', 'custom_metrics']
+        for dk in del_keys:
+            r.delete(dk)
+
+        try:
+            for service in p['services']:
+                r.sadd('visibility_services', service['name'])
+        except (KeyError, ValueError) as e:
+            logging.debug(e)
+            return Response(
+                         "Specify at least one service to track", status=400)
+        if p['metric_prefixes'] and p['metric_suffixes']:
+            for prefix in p['metric_prefixes']:
+                r.sadd('metric_prefixes', prefix['prefix'])
+            for suffix in p['metric_suffixes']:
+                r.sadd('metric_suffixes', suffix['suffix'])
+        if p['custom_metrics']:
+            for metric in p['custom_metrics']:
+                r.sadd('custom_metrics', metric['metric'])
+
     except Exception as e:
         logging.debug(e)
-        if e.__class__.__name__ == "_Rendezvous":
-            return Response("Error connecting via gRPC", status=400)
-        else:
-            return Response("Error getting visibility stats", status=400)
-    return response
+        return Response("Error setting visibility config", status=400)
+    return Response("Updated visibility config", status=200)
 
 
 @collector.route("/collector/test")
diff --git a/clover/controller/control/api/visibility.py b/clover/controller/control/api/visibility.py
new file mode 100644
index 0000000..23eb714
--- /dev/null
+++ b/clover/controller/control/api/visibility.py
@@ -0,0 +1,100 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, jsonify, Response
+import redis
+import logging
+import collector
+
+visibility_api = Blueprint('visibility_api', __name__)
+
+HOST_IP = 'redis.default'
+
+
+@visibility_api.route("/visibility/clear")
+def clear_visibility():
+    # Zero out or delete redis keys with results
+    r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+    r.set('proxy_rt', 0)
+    r.set('trace_count', 0)
+    r.set('span_count', 0)
+    del_keys = ['span_user_agent', 'span_urls', 'span_urls_z',
+                'span_status_codes_z', 'span_node_url_z', 'span_node_id_z',
+                'span_user_agent_z']
+    for dk in del_keys:
+        r.delete(dk)
+    # Truncate cassandra tables
+    return collector.truncate()
+
+
+@visibility_api.route(
+             "/visibility/get/stats/<s_type>", methods=['GET', 'POST'])
+def get_visibility_stats(s_type):
+    try:
+
+        p = request.json
+        if not p:
+            stat_type = s_type
+        else:
+            stat_type = p['stat_type']
+
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+
+        content = {}
+        if stat_type == 'system' or s_type == 'all':
+            content['trace_count'] = r.get('trace_count')
+            content['span_count'] = r.get('span_count')
+        if stat_type == 'metrics' or s_type == 'all':
+            content['metrics_test'] = r.lrange('metrics_test', 0, 200)
+        if stat_type == 'tracing' or s_type == 'all':
+            content['proxy_rt'] = r.get('proxy_rt')
+            content['span_urls'] = list(r.smembers('span_urls'))
+            content['user_agents'] = list(r.smembers('span_user_agent'))
+            content['user_agent_count'] = r.zrange(
+                                       "span_user_agent_z", 0, 50, False, True)
+            content['request_url_count'] = r.zrange(
+                                             "span_urls_z", 0, 50, False, True)
+            content['status_code_count'] = r.zrange(
+                                     "span_status_codes_z", 0, 50, False, True)
+            content['node_url_count'] = r.zrange(
+                                         "span_node_url_z", 0, 50, False, True)
+            content['node_id_count'] = r.zrange(
+                                          "span_node_id_z", 0, 50, False, True)
+
+        response = jsonify(content)
+        return response
+    except Exception as e:
+        logging.debug(e)
+        return Response("Error getting visibility stats", status=400)
+
+
+@visibility_api.route("/visibility/get/<c_type>", methods=['GET', 'POST'])
+def get_visibility_config(c_type):
+    try:
+
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+
+        content = {}
+
+        if c_type == 'services' or c_type == 'all':
+            services = list(r.smembers('visibility_services'))
+            content['visibility_services'] = services
+        if c_type == 'metrics' or c_type == 'all':
+            content['metric_prefixes'] = list(r.smembers('metric_prefixes'))
+            content['metric_suffixes'] = list(r.smembers('metric_suffixes'))
+            content['custom_metrics'] = list(r.smembers('custom_metrics'))
+
+        response = jsonify(content)
+        return response
+    except Exception as e:
+        logging.debug(e)
+        return Response("Error getting visibility config", status=400)
+
+
+@visibility_api.route("/visibility/set", methods=['GET', 'POST'])
+def set_visibility():
+    return collector.set_collector()
diff --git a/clover/controller/control/control.py b/clover/controller/control/control.py
index 70eeacd..a54c762 100644
--- a/clover/controller/control/control.py
+++ b/clover/controller/control/control.py
@@ -8,6 +8,7 @@
 from flask import Flask, request, jsonify
 from views.dashboard import simple_page
 from api.collector import collector
+from api.visibility_api import visibility_api
 from api.snort import snort
 from api.halyard import halyard
 from api.nginx import nginx
@@ -23,6 +24,7 @@ try:
     # Register blueprints
     application.register_blueprint(simple_page)
     application.register_blueprint(collector)
+    application.register_blueprint(visibility_api)
     application.register_blueprint(snort)
     application.register_blueprint(halyard)
     application.register_blueprint(nginx)