Implement initial clover-controller service

- First pass of clover-controller which resides within the k8s
cluster and provides interfaces to all Clover services
- Only service that should need to be exposed outside of
cluster
- Docker build of container that uses stack of nginx, gunicorn
and flask to provide REST interface
- REST interface is intended to serve cloverctl CLI and
dashboard browser UI
- Implements GRPC messaging to clover-collector and snort
- GRPC interfaces files for snort/nginx are added to
container from repo. Collector GRPC files will be removed
from controller/control/api once patch below is merged
https://gerrit.opnfv.org/gerrit/#/c/57245/ and added
similarly
- Provides first pass callback for file upload from
clover-server.
- Some REST messages implement JSON for passing params
to internal services
- Redis interface added to obtain data from services.
Currently, a simple interface to retrieve snort event
information
- YAML manifest renderer to add to k8s. Uses NodePort
service currently, defaulting to port 32044.

- Removed collector gRPC interface files with merge of collector
- Expose tracing and monitoring host/port parameters, as these vary
depending on Istio version and Jaeger version
- Add logging to flask blueprints
- Added jmeter blueprint interface with REST for
testplan generation, start test and result retrieval
- Added flask Response to REST reply messages
- Retrieve some basic stats from collector in json
response

Change-Id: I59eaeb860445ade4b45bba22747a61fb0cf0bbd4
Signed-off-by: Eddie Arrage <eddie.arrage@huawei.com>

12 files changed, 527 insertions, 0 deletions

diff --git a/clover/controller/control/__init__.py b/clover/controller/control/__init__.py
new file mode 100644
index 0000000..d67a6c0
--- /dev/null
+++ b/clover/controller/control/__init__.py
@@ -0,0 +1,11 @@
+from flask import Flask, Response
+
+
+app = Flask(__name__)
+
+@app.route("/")
+def index():
+    return Response("It works!"), 200
+
+if __name__ == "__main__":
+    app.run(debug=True)
diff --git a/clover/controller/control/api/__init__.py b/clover/controller/control/api/__init__.py
new file mode 100644
index 0000000..d67a6c0
--- /dev/null
+++ b/clover/controller/control/api/__init__.py
@@ -0,0 +1,11 @@
+from flask import Flask, Response
+
+
+app = Flask(__name__)
+
+@app.route("/")
+def index():
+    return Response("It works!"), 200
+
+if __name__ == "__main__":
+    app.run(debug=True)
diff --git a/clover/controller/control/api/collector.py b/clover/controller/control/api/collector.py
new file mode 100644
index 0000000..c82c543
--- /dev/null
+++ b/clover/controller/control/api/collector.py
@@ -0,0 +1,131 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, jsonify, Response
+import grpc
+import pickle
+import collector_pb2
+import collector_pb2_grpc
+import redis
+import logging
+
+
+collector = Blueprint('collector', __name__)
+
+grpc_port = '50054'
+pod_name = 'clover-collector'
+collector_grpc = pod_name + ':' + grpc_port
+channel = grpc.insecure_channel(collector_grpc)
+stub = collector_pb2_grpc.ControllerStub(channel)
+CASSANDRA_HOSTS = pickle.dumps(['cassandra.default'])
+
+HOST_IP = 'redis'
+
+
+@collector.route("/collector/init")
+def init():
+    try:
+        response = stub.InitVisibility(collector_pb2.ConfigCassandra(
+            cassandra_hosts=CASSANDRA_HOSTS, cassandra_port=9042))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting via gRPC", status=400)
+        else:
+            return Response("Error initializing visibility", status=400)
+    return response.message
+
+
+@collector.route("/collector/truncate")
+def truncate():
+    try:
+        schemas = pickle.dumps(['spans', 'traces', 'metrics'])
+        response = stub.TruncateVisibility(collector_pb2.Schemas(
+            schemas=schemas, cassandra_hosts=CASSANDRA_HOSTS,
+            cassandra_port=9042))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting via gRPC", status=400)
+        else:
+            return Response("Error truncating visibility", status=400)
+    return response.message
+
+
+@collector.route("/collector/start", methods=['GET', 'POST'])
+def start():
+    try:
+        p = request.json
+        if not p:
+            sample_interval = '5'
+            t_host = 'jaeger-deployment.istio-system'
+            t_port = '16686'
+            m_host = 'prometheus.istio-system'
+            m_port = '9090'
+        else:
+            try:
+                sample_interval = p['sample_interval']
+                t_host = p['t_host']
+                t_port = p['t_port']
+                m_host = p['m_host']
+                m_port = p['m_port']
+            except (KeyError, ValueError) as e:
+                logging.debug(e)
+                return Response("Invalid value in json/yaml", status=400)
+        response = stub.StartCollector(collector_pb2.ConfigCollector(
+            t_port=t_port, t_host=t_host,
+            m_port=m_port, m_host=m_host,
+            c_port='9042', c_hosts=CASSANDRA_HOSTS,
+            sinterval=sample_interval))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting via gRPC", status=400)
+        else:
+            return Response("Error starting visibility", status=400)
+    return response.message
+
+
+@collector.route("/collector/stop")
+def stop():
+    try:
+        response = stub.StopCollector(collector_pb2.ConfigCollector())
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting via gRPC", status=400)
+        else:
+            return Response("Error stopping visibility", status=400)
+    return response.message
+
+
+@collector.route("/collector/stats", methods=['GET', 'POST'])
+def stats():
+    try:
+        p = request.json
+        if not p:
+            stat_type = 'toplevel'
+        else:
+            stat_type = p['stat_type']
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+        content = {}
+        content['proxy_rt'] = r.get('proxy_rt')
+        content['trace_count'] = r.get('trace_count')
+        content['span_urls'] = list(r.smembers('span_urls'))
+        response = jsonify(content)
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting via gRPC", status=400)
+        else:
+            return Response("Error getting visibility stats", status=400)
+    return response
+
+
+@collector.route("/collector/test")
+def test():
+    return "<h1 style='color:blue'>Collector API Test Response</h1>"
diff --git a/clover/controller/control/api/file_upload.py b/clover/controller/control/api/file_upload.py
new file mode 100644
index 0000000..a479c30
--- /dev/null
+++ b/clover/controller/control/api/file_upload.py
@@ -0,0 +1,28 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, Response
+import redis
+import logging
+
+file_upload = Blueprint('file_upload', __name__)
+
+HOST_IP = 'redis'
+
+
+@file_upload.route("/upload", methods=['GET', 'POST'])
+def upload_meta():
+    try:
+        content = request.form
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+        response = content.get('upload.name')
+        r.set('upload_meta', response)
+    except Exception as e:
+        logging.debug(e)
+        r.set('upload_meta', "failure")
+        return Response('Unable to write file metadata to redis', status=400)
+    return response
diff --git a/clover/controller/control/api/jmeter.py b/clover/controller/control/api/jmeter.py
new file mode 100644
index 0000000..09625f5
--- /dev/null
+++ b/clover/controller/control/api/jmeter.py
@@ -0,0 +1,101 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, Response
+import grpc
+import jmeter_pb2
+import jmeter_pb2_grpc
+import pickle
+import logging
+
+
+jmeter = Blueprint('jmeter', __name__)
+
+grpc_port = '50054'
+pod_name = 'clover-jmeter-master'
+jmeter_grpc = pod_name + ':' + grpc_port
+channel = grpc.insecure_channel(jmeter_grpc)
+stub = jmeter_pb2_grpc.ControllerStub(channel)
+
+
+@jmeter.route("/jmeter/gen", methods=['GET', 'POST'])
+def gentest():
+    try:
+        p = request.json
+        u_list = []
+        u_names = []
+        u_methods = []
+        try:
+            for u in p['url_list']:
+                u_list.append(u['url'])
+                u_names.append(u['name'])
+                u_methods.append(u['method'])
+            url_list = pickle.dumps(u_list)
+            url_names = pickle.dumps(u_names)
+            url_methods = pickle.dumps(u_methods)
+            num_threads = p['load_spec']['num_threads']
+            ramp_time = p['load_spec']['ramp_time']
+            loops = p['load_spec']['loops']
+        except (KeyError, ValueError) as e:
+            logging.debug(e)
+            return Response('Invalid value in test plan json/yaml', status=400)
+        response = stub.GenTest(jmeter_pb2.ConfigJmeter(
+            url_list=url_list, url_names=url_names, url_methods=url_methods,
+            num_threads=str(num_threads), ramp_time=str(ramp_time),
+            loops=str(loops)))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error generating test plan", status=400)
+    return response.message
+
+
+@jmeter.route("/jmeter/start", methods=['GET', 'POST'])
+def start():
+    try:
+        p = request.json
+        if not p:
+            slave_list = ''
+            num_slaves = '0'
+        else:
+            slave_list = p['slave_list']
+            num_slaves = p['num_slaves']
+        response = stub.StartTest(jmeter_pb2.TestParams(
+             num_slaves=num_slaves, test_plan='test.jmx',
+             slave_ips=slave_list))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error starting jmeter test", status=400)
+    return response.message
+
+
+@jmeter.route("/jmeter/results/<r_type>", methods=['GET'])
+def results(r_type):
+    try:
+        if not r_type:
+            r_file = 'results'
+        else:
+            r_file = r_type
+        response = stub.GetResults(jmeter_pb2.JResults(
+            r_format='csv', r_file=r_file))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error returning results", status=400)
+    return response.message
+
+
+@jmeter.route("/jmeter/test")
+def test():
+    return "<h1 style='color:blue'>Jmeter API Test Response</h1>"
diff --git a/clover/controller/control/api/nginx.py b/clover/controller/control/api/nginx.py
new file mode 100644
index 0000000..ba99b94
--- /dev/null
+++ b/clover/controller/control/api/nginx.py
@@ -0,0 +1,51 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, Response
+import grpc
+import nginx_pb2
+import nginx_pb2_grpc
+import pickle
+import logging
+
+nginx = Blueprint('nginx', __name__)
+
+
+@nginx.route("/nginx/slb", methods=['GET', 'POST'])
+def slblist():
+    grpc_port = '50054'
+    try:
+        p = request.json
+        try:
+            slb_name = p['slb_name']
+            nginx_grpc = slb_name + ':' + grpc_port
+            channel = grpc.insecure_channel(nginx_grpc)
+            stub = nginx_pb2_grpc.ControllerStub(channel)
+
+            s_list = []
+            for s in p['slb_list']:
+                s_list.append(s['url'])
+            slb_list = pickle.dumps(s_list)
+            response = stub.ModifyLB(nginx_pb2.ConfigLB(
+                server_port=p['server_port'], server_name=p['server_name'],
+                slb_list=slb_list,
+                slb_group=p['slb_group'], lb_path=p['lb_path']))
+        except (KeyError, ValueError) as e:
+            logging.debug(e)
+            return Response('Invalid value in test plan json/yaml', status=400)
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to LB via gRPC", status=400)
+        else:
+            return Response("Error modifying LB server list", status=400)
+    return response.message
+
+
+@nginx.route("/nginx/test")
+def test():
+    return "<h1 style='color:blue'>Nginx API Test Response</h1>"
diff --git a/clover/controller/control/api/snort.py b/clover/controller/control/api/snort.py
new file mode 100644
index 0000000..e2177be
--- /dev/null
+++ b/clover/controller/control/api/snort.py
@@ -0,0 +1,99 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, request, Response
+import grpc
+import snort_pb2
+import snort_pb2_grpc
+import logging
+import redis
+
+snort = Blueprint('snort', __name__)
+
+grpc_port = '50052'
+pod_name = 'snort-ids'
+snort_grpc = pod_name + ':' + grpc_port
+channel = grpc.insecure_channel(snort_grpc)
+stub = snort_pb2_grpc.ControllerStub(channel)
+
+HOST_IP = 'redis'
+
+
+@snort.route("/snort/addrule", methods=['GET', 'POST'])
+def addrule():
+    try:
+        try:
+            p = request.json
+            if p['content'] != "":
+                response = stub.AddRules(snort_pb2.AddRule(
+                    protocol=p['protocol'], dest_port=p['dest_port'],
+                    dest_ip=p['dest_ip'], src_port=p['src_port'],
+                    src_ip=p['src_ip'], msg=p['msg'], sid=p['sid'],
+                    rev=p['rev'], content=p['content']))
+            else:
+                response = stub.AddRules(snort_pb2.AddRule(
+                    protocol=p['protocol'], dest_port=p['dest_port'],
+                    dest_ip=p['dest_ip'], src_port=p['src_port'],
+                    src_ip=p['src_ip'], msg=p['msg'], sid=p['sid'],
+                    rev=p['rev']))
+        except (KeyError, ValueError) as e:
+            logging.debug(e)
+            return Response('Invalid value in IDS rule json/yaml', status=400)
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to IDS via gRPC", status=400)
+        else:
+            return Response("Error adding IDS rule", status=400)
+    return response.message
+
+
+@snort.route("/snort/start")
+def start():
+    try:
+        response = stub.StartSnort(snort_pb2.ControlSnort(pid='0'))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error starting IDS", status=400)
+    return response.message
+
+
+@snort.route("/snort/stop")
+def stop():
+    try:
+        response = stub.StopSnort(snort_pb2.ControlSnort(pid='0'))
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error stopping IDS", status=400)
+    return response.message
+
+
+@snort.route("/snort/get_events", methods=['GET'])
+def get_events():
+    try:
+        p = request.json
+        r = redis.StrictRedis(host=HOST_IP, port=6379, db=0)
+        event_data = r.hget(p['event_key'], p['field'])
+        response = event_data
+    except Exception as e:
+        logging.debug(e)
+        if e.__class__.__name__ == "_Rendezvous":
+            return Response("Error connecting to jmeter via gRPC", status=400)
+        else:
+            return Response("Error returning IDS event", status=400)
+    return response
+
+
+@snort.route("/snort/test")
+def test():
+    return "<h1 style='color:blue'>Snort API Test Response</h1>"
diff --git a/clover/controller/control/control.py b/clover/controller/control/control.py
new file mode 100644
index 0000000..54f713a
--- /dev/null
+++ b/clover/controller/control/control.py
@@ -0,0 +1,55 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Flask, request, jsonify
+from views.dashboard import simple_page
+from api.collector import collector
+from api.snort import snort
+from api.nginx import nginx
+from api.jmeter import jmeter
+from api.file_upload import file_upload
+import logging
+
+logging.basicConfig(filename='flask.log', level=logging.DEBUG)
+
+application = Flask(__name__)
+
+try:
+    # Register blueprints
+    application.register_blueprint(simple_page)
+    application.register_blueprint(collector)
+    application.register_blueprint(snort)
+    application.register_blueprint(nginx)
+    application.register_blueprint(jmeter)
+    application.register_blueprint(file_upload)
+except Exception as e:
+    logging.debug(e)
+
+
+@application.route("/")
+def test():
+    return "<h1 style='color:blue'>clover-controller up</h1>"
+
+
+@application.route("/config_server/<server>")
+def show_server(server):
+    return "User %s" % server
+
+
+@application.route("/get_json", methods=['GET', 'POST'])
+def get_json():
+    try:
+        content = request.json
+        cmd = content["cmd"]
+        resp = jsonify({"cmd": cmd})
+    except Exception as e:
+        resp = e
+    return resp
+
+
+if __name__ == "__main__":
+    application.run(host='0.0.0.0')
diff --git a/clover/controller/control/templates/home.html b/clover/controller/control/templates/home.html
new file mode 100644
index 0000000..6de644e
--- /dev/null
+++ b/clover/controller/control/templates/home.html
@@ -0,0 +1,6 @@
+<!doctype html>
+<html>
+   <body>
+      <h1>Clover Dashboard</h1>
+   </body>
+</html>
diff --git a/clover/controller/control/views/__init__.py b/clover/controller/control/views/__init__.py
new file mode 100644
index 0000000..d67a6c0
--- /dev/null
+++ b/clover/controller/control/views/__init__.py
@@ -0,0 +1,11 @@
+from flask import Flask, Response
+
+
+app = Flask(__name__)
+
+@app.route("/")
+def index():
+    return Response("It works!"), 200
+
+if __name__ == "__main__":
+    app.run(debug=True)
diff --git a/clover/controller/control/views/dashboard.py b/clover/controller/control/views/dashboard.py
new file mode 100644
index 0000000..8b6969c
--- /dev/null
+++ b/clover/controller/control/views/dashboard.py
@@ -0,0 +1,19 @@
+# Copyright (c) Authors of Clover
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+
+from flask import Blueprint, render_template, abort
+from jinja2 import TemplateNotFound
+
+simple_page = Blueprint('simple_page', __name__)
+
+
+@simple_page.route('/dashboard', defaults={'page': 'index'})
+def show(page):
+    try:
+        return render_template('home.html')
+    except TemplateNotFound:
+        abort(404)
diff --git a/clover/controller/control/wsgi.py b/clover/controller/control/wsgi.py
new file mode 100644
index 0000000..b787e5f
--- /dev/null
+++ b/clover/controller/control/wsgi.py
@@ -0,0 +1,4 @@
+from control import application
+
+if __name__ == "__main__":
+    application.run()