diff options
42 files changed, 953 insertions, 191 deletions
diff --git a/ci/deploy.sh b/ci/deploy.sh index 2580dbc..4bdce89 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -15,7 +15,7 @@ MASTER_NODE_NAME="master" SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" K8S_ISTIO_DEPLOY_TIMEOUT=3600 -FUNCTEST_IMAGE="opnfv/functest-kubernetes:latest" +FUNCTEST_IMAGE="opnfv/functest-kubernetes-features:amd64-gambia" INSTALLER_TYPE="container4nfv" DEPLOY_SCENARIO="k8-istio-clover" @@ -42,8 +42,8 @@ MASTER_NODE_KEY=$(vagrant ssh-config $MASTER_NODE_NAME | awk '/IdentityFile /{pr ssh $SSH_OPTIONS -i $MASTER_NODE_KEY ${MASTER_NODE_USER}@${MASTER_NODE_HOST} rm -rf clover scp $SSH_OPTIONS -i $MASTER_NODE_KEY -r $CLOVER_BASE_DIR ${MASTER_NODE_USER}@${MASTER_NODE_HOST}:clover -# Run test -ssh $SSH_OPTIONS -i $MASTER_NODE_KEY ${MASTER_NODE_USER}@${MASTER_NODE_HOST} ./clover/ci/test.sh +# Deploy SDC sample for Functest test +ssh $SSH_OPTIONS -i $MASTER_NODE_KEY ${MASTER_NODE_USER}@${MASTER_NODE_HOST} ./clover/ci/sdc_setup.sh echo "Clover deploy complete!" @@ -63,6 +63,9 @@ echo "export KUBE_MASTER_URL=$KUBE_MASTER_URL" >> $RC_FILE KUBE_MASTER_IP=$(echo $KUBE_MASTER_URL | awk -F'https://|:[0-9]+' '$0=$2') echo "export KUBE_MASTER_IP=$KUBE_MASTER_IP" >> $RC_FILE +# Restart docker service in case Container4NFV CI job delete docker0 bridge +systemctl restart docker + # Run functest sudo docker pull $FUNCTEST_IMAGE sudo docker run --rm \ diff --git a/ci/sdc_setup.sh b/ci/sdc_setup.sh new file mode 100755 index 0000000..f4d82f5 --- /dev/null +++ b/ci/sdc_setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +set -ex + +# Deploy Clover SDC sample +kubectl apply -f <(istioctl kube-inject -f ~/clover/samples/scenarios/service_delivery_controller_opnfv.yaml) + +# Wait for SDC sample deployed +kubectl get services +kubectl get pods + +r="0" +while [ $r -ne "10" ] +do + sleep 30 + kubectl get pods + r=$(kubectl get pods | grep Running | wc -l) +done + +echo "Set up Clover SDC sample complete!" diff --git a/ci/test.sh b/ci/test.sh deleted file mode 100755 index 4e0ccc5..0000000 --- a/ci/test.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Copyright (c) Authors of Clover -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -set -ex - -echo "Clover test complete!" diff --git a/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go b/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go index 9468714..2ad43f1 100644 --- a/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go +++ b/clover/cloverctl/src/cloverctl/cmd/clear_visibility.go @@ -28,7 +28,7 @@ func init() { } func clearCollector() { - url := controllerIP + "/collector/truncate" + url := controllerIP + "/visibility/clear" resp, err := resty.R(). Get(url) diff --git a/clover/cloverctl/src/cloverctl/cmd/get_visibility.go b/clover/cloverctl/src/cloverctl/cmd/get_visibility.go index d987412..820b25a 100644 --- a/clover/cloverctl/src/cloverctl/cmd/get_visibility.go +++ b/clover/cloverctl/src/cloverctl/cmd/get_visibility.go @@ -13,23 +13,38 @@ import ( "github.com/spf13/cobra" ) +var VisibilityStat string +var VisibilityConfig string -var visibilitystatsCmd = &cobra.Command{ +var visibilitygetCmd = &cobra.Command{ Use: "visibility", - Short: "Get toplevel visibility stats", + Short: "Get visibility config & stats", Long: ``, Run: func(cmd *cobra.Command, args []string) { - statsCollector() + getVisibility() }, } func init() { - getCmd.AddCommand(visibilitystatsCmd) - //visibilitystartCmd.PersistentFlags().StringVarP(&cloverFile, "f", "f", "", "Input yaml file with test plan params") + getCmd.AddCommand(visibilitygetCmd) + visibilitygetCmd.PersistentFlags().StringVarP(&VisibilityStat, "stat", "s", "", "Visibility stats type to get") + visibilitygetCmd.PersistentFlags().StringVarP(&VisibilityConfig, "conf", "c", "", "Visibility config type to get") } -func statsCollector() { - url := controllerIP + "/collector/stats" +func getVisibility() { + + url_prefix := "/visibility/get/" + get_data := "all" + response_prefix := "Config" + if VisibilityStat != "" { + url_prefix = "/visibility/get/stats/" + get_data = VisibilityStat + response_prefix = "Stat" + } else if VisibilityConfig != "" { + get_data = VisibilityConfig + } + + url := controllerIP + url_prefix + get_data resp, err := resty.R(). SetHeader("Accept", "application/json"). @@ -37,5 +52,5 @@ func statsCollector() { if err != nil { panic(err.Error()) } - fmt.Printf("\nProxy Response Time: %v\n", resp) + fmt.Printf("\n%s %s: %v\n", response_prefix, get_data, resp) } diff --git a/clover/cloverctl/src/cloverctl/cmd/set_nginx.go b/clover/cloverctl/src/cloverctl/cmd/set_nginx.go new file mode 100644 index 0000000..e7e65c2 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/cmd/set_nginx.go @@ -0,0 +1,76 @@ +// Copyright (c) Authors of Clover +// +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Apache License, Version 2.0 +// which accompanies this distribution, and is available at +// http://www.apache.org/licenses/LICENSE-2.0 + +package cmd + +import ( + "fmt" + "io/ioutil" + "gopkg.in/resty.v1" + "github.com/spf13/cobra" + "github.com/ghodss/yaml" +) + + +var setserverCmd = &cobra.Command{ + Use: "server", + Short: "Modify nginx server configuration", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + setNginx("server") + }, +} + +var setlbCmd = &cobra.Command{ + Use: "lb", + Short: "Modify nginx lb configuration", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + setNginx("lb") + }, +} + +func init() { + setCmd.AddCommand(setserverCmd) + setserverCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file for server config") + setserverCmd.MarkFlagRequired("file") + + setCmd.AddCommand(setlbCmd) + setlbCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file for lb config") + setlbCmd.MarkFlagRequired("file") + +} + +func setNginx(nginx_service string) { + + url := "" + if nginx_service == "server" { + url = controllerIP + "/nginx/server" + } else { + url = controllerIP + "/nginx/lb" + } + + in, err := ioutil.ReadFile(cloverFile) + if err != nil { + fmt.Println("Please specify a valid yaml file") + return + } + out_json, err := yaml.YAMLToJSON(in) + if err != nil { + panic(err.Error()) + } + resp, err := resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(out_json). + Post(url) + if err != nil { + panic(err.Error()) + } + fmt.Printf("\n%v\n", resp) + + +} diff --git a/clover/cloverctl/src/cloverctl/cmd/set_visibility.go b/clover/cloverctl/src/cloverctl/cmd/set_visibility.go new file mode 100644 index 0000000..685b250 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/cmd/set_visibility.go @@ -0,0 +1,57 @@ +// Copyright (c) Authors of Clover +// +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Apache License, Version 2.0 +// which accompanies this distribution, and is available at +// http://www.apache.org/licenses/LICENSE-2.0 + +package cmd + +import ( + "fmt" + "io/ioutil" + "gopkg.in/resty.v1" + "github.com/spf13/cobra" + "github.com/ghodss/yaml" +) + + +var setvisibilitystatsCmd = &cobra.Command{ + Use: "visibility", + Short: "Set visibility config for services and metrics to track", + Long: ``, + Run: func(cmd *cobra.Command, args []string) { + setCollector() + }, +} + +func init() { + setCmd.AddCommand(setvisibilitystatsCmd) + setvisibilitystatsCmd.Flags().StringVarP(&cloverFile, "file", "f", "", "Input yaml file to set visibility config") + setvisibilitystatsCmd.MarkFlagRequired("file") + +} + +func setCollector() { + url := controllerIP + "/visibility/set" + + in, err := ioutil.ReadFile(cloverFile) + if err != nil { + fmt.Println("Please specify a valid yaml file") + return + } + out_json, err := yaml.YAMLToJSON(in) + if err != nil { + panic(err.Error()) + } + resp, err := resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(out_json). + Post(url) + if err != nil { + panic(err.Error()) + } + fmt.Printf("\n%v\n", resp) + + +} diff --git a/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml b/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml new file mode 100644 index 0000000..3bb841d --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/lbv1.yaml @@ -0,0 +1,9 @@ +server_port: "9180" +server_name: "http-lb-v1" +lb_name: "http-lb-v1.default" +lb_group: "cloverlb" +lb_path: "/" +lb_list: + - url: "clover-server1:9180" + - url: "clover-server2:9180" + - url: "clover-server3:9180" diff --git a/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml b/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml new file mode 100644 index 0000000..7df9376 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/lbv2.yaml @@ -0,0 +1,8 @@ +server_port: "9180" +server_name: "http-lb-v2" +lb_name: "http-lb-v2.default" +lb_group: "cloverlb" +lb_path: "/" +lb_list: + - url: "clover-server4:9180" + - url: "clover-server5:9180" diff --git a/clover/cloverctl/src/cloverctl/yaml/server.yaml b/clover/cloverctl/src/cloverctl/yaml/server.yaml new file mode 100644 index 0000000..b44779a --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/server.yaml @@ -0,0 +1,16 @@ +server_port: "9180" +server_name: "clover-server1" +site_root: "/var/www/html" +site_index: index.html +upload_path_config: "/upload" +upload_path_test: "/upload_test" +locations: + - uri_match: "/clover/testurl" + directive: "try_files $uri @default2" + path: "/clover/testurl" + - uri_match: "/test" + directive: "try_files $uri @default1" + path: "/test" +files: + - src_file: "/var/www/html/upload/0000000001" + dest_file: "var/www/html/clover/testurl/file1.png" diff --git a/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml new file mode 100644 index 0000000..f88c673 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/set_visibility.yaml @@ -0,0 +1,13 @@ +services: + - name: proxy_access_control + - name: clover_server1 + - name: clover_server2 + - name: clover_server3 +metric_prefixes: + - prefix: envoy_cluster_outbound_9180__ + - prefix: envoy_cluster_inbound_9180__ +metric_suffixes: + - suffix: _default_svc_cluster_local_upstream_rq_2xx + - suffix: _default_svc_cluster_local_upstream_cx_active +custom_metrics: + - metric: envoy_tracing_zipkin_spans_sent diff --git a/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml new file mode 100644 index 0000000..f27f0c6 --- /dev/null +++ b/clover/cloverctl/src/cloverctl/yaml/start_visibility.yaml @@ -0,0 +1,5 @@ +sample_interval: "2" +t_host: tracing.istio-system +t_port: "80" +m_port: "9090" +m_host: prometheus.istio-system diff --git a/clover/cloverctl/src/cloverctl/yaml/visibility.yaml b/clover/cloverctl/src/cloverctl/yaml/visibility.yaml deleted file mode 100644 index 20264d2..0000000 --- a/clover/cloverctl/src/cloverctl/yaml/visibility.yaml +++ /dev/null @@ -1,7 +0,0 @@ -sample_interval: "10" -#t_host: jaeger-deployment.istio-system -#t_port: "16686" -t_host: jaeger-query.istio-system -t_port: "80" -m_port: "9090" -m_host: prometheus.istio-system diff --git a/clover/collector/db/cassops.py b/clover/collector/db/cassops.py index 6553cff..0bc9d84 100644 --- a/clover/collector/db/cassops.py +++ b/clover/collector/db/cassops.py @@ -9,7 +9,7 @@ from cassandra.cluster import Cluster from cassandra.query import BatchStatement import logging -CASSANDRA_HOSTS = ['cassandra.default'] +CASSANDRA_HOSTS = ['cassandra.clover-system'] class CassandraOps: @@ -57,13 +57,18 @@ class CassandraOps: spanid text, traceid text, duration int, - start_time int, + start_time timestamp, processid text, operation_name text, node_id text, http_url text, + user_agent text, + request_size text, + response_size text, + status_code text, upstream_cluster text, - PRIMARY KEY (spanid, traceid) + insert_time timestamp, + PRIMARY KEY (traceid, spanid) ) """) @@ -82,11 +87,18 @@ class CassandraOps: def set_prepared(self): self.session.set_keyspace(self.keyspace) - self.insert_tracing_stmt = self.session.prepare( + self.insert_span_stmt = self.session.prepare( """ INSERT INTO spans (spanid, traceid, duration, operation_name, - node_id, http_url, upstream_cluster) - VALUES (?, ?, ?, ?, ?, ?, ?) + node_id, http_url, upstream_cluster, start_time, user_agent, + request_size, response_size, status_code, insert_time) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, toTimestamp(now())) + """ + ) + self.insert_trace_stmt = self.session.prepare( + """ + INSERT INTO traces (traceid, processes) + VALUES (?, ?) """ ) self.insert_metric_stmt = self.session.prepare( @@ -103,31 +115,31 @@ class CassandraOps: def execute_batch(self): self.session.execute(self.batch) - def insert_tracing(self, table, traceid, s, tags): + def insert_span(self, traceid, s, tags): self.session.set_keyspace(self.keyspace) if 'upstream_cluster' not in tags: - logging.debug('NO UPSTREAM_CLUSTER KEY') + # logging.debug('NO UPSTREAM_CLUSTER KEY') tags['upstream_cluster'] = 'none' try: - self.batch.add(self.insert_tracing_stmt, + self.batch.add(self.insert_span_stmt, (s['spanID'], traceid, s['duration'], s['operationName'], tags['node_id'], - tags['http.url'], tags['upstream_cluster'])) + tags['http.url'], tags['upstream_cluster'], + int(str(s['startTime'])[0:13]), tags['user_agent'], + tags['request_size'], tags['response_size'], + tags['http.status_code'])) + except KeyError as e: + logging.debug('Insert span error: {}, Tags: {}'.format(e, tags)) except Exception as e: - logging.debug('{} {} {} {} {} {} {}'.format(s['spanID'], traceid, - s['duration'], s['operationName'], tags['node_id'], - tags['http.url'], tags['upstream_cluster'])) - logging.debug(e) + logging.debug('Insert span error: {}'.format(e)) + logging.debug('Tags: {}'.format(tags)) + logging.debug('Span toplevel: {}'.format(s)) + logging.debug( + 'startTime: {}'.format(int(str(s['startTime'])[0:13]))) def insert_trace(self, traceid, processes): self.session.set_keyspace(self.keyspace) - self.session.execute( - """ - INSERT INTO traces (traceid, processes) - VALUES (%s, %s) - """, - (traceid, processes) - ) + self.batch.add(self.insert_trace_stmt, (traceid, processes)) def insert_metric(self, m_name, m_value, m_time, service): self.session.set_keyspace(self.keyspace) diff --git a/clover/collector/db/redisops.py b/clover/collector/db/redisops.py index e80c417..24fbeb9 100644 --- a/clover/collector/db/redisops.py +++ b/clover/collector/db/redisops.py @@ -8,8 +8,7 @@ import redis import logging -REDIS_HOST = 'redis' -# REDIS_HOST = '10.244.0.85' +REDIS_HOST = 'redis.default' class RedisOps: @@ -27,11 +26,16 @@ class RedisOps: for s in service_names: self.r.sadd(skey, s) + def set_tracing_services(self, services, skey='tracing_services'): + self.r.delete(skey) + for s in services: + self.r.sadd(skey, s) + def init_metrics(self, pkey='metric_prefixes', skey='metric_suffixes'): - metric_prefixes = ['envoy_cluster_out_', 'envoy_cluster_in_'] + metric_prefixes = ['envoy_cluster_outbound_', 'envoy_cluster_inbound_'] metric_suffixes = [ - '_default_svc_cluster_local_http_internal_upstream_rq_2xx', - '_default_svc_cluster_local_http_upstream_cx_active'] + '_default_svc_cluster_local_upstream_rq_2xx', + '_default_svc_cluster_local_upstream_cx_active'] for p in metric_prefixes: self.r.sadd(pkey, p) for s in metric_suffixes: diff --git a/clover/collector/docker/Dockerfile b/clover/collector/docker/Dockerfile index 1714420..7b6effd 100644 --- a/clover/collector/docker/Dockerfile +++ b/clover/collector/docker/Dockerfile @@ -16,15 +16,6 @@ ENV CLOVER_REPO_DIR="${REPOS_DIR}/clover" RUN python -m pip install cassandra-driver redis # Set work directory -WORKDIR ${CLOVER_REPO_DIR} - -COPY /process clover/collector/process -COPY /grpc clover/collector/grpc -COPY /db clover/collector/db -COPY __init__.py clover/collector/__init__.py - -RUN pip install . - WORKDIR "${CLOVER_REPO_DIR}/clover/collector" CMD ./process/grpc_process.sh no_schema_init diff --git a/clover/collector/grpc/collector_client.py b/clover/collector/grpc/collector_client.py index b9e9f67..65ff2ff 100644 --- a/clover/collector/grpc/collector_client.py +++ b/clover/collector/grpc/collector_client.py @@ -55,7 +55,7 @@ def get_podip(pod_name): def init_visibility(stub): try: - cassandra_hosts = pickle.dumps(['cassandra.default']) + cassandra_hosts = pickle.dumps(['cassandra.clover-system']) response = stub.InitVisibility(collector_pb2.ConfigCassandra( cassandra_hosts=cassandra_hosts, cassandra_port=9042)) except Exception as e: @@ -65,7 +65,7 @@ def init_visibility(stub): def clean_visibility(stub): try: - cassandra_hosts = pickle.dumps(['cassandra.default']) + cassandra_hosts = pickle.dumps(['cassandra.clover-system']) schemas = pickle.dumps(['spans', 'traces', 'metrics']) response = stub.TruncateVisibility(collector_pb2.Schemas( schemas=schemas, cassandra_hosts=cassandra_hosts, @@ -77,7 +77,7 @@ def clean_visibility(stub): def start_collector(stub): try: - cassandra_hosts = pickle.dumps(['cassandra.default']) + cassandra_hosts = pickle.dumps(['cassandra.clover-system']) response = stub.StartCollector(collector_pb2.ConfigCollector( t_port='16686', t_host='jaeger-deployment.istio-system', m_port='9090', m_host='prometheus.istio-system', diff --git a/clover/collector/grpc/collector_server.py b/clover/collector/grpc/collector_server.py index c2eb221..a10078e 100644 --- a/clover/collector/grpc/collector_server.py +++ b/clover/collector/grpc/collector_server.py @@ -29,7 +29,7 @@ class Controller(collector_pb2_grpc.ControllerServicer): level=logging.DEBUG) self.collector = 0 if init_visibility == 'set_schemas': - cassandra_hosts = pickle.dumps(['cassandra.default']) + cassandra_hosts = pickle.dumps(['cassandra.clover-system']) self.InitVisibility(collector_pb2.ConfigCassandra( cassandra_port=9042, cassandra_hosts=cassandra_hosts), "") diff --git a/clover/collector/process/collect.py b/clover/collector/process/collect.py index d8beb49..3d9df8a 100644 --- a/clover/collector/process/collect.py +++ b/clover/collector/process/collect.py @@ -16,19 +16,25 @@ import argparse import logging import ast -TRACING_SERVICES = ['istio-ingress'] TRACING_PORT = "16686" MONITORING_PORT = "9090" CASSANDRA_PORT = 9042 # Provide as integer MONITORING_HOST = "prometheus.istio-system" -TRACING_HOST = "jaeger-deployment.istio-system" -CASSANDRA_HOSTS = ['cassandra.default'] +TRACING_HOST = "tracing.istio-system" +CASSANDRA_HOSTS = ['cassandra.clover-system'] class Collector: def __init__(self, t_port, t_host, m_port, m_host, c_port, c_hosts): - logging.basicConfig(filename='collector.log', level=logging.DEBUG) + + # logging.basicConfig(filename='collector.log', level=logging.DEBUG) + logging.basicConfig(filename='collector.log', level=logging.ERROR) + # logging.getLogger("requests").setLevel(logging.DEBUG) + logging.getLogger("requests").setLevel(logging.ERROR) + # logging.getLogger("urllib3").setLevel(logging.DEBUG) + logging.getLogger("urllib3").setLevel(logging.ERROR) + try: self.t = Tracing(t_host, t_port, '', False) monitoring_url = "http://{}:{}".format(m_host, m_port) @@ -40,63 +46,89 @@ class Collector: logging.debug(e) # Toplevel tracing retrieval and batch insert - def get_tracing(self, services, time_back=20): - self.c.set_batch() - for service in services: - traces = self.t.getTraces(service, time_back) - try: - self.set_tracing(traces) - except Exception as e: - logging.debug(e) - self.c.execute_batch() + def get_tracing(self, time_back=300): + try: + services = self.r.get_services() + for service in services: + traces = self.t.getTraces(service.replace("_", "-"), time_back, + '20000') + try: + self.set_tracing(traces) + except Exception as e: + logging.debug(e) + + # Update list of available services from tracing + services = self.t.getServices() + self.r.set_tracing_services(services) + except Exception as e: + logging.debug(e) # Insert to cassandra visibility traces and spans tables def set_tracing(self, trace): for traces in trace['data']: + self.c.set_batch() for spans in traces['spans']: + try: span = {} span['spanID'] = spans['spanID'] span['duration'] = spans['duration'] span['startTime'] = spans['startTime'] span['operationName'] = spans['operationName'] + tag = {} for tags in spans['tags']: tag[tags['key']] = tags['value'] - self.c.insert_tracing('spans', traces['traceID'], - span, tag) + self.c.insert_span(traces['traceID'], span, tag) + except Exception as e: + logging.debug("spans loop") + logging.debug(e) + process_list = [] for p in traces['processes']: process_list.append(p) service_names = [] for pname in process_list: service_names.append(traces['processes'][pname]['serviceName']) - self.c.insert_trace(traces['traceID'], service_names) + try: + self.c.insert_trace(traces['traceID'], service_names) + self.c.execute_batch() + except Exception as e: + logging.debug(e) # Insert to cassandra visibility metrics table def get_monitoring(self): - # Fetch collector service/metric lists from redis - service_names = self.r.get_services() - metric_prefixes, metric_suffixes = self.r.get_metrics() - - self.c.set_batch() - for sname in service_names: - for prefix in metric_prefixes: - for suffix in metric_suffixes: - try: - metric_name = prefix + sname + suffix - query_params = { - "type": "instant", - "query": metric_name - } - data = self.m.query(query_params) - m_value = data['data']['result'][0]['value'][1] - m_time = data['data']['result'][0]['value'][0] - mn = data['data']['result'][0]['metric']['__name__'] - self.c.insert_metric(mn, m_value, str(m_time), sname) - except Exception as e: - logging.debug(e) - self.c.execute_batch() + try: + # Fetch collector service/metric lists from redis + service_names = self.r.get_services() + metric_prefixes, metric_suffixes = self.r.get_metrics() + + self.c.set_batch() + for sname in service_names: + for prefix in metric_prefixes: + for suffix in metric_suffixes: + try: + metric_name = prefix + sname + suffix + query_params = { + "type": "instant", + "query": metric_name + } + data = self.m.query(query_params) + m_value = data['data']['result'][0]['value'][1] + m_time = data['data']['result'][0]['value'][0] + mn = data[ + 'data']['result'][0]['metric']['__name__'] + self.c.insert_metric( + mn, m_value, str(m_time), sname) + + # Add to redis temporarily + self.r.r.set(mn, m_value) + + except Exception as e: + logging.debug(e) + self.c.execute_batch() + except Exception as e: + logging.debug(e) # TODO add batch retrieval for monitoring metrics # query_range_param = { @@ -124,11 +156,13 @@ def main(args): loop = True while loop: try: - c.get_tracing(args['t_services']) + c.get_tracing() c.get_monitoring() time.sleep(int(args['sinterval'])) except KeyboardInterrupt: loop = False + except Exception as e: + logging.debug(e) if __name__ == '__main__': @@ -154,9 +188,6 @@ if __name__ == '__main__': parser.add_argument( '-c_port', default=CASSANDRA_PORT, help='Port to access Cassandra cluster') - parser.add_argument( - '-t_services', default=TRACING_SERVICES, - help='Collect services on this list of services') args, unknown = parser.parse_known_args() print(main(vars(args))) diff --git a/clover/collector/yaml/manifest.template b/clover/collector/yaml/manifest.template index c7aa3e7..795bd8f 100644 --- a/clover/collector/yaml/manifest.template +++ b/clover/collector/yaml/manifest.template @@ -5,6 +5,7 @@ metadata: name: {{ deploy_name }} labels: app: {{ deploy_name }} + namespace: clover-system spec: template: metadata: @@ -27,6 +28,7 @@ metadata: name: {{ deploy_name }} labels: app: {{ deploy_name }} + namespace: clover-system spec: ports: - port: {{ grpc_port }} diff --git a/clover/controller/__init__.py b/clover/controller/__init__.py index d67a6c0..e69de29 100644 --- a/clover/controller/__init__.py +++ b/clover/controller/__init__.py @@ -1,11 +0,0 @@ -from flask import Flask, Response - - -app = Flask(__name__) - -@app.route("/") -def index(): - return Response("It works!"), 200 - -if __name__ == "__main__": - app.run(debug=True) diff --git a/clover/controller/control/__init__.py b/clover/controller/control/__init__.py index d67a6c0..e69de29 100644 --- a/clover/controller/control/__init__.py +++ b/clover/controller/control/__init__.py @@ -1,11 +0,0 @@ -from flask import Flask, Response - - -app = Flask(__name__) - -@app.route("/") -def index(): - return Response("It works!"), 200 - -if __name__ == "__main__": - app.run(debug=True) diff --git a/clover/controller/control/api/collector.py b/clover/controller/control/api/collector.py index c82c543..3abcba7 100644 --- a/clover/controller/control/api/collector.py +++ b/clover/controller/control/api/collector.py @@ -5,7 +5,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -from flask import Blueprint, request, jsonify, Response +from flask import Blueprint, request, Response import grpc import pickle import collector_pb2 @@ -17,13 +17,13 @@ import logging collector = Blueprint('collector', __name__) grpc_port = '50054' -pod_name = 'clover-collector' +pod_name = 'clover-collector.clover-system' collector_grpc = pod_name + ':' + grpc_port channel = grpc.insecure_channel(collector_grpc) stub = collector_pb2_grpc.ControllerStub(channel) -CASSANDRA_HOSTS = pickle.dumps(['cassandra.default']) +CASSANDRA_HOSTS = pickle.dumps(['cassandra.clover-system']) -HOST_IP = 'redis' +HOST_IP = 'redis.default' @collector.route("/collector/init") @@ -62,7 +62,7 @@ def start(): p = request.json if not p: sample_interval = '5' - t_host = 'jaeger-deployment.istio-system' + t_host = 'tracing.istio-system' t_port = '16686' m_host = 'prometheus.istio-system' m_port = '9090' @@ -103,27 +103,36 @@ def stop(): return response.message -@collector.route("/collector/stats", methods=['GET', 'POST']) -def stats(): +@collector.route("/collector/set", methods=['GET', 'POST']) +def set_collector(): try: p = request.json - if not p: - stat_type = 'toplevel' - else: - stat_type = p['stat_type'] r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) - content = {} - content['proxy_rt'] = r.get('proxy_rt') - content['trace_count'] = r.get('trace_count') - content['span_urls'] = list(r.smembers('span_urls')) - response = jsonify(content) + del_keys = ['visibility_services', 'metric_prefixes', + 'metric_suffixes', 'custom_metrics'] + for dk in del_keys: + r.delete(dk) + + try: + for service in p['services']: + r.sadd('visibility_services', service['name']) + except (KeyError, ValueError) as e: + logging.debug(e) + return Response( + "Specify at least one service to track", status=400) + if p['metric_prefixes'] and p['metric_suffixes']: + for prefix in p['metric_prefixes']: + r.sadd('metric_prefixes', prefix['prefix']) + for suffix in p['metric_suffixes']: + r.sadd('metric_suffixes', suffix['suffix']) + if p['custom_metrics']: + for metric in p['custom_metrics']: + r.sadd('custom_metrics', metric['metric']) + except Exception as e: logging.debug(e) - if e.__class__.__name__ == "_Rendezvous": - return Response("Error connecting via gRPC", status=400) - else: - return Response("Error getting visibility stats", status=400) - return response + return Response("Error setting visibility config", status=400) + return Response("Updated visibility config", status=200) @collector.route("/collector/test") diff --git a/clover/controller/control/api/file_upload.py b/clover/controller/control/api/file_upload.py index a479c30..a532bc8 100644 --- a/clover/controller/control/api/file_upload.py +++ b/clover/controller/control/api/file_upload.py @@ -5,24 +5,50 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -from flask import Blueprint, request, Response +from flask import Blueprint, request, Response, jsonify import redis import logging file_upload = Blueprint('file_upload', __name__) -HOST_IP = 'redis' +HOST_IP = 'redis.default' @file_upload.route("/upload", methods=['GET', 'POST']) -def upload_meta(): +def set_upload_metadata(): try: + response = "Uploaded file(s) successfully" content = request.form + logging.debug(content) r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) - response = content.get('upload.name') - r.set('upload_meta', response) + # Try various variable names + upload_var_names = ['upload', 'file1', 'file2', 'file3', + 'file4', 'file5', 'file6'] + for n in upload_var_names: + try: + param_name = n + '.name' + meta_name = content.get(param_name) + if meta_name: + param_path = n + '.path' + param_server = n + '.server' + meta_path = content.get(param_path) + meta_server = content.get(param_server) + entry = meta_name + ':' + meta_path + ':' + meta_server + r.sadd('upload_metadata', entry) + except Exception as e: + print("no metadata") except Exception as e: logging.debug(e) - r.set('upload_meta', "failure") return Response('Unable to write file metadata to redis', status=400) return response + + +@file_upload.route("/upload/get", methods=['GET', 'POST']) +def get_upload_metadata(): + try: + r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) + response = jsonify(list(r.smembers('upload_metadata'))) + except Exception as e: + logging.debug(e) + return Response('Unable to retrieve upload metadata', status=400) + return response diff --git a/clover/controller/control/api/nginx.py b/clover/controller/control/api/nginx.py index ba99b94..a7d902d 100644 --- a/clover/controller/control/api/nginx.py +++ b/clover/controller/control/api/nginx.py @@ -15,28 +15,28 @@ import logging nginx = Blueprint('nginx', __name__) -@nginx.route("/nginx/slb", methods=['GET', 'POST']) -def slblist(): +@nginx.route("/nginx/lb", methods=['GET', 'POST']) +def modify_lb(): grpc_port = '50054' try: p = request.json try: - slb_name = p['slb_name'] - nginx_grpc = slb_name + ':' + grpc_port + lb_name = p['lb_name'] + nginx_grpc = lb_name + ':' + grpc_port channel = grpc.insecure_channel(nginx_grpc) stub = nginx_pb2_grpc.ControllerStub(channel) s_list = [] - for s in p['slb_list']: + for s in p['lb_list']: s_list.append(s['url']) - slb_list = pickle.dumps(s_list) + lb_list = pickle.dumps(s_list) response = stub.ModifyLB(nginx_pb2.ConfigLB( server_port=p['server_port'], server_name=p['server_name'], - slb_list=slb_list, - slb_group=p['slb_group'], lb_path=p['lb_path'])) + slb_list=lb_list, + slb_group=p['lb_group'], lb_path=p['lb_path'])) except (KeyError, ValueError) as e: logging.debug(e) - return Response('Invalid value in test plan json/yaml', status=400) + return Response('Invalid value in LB json/yaml', status=400) except Exception as e: logging.debug(e) if e.__class__.__name__ == "_Rendezvous": @@ -46,6 +46,37 @@ def slblist(): return response.message +@nginx.route("/nginx/server", methods=['GET', 'POST']) +def modify_server(): + grpc_port = '50054' + try: + p = request.json + try: + server_name = p['server_name'] + nginx_grpc = server_name + ':' + grpc_port + channel = grpc.insecure_channel(nginx_grpc) + stub = nginx_pb2_grpc.ControllerStub(channel) + + locations = pickle.dumps(p['locations']) + files = pickle.dumps(p['files']) + response = stub.ModifyServer(nginx_pb2.ConfigServer( + server_port=p['server_port'], server_name=p['server_name'], + site_root=p['site_root'], locations=locations, files=files, + site_index=p['site_index'], + upload_path_config=p['upload_path_config'], + upload_path_test=p['upload_path_test'])) + except (KeyError, ValueError) as e: + logging.debug(e) + return Response('Invalid value in server json/yaml', status=400) + except Exception as e: + logging.debug(e) + if e.__class__.__name__ == "_Rendezvous": + return Response("Error connecting to server via gRPC", status=400) + else: + return Response("Error modifying server", status=400) + return response.message + + @nginx.route("/nginx/test") def test(): return "<h1 style='color:blue'>Nginx API Test Response</h1>" diff --git a/clover/controller/control/api/visibility.py b/clover/controller/control/api/visibility.py new file mode 100644 index 0000000..23eb714 --- /dev/null +++ b/clover/controller/control/api/visibility.py @@ -0,0 +1,100 @@ +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +from flask import Blueprint, request, jsonify, Response +import redis +import logging +import collector + +visibility_api = Blueprint('visibility_api', __name__) + +HOST_IP = 'redis.default' + + +@visibility_api.route("/visibility/clear") +def clear_visibility(): + # Zero out or delete redis keys with results + r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) + r.set('proxy_rt', 0) + r.set('trace_count', 0) + r.set('span_count', 0) + del_keys = ['span_user_agent', 'span_urls', 'span_urls_z', + 'span_status_codes_z', 'span_node_url_z', 'span_node_id_z', + 'span_user_agent_z'] + for dk in del_keys: + r.delete(dk) + # Truncate cassandra tables + return collector.truncate() + + +@visibility_api.route( + "/visibility/get/stats/<s_type>", methods=['GET', 'POST']) +def get_visibility_stats(s_type): + try: + + p = request.json + if not p: + stat_type = s_type + else: + stat_type = p['stat_type'] + + r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) + + content = {} + if stat_type == 'system' or s_type == 'all': + content['trace_count'] = r.get('trace_count') + content['span_count'] = r.get('span_count') + if stat_type == 'metrics' or s_type == 'all': + content['metrics_test'] = r.lrange('metrics_test', 0, 200) + if stat_type == 'tracing' or s_type == 'all': + content['proxy_rt'] = r.get('proxy_rt') + content['span_urls'] = list(r.smembers('span_urls')) + content['user_agents'] = list(r.smembers('span_user_agent')) + content['user_agent_count'] = r.zrange( + "span_user_agent_z", 0, 50, False, True) + content['request_url_count'] = r.zrange( + "span_urls_z", 0, 50, False, True) + content['status_code_count'] = r.zrange( + "span_status_codes_z", 0, 50, False, True) + content['node_url_count'] = r.zrange( + "span_node_url_z", 0, 50, False, True) + content['node_id_count'] = r.zrange( + "span_node_id_z", 0, 50, False, True) + + response = jsonify(content) + return response + except Exception as e: + logging.debug(e) + return Response("Error getting visibility stats", status=400) + + +@visibility_api.route("/visibility/get/<c_type>", methods=['GET', 'POST']) +def get_visibility_config(c_type): + try: + + r = redis.StrictRedis(host=HOST_IP, port=6379, db=0) + + content = {} + + if c_type == 'services' or c_type == 'all': + services = list(r.smembers('visibility_services')) + content['visibility_services'] = services + if c_type == 'metrics' or c_type == 'all': + content['metric_prefixes'] = list(r.smembers('metric_prefixes')) + content['metric_suffixes'] = list(r.smembers('metric_suffixes')) + content['custom_metrics'] = list(r.smembers('custom_metrics')) + + response = jsonify(content) + return response + except Exception as e: + logging.debug(e) + return Response("Error getting visibility config", status=400) + + +@visibility_api.route("/visibility/set", methods=['GET', 'POST']) +def set_visibility(): + return collector.set_collector() diff --git a/clover/controller/control/control.py b/clover/controller/control/control.py index 70eeacd..a54c762 100644 --- a/clover/controller/control/control.py +++ b/clover/controller/control/control.py @@ -8,6 +8,7 @@ from flask import Flask, request, jsonify from views.dashboard import simple_page from api.collector import collector +from api.visibility_api import visibility_api from api.snort import snort from api.halyard import halyard from api.nginx import nginx @@ -23,6 +24,7 @@ try: # Register blueprints application.register_blueprint(simple_page) application.register_blueprint(collector) + application.register_blueprint(visibility_api) application.register_blueprint(snort) application.register_blueprint(halyard) application.register_blueprint(nginx) diff --git a/clover/functest/clover_k8s.py b/clover/functest/clover_k8s.py index 654c8e5..25850c6 100644 --- a/clover/functest/clover_k8s.py +++ b/clover/functest/clover_k8s.py @@ -7,6 +7,7 @@ import functest_kubernetes.k8stest as k8stest +import clover.servicemesh.validate as istio_validate class K8sCloverTest(k8stest.K8sTesting): """Clover test suite""" @@ -15,12 +16,10 @@ class K8sCloverTest(k8stest.K8sTesting): if "case_name" not in kwargs: kwargs.get("case_name", 'clover_k8s') super(K8sCloverTest, self).__init__(**kwargs) - self.check_envs() def run_kubetest(self): - success = True + success = istio_validate.validateDeploy() if success: self.result = 100 - elif failure: + else: self.result = 0 - diff --git a/clover/servicemesh/validate.py b/clover/servicemesh/validate.py new file mode 100644 index 0000000..0e2e59a --- /dev/null +++ b/clover/servicemesh/validate.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright (c) Authors of Clover +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +from kubernetes import client, config + +ISTIO_NAMESPACE = "istio-system" +ISTIO_DEPLOYMENT = "istio-pilot" + + +def validateDeploy(): + config.load_kube_config() + appsv1 = client.AppsV1Api() + corev1 = client.CoreV1Api() + find_flag = False + + # check deploytment + ret = appsv1.list_deployment_for_all_namespaces(watch=False) + for i in ret.items: + if ISTIO_DEPLOYMENT == i.metadata.name and \ + ISTIO_NAMESPACE == i.metadata.namespace: + find_flag = True + break + if find_flag == False: + print("ERROR: Deployment: {} doesn't present in {} namespace".format( + ISTIO_DEPLOYMENT, ISTIO_NAMESPACE)) + return False + + return True + + +def main(): + if validateDeploy(): + print"Istio install validation has passed" + return True + else: + print"ERROR: Istio install validation has failed" + return False + + +if __name__ == '__main__': + main() diff --git a/clover/spinnaker/install/minio-pv.yml b/clover/spinnaker/install/minio-pv.yml new file mode 100644 index 0000000..6b5507d --- /dev/null +++ b/clover/spinnaker/install/minio-pv.yml @@ -0,0 +1,14 @@ +kind: PersistentVolume +apiVersion: v1 +metadata: + name: minio-pv-volume + labels: + type: local +spec: + storageClassName: standard + capacity: + storage: 10Gi + accessModes: + - ReadWriteOnce + hostPath: + path: "/mnt/minio" diff --git a/clover/spinnaker/install/quick-install-spinnaker.yml b/clover/spinnaker/install/quick-install-spinnaker.yml index c935453..31a2b27 100644 --- a/clover/spinnaker/install/quick-install-spinnaker.yml +++ b/clover/spinnaker/install/quick-install-spinnaker.yml @@ -227,7 +227,7 @@ data: timezone: America/Los_Angeles ci: jenkins: - enabled: true + enabled: false masters: [] travis: enabled: false diff --git a/clover/tools/jmeter/jmeter-master/Dockerfile b/clover/tools/jmeter/jmeter-master/Dockerfile index da0e474..5987003 100644 --- a/clover/tools/jmeter/jmeter-master/Dockerfile +++ b/clover/tools/jmeter/jmeter-master/Dockerfile @@ -7,7 +7,7 @@ FROM java:8 -RUN wget http://apache.mirrors.hoobly.com//jmeter/binaries/apache-jmeter-4.0.tgz +RUN wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-4.0.tgz RUN tar -xvzf apache-jmeter-4.0.tgz RUN rm apache-jmeter-4.0.tgz RUN mv apache-jmeter-4.0 /jmeter diff --git a/clover/tools/jmeter/jmeter-slave/Dockerfile b/clover/tools/jmeter/jmeter-slave/Dockerfile index b5ccbcd..fc06fe9 100644 --- a/clover/tools/jmeter/jmeter-slave/Dockerfile +++ b/clover/tools/jmeter/jmeter-slave/Dockerfile @@ -7,7 +7,7 @@ FROM java:8 -RUN wget http://apache.mirrors.hoobly.com//jmeter/binaries/apache-jmeter-4.0.tgz +RUN wget https://archive.apache.org/dist/jmeter/binaries/apache-jmeter-4.0.tgz RUN tar -xvzf apache-jmeter-4.0.tgz RUN rm apache-jmeter-4.0.tgz RUN mv apache-jmeter-4.0 /jmeter diff --git a/clover/tools/yaml/cassandra.yaml b/clover/tools/yaml/cassandra.yaml index 0206d75..dc1c46f 100644 --- a/clover/tools/yaml/cassandra.yaml +++ b/clover/tools/yaml/cassandra.yaml @@ -36,6 +36,7 @@ metadata: labels: app: cassandra name: cassandra + namespace: clover-system spec: clusterIP: None ports: @@ -49,6 +50,7 @@ metadata: name: cassandra labels: app: cassandra + namespace: clover-system spec: serviceName: cassandra replicas: 1 @@ -76,18 +78,18 @@ spec: name: cql resources: limits: - cpu: "500m" - memory: 1Gi + cpu: "1000m" + memory: 5Gi requests: - cpu: "500m" - memory: 1Gi + cpu: "1000m" + memory: 5Gi env: - name: MAX_HEAP_SIZE value: 512M - name: HEAP_NEWSIZE value: 100M - name: CASSANDRA_SEEDS - value: "cassandra-0.cassandra.default.svc.cluster.local" + value: "cassandra-0.cassandra.clover-system.svc.cluster.local" - name: CASSANDRA_CLUSTER_NAME value: "MyCassandraDemo" - name: CASSANDRA_DC diff --git a/docs/conf.yaml b/docs/conf.yaml index f871056..df012a4 100644 --- a/docs/conf.yaml +++ b/docs/conf.yaml @@ -1,3 +1,3 @@ --- project_cfg: opnfv -project: CLOVER +project: Clover diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4f9b38a --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,18 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 +.. International License. +.. SPDX-License-Identifier CC-BY-4.0 +.. (c) OPNFV, Authors of Clover + +.. _clover: + +============ +OPNFV Clover +============ + +.. toctree:: + :maxdepth: 1 + + release/userguide/index.rst + release/release-notes/index.rst + release/configguide/index.rst + development/design/index.rst diff --git a/docs/release/configguide/imgs/spinnaker.png b/docs/release/configguide/imgs/spinnaker.png Binary files differnew file mode 100644 index 0000000..95dc769 --- /dev/null +++ b/docs/release/configguide/imgs/spinnaker.png diff --git a/docs/release/configguide/spinnaker_config_guide.rst b/docs/release/configguide/spinnaker_config_guide.rst new file mode 100644 index 0000000..f4a3e12 --- /dev/null +++ b/docs/release/configguide/spinnaker_config_guide.rst @@ -0,0 +1,242 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. SPDX-License-Identifier CC-BY-4.0 +.. (c) Authors of Clover + +.. _spinnaker_config_guide: + +========================================= +Spinnaker Configuration Guide +========================================= + +This document provides a guide to setup the spinnaker in kubernetes as a continuous delivery platform. + + +Spinnaker Overview +===================== + +Spinnaker is an open-source, multi-cloud continuous delivery platform that helps you release software changes with high velocity and confidence. + +Spinnaker provides two core sets of features: + +**1. application management** + + You use Spinnaker’s application management features to view and manage your cloud resources. + +**2. application deployment** + + You use Spinnaker’s application deployment features to construct and manage continuous delivery workflows. + +For more information on Spinnaker and its capabilities, please refer to `documentation <https://www.spinnaker.io/>`_. + + +Setup Spinnaker +====================================== + +Prerequisites +------------- + +The following assumptions must be met before continuing on to deployment: + + * Ubuntu 16.04 was used heavily for development and is advised for greenfield deployments. + * Installation of Docker has already been performed. It's preferable to install Docker CE. + * Installation of Kubernetes has already been performed. + * A PersistentVolume resource need to be setup in k8s for the PersistentVolumeClaim to use. we supply the manifest file `minio-pv.yml <https://github.com/opnfv/clover/blob/master/clover/spinnaker/install/minio-pv.yml>`_ to create the PV, But it is not suitable for use in production. + + +Deploy from source +------------------ + +Clone the Clover git repository and navigate within the samples directory as shown below: + +.. code-block:: bash + + $ git clone https://gerrit.opnfv.org/gerrit/clover + $ cd clover/clover/spinnaker/install + $ git checkout stable/gambia + +To deploy the Spinnaker in the "spinnaker" Kubernetes namespace, use the following command: + +.. code-block:: bash + + $ kubectl create -f quick-install-spinnaker.yml + +**NOTE:** The quick-install-spinnaker.yml is obtained from https://www.spinnaker.io/downloads/kubernetes/quick-install.yml and modified. + +Verifying the deployment +------------------------ + +To verify the Spinnaker pods is deployed, executing the command below: + +.. code-block:: bash + + $ kubectl get pod -n spinnaker + +The listing below must include the following Spinnaker pods: + +.. code-block:: bash + + $ NAME READY STATUS RESTARTS AGE + minio-deployment-5d84f45dd5-zjdzb 1/1 Running 0 22h + spin-clouddriver-795575c5cb-ph8qc 1/1 Running 0 22h + spin-deck-7c5d75bfcd-vr58q 1/1 Running 0 22h + spin-echo-7986796c94-4285v 1/1 Running 0 22h + spin-front50-5744674fdc-d9xsw 1/1 Running 0 22h + spin-gate-7978d55d57-jcsmq 1/1 Running 0 22h + spin-halyard 1/1 Running 0 22h + spin-igor-6f8c86bbbb-cs8gd 1/1 Running 0 22h + spin-orca-8659c57c5c-rs69z 1/1 Running 0 22h + spin-redis-558db8d5bd-kdmjz 1/1 Running 0 22h + spin-rosco-dfbbcbccd-db65b 1/1 Running 0 22h + +To verify the Spinnaker services is created, executing the command below: + +.. code-block:: bash + + $ kubectl get svc -n spinnaker + +The listing below must include the following Spinnaker services: + +.. code-block:: bash + + $ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + minio-service ClusterIP 10.233.21.175 <none> 9000/TCP 22h + spin-clouddriver ClusterIP 10.233.9.27 <none> 7002/TCP 22h + spin-deck ClusterIP 10.233.34.86 <none> 9000/TCP 22h + spin-echo ClusterIP 10.233.29.150 <none> 8089/TCP 22h + spin-front50 ClusterIP 10.233.5.221 <none> 8080/TCP 22h + spin-gate ClusterIP 10.233.33.196 <none> 8084/TCP 22h + spin-halyard ClusterIP 10.233.2.187 <none> 8064/TCP 22h + spin-igor ClusterIP 10.233.29.93 <none> 8088/TCP 22h + spin-orca ClusterIP 10.233.23.140 <none> 8083/TCP 22h + spin-redis ClusterIP 10.233.20.95 <none> 6379/TCP 22h + spin-rosco ClusterIP 10.233.48.79 <none> 8087/TCP 22h + +To publish the spin-deck service, we need change the type to NodePort, executing the command below: + +.. code-block:: bash + + $ kubectl get svc spin-deck -n spinnaker -o yaml |sed 's/ClusterIP/NodePort/' |kubectl replace -f - + $ kubectl get svc -n spinnaker + +The listing below must include the following services + +.. code-block:: bash + + $ NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + minio-service ClusterIP 10.233.21.175 <none> 9000/TCP 22h + spin-clouddriver ClusterIP 10.233.9.27 <none> 7002/TCP 22h + spin-deck NodePort 10.233.34.86 <none> 9000:31747/TCP 22h + spin-echo ClusterIP 10.233.29.150 <none> 8089/TCP 22h + spin-front50 ClusterIP 10.233.5.221 <none> 8080/TCP 22h + spin-gate ClusterIP 10.233.33.196 <none> 8084/TCP 22h + spin-halyard ClusterIP 10.233.2.187 <none> 8064/TCP 22h + spin-igor ClusterIP 10.233.29.93 <none> 8088/TCP 22h + spin-orca ClusterIP 10.233.23.140 <none> 8083/TCP 22h + spin-redis ClusterIP 10.233.20.95 <none> 6379/TCP 22h + spin-rosco ClusterIP 10.233.48.79 <none> 8087/TCP 22h + +In your browser, navigate to the following URLs for Spinnaker respectively:: + + http://<node IP>:31747 + +Where node IP is an IP from one of the Kubernetes cluster node(s). + +.. image:: imgs/spinnaker.png + :align: center + :scale: 100% + +Spinnaker Configuration +======================= + +When the default installation is ready, there are many different components that you can turn on with Spinnaker. In order to customize Spinnaker, you can use the halyard command line or clover command line to edit the configuration and apply it to what has already been deployed. + +Halyard Command +--------------- + +Halyard has an in-cluster daemon that stores your configuration. You can exec a shell in this pod to make and apply your changes. + +For example: + +.. code-block:: bash + + $ kubectl exec spin-halyard -n spinnaker -it -- bash -il + spinnaker@spin-halyard:/workdir$ hal version list + +How to use the halyard command line to configurate the spinnaker, please refer to `commands documentation <https://www.spinnaker.io/reference/halyard/commands/>`_. + +Clover Command +-------------- + +Clover provider the cloverctl and clover-controller to controll the server. So we can use the cloverctl to configurate the spinnaker. So far, clover provide the capabilities to create/get/delete docker-registry and kubernetes provider in spinnaker. + +**NOTE:** Before using clover command, you need build the clover command and setup the clover-controller in your local kubernetes cluster, where spinnaker deploy in. + +Docker Registry +::::::::::::::: + +You need a configuration file written in YAML that describe the information about you Docker Registry as shown below: + +docker.yml:: + + name: mydockerhub + address: https://index.docker.io + username: if-you-images-aren't-publicly-available + password: fill-this-field + repositories: + - opnfv/clover + +If any of your images aren’t publicly available, you need fill your DockerHub username & password. Ortherwise you can delete the username & password field. + +Creating the Docker Registry in spinnaker: + +.. code-block:: bash + + $ cloverctl create provider docker-registry -f docker.yml + +Getting the Docker Registry in spinnaker: + +.. code-block:: bash + + $ cloverctl get provider docker-registry + +Deleting the Docker Registry in spinnaker: + +.. code-block:: bash + + $ cloverctl delete provider docker-registry -n dockerhub + +Kubernetes +:::::::::: + +By default, installing the manifest only registers the local cluster as a deploy target for Spinnaker. If you want to add arbitrary clusters you can use the cloverctl command + +You need a running Kubernetes cluster, with corresponding credentials in a kubeconfig file(/path/to/kubeconfig). And You also need a configuration file written in YAML that describe the information about your kubernetes cluseter as shown below: + +kubernetes.yml:: + + # name must match pattern ^[a-z0-9]+([-a-z0-9]*[a-z0-9])?$' + name: my-kubernetes + providerVersion: V1 + # make sure the kubeconfigFile can be use + kubeconfigFile: /path/to/kubeconfig + dockerRegistries: + - accountName: dockerhub + +Creating the kubernetes provider in spinnaker: + +.. code-block:: bash + + $ cloverctl create provider kubernetes -f kubernetes.yml + +Getting the kubernetes provider in spinnaker: + +.. code-block:: bash + + $ cloverctl get provider kubernetes + +Deleting the kubernetes provider in spinnaker: + +.. code-block:: bash + + $ cloverctl delete provider kubernetes -n my-kubernetes diff --git a/requirements.txt b/requirements.txt index eeaa324..670afbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -kubernetes>=4.0.0 # Apache-2.0 -pbr!=2.1.0,>=2.0.0 # Apache-2.0 +kubernetes # Apache-2.0 +pbr!=2.1.0 # Apache-2.0 sh # MIT xtesting # Apache-2.0 -redis>=2.10.0 # MIT +redis # MIT diff --git a/samples/scenarios/service_delivery_controller.yaml b/samples/scenarios/service_delivery_controller.yaml index b9c3506..34e2df0 100644 --- a/samples/scenarios/service_delivery_controller.yaml +++ b/samples/scenarios/service_delivery_controller.yaml @@ -226,6 +226,7 @@ spec: labels: app: http-lb version: v1 + name: http-lb-v1 spec: containers: - name: http-lb @@ -247,6 +248,7 @@ spec: labels: app: http-lb version: v2 + name: http-lb-v2 spec: containers: - name: http-lb @@ -263,13 +265,33 @@ metadata: app: http-lb spec: ports: - - port: 50054 - name: grpc - port: 9180 name: http selector: app: http-lb --- +apiVersion: v1 +kind: Service +metadata: + name: http-lb-v1 +spec: + ports: + - port: 50054 + name: grpc + selector: + name: http-lb-v1 +--- +apiVersion: v1 +kind: Service +metadata: + name: http-lb-v2 +spec: + ports: + - port: 50054 + name: grpc + selector: + name: http-lb-v2 +--- apiVersion: extensions/v1beta1 kind: Deployment metadata: diff --git a/samples/scenarios/service_delivery_controller_opnfv.yaml b/samples/scenarios/service_delivery_controller_opnfv.yaml index ceba36f..cb4743f 100644 --- a/samples/scenarios/service_delivery_controller_opnfv.yaml +++ b/samples/scenarios/service_delivery_controller_opnfv.yaml @@ -226,6 +226,7 @@ spec: labels: app: http-lb version: v1 + name: http-lb-v1 spec: containers: - name: http-lb @@ -247,6 +248,7 @@ spec: labels: app: http-lb version: v2 + name: http-lb-v2 spec: containers: - name: http-lb @@ -263,13 +265,33 @@ metadata: app: http-lb spec: ports: - - port: 50054 - name: grpc - port: 9180 name: http selector: app: http-lb --- +apiVersion: v1 +kind: Service +metadata: + name: http-lb-v1 +spec: + ports: + - port: 50054 + name: grpc + selector: + name: http-lb-v1 +--- +apiVersion: v1 +kind: Service +metadata: + name: http-lb-v2 +spec: + ports: + - port: 50054 + name: grpc + selector: + name: http-lb-v2 +--- apiVersion: extensions/v1beta1 kind: Deployment metadata: diff --git a/samples/services/nginx/docker/grpc/templates/server.template b/samples/services/nginx/docker/grpc/templates/server.template index c1582fa..2ca5410 100644 --- a/samples/services/nginx/docker/grpc/templates/server.template +++ b/samples/services/nginx/docker/grpc/templates/server.template @@ -116,7 +116,7 @@ http { } location @return_config { - proxy_pass http://clover-controller:80; + proxy_pass http://clover-controller.clover-system:80; proxy_http_version 1.1; } |
