From da9564a9b0b78bbe341de9b039aab3c378eb027f Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Mon, 22 Jan 2018 15:25:53 -0800 Subject: Implement component deployment via cloudify JIRA: VES-2 Change-Id: Ic696f13d2a32e10663f50cd4e26b9a060525ff92 Signed-off-by: Bryan Sullivan --- build/ves-agent.sh | 5 +- build/ves-agent/Dockerfile | 29 +- build/ves-barometer.sh | 51 ++ build/ves-barometer/Dockerfile | 38 ++ build/ves-barometer/start.sh | 151 ++++++ build/ves-collector.sh | 8 +- build/ves-collector/Dashboard.json | 996 +++++++++++++++++++++++++++++++++++++ build/ves-collector/Dockerfile | 14 +- build/ves-collector/start.sh | 59 ++- build/ves-kafka.sh | 2 +- build/ves-kafka/start.sh | 6 +- 11 files changed, 1313 insertions(+), 46 deletions(-) create mode 100644 build/ves-barometer.sh create mode 100644 build/ves-barometer/Dockerfile create mode 100644 build/ves-barometer/start.sh create mode 100644 build/ves-collector/Dashboard.json (limited to 'build') diff --git a/build/ves-agent.sh b/build/ves-agent.sh index 87d4b07..396d7ea 100644 --- a/build/ves-agent.sh +++ b/build/ves-agent.sh @@ -19,7 +19,7 @@ #. Docker hub user logged in e.g. via "sudo docker login" #. #. Usage: -#. bash ves-agent.sh +#. bash ves-agent.sh #. hub-user: username for dockerhub #. #. NOTE: To allow patch testing, this script will not reclone the VES repo @@ -31,9 +31,6 @@ echo; echo "$0 $(date): Update package repos" sudo apt-get update echo; echo "$0 $(date): Starting VES agent build process" -if [[ -d /tmp/ves ]]; then rm -rf /tmp/ves; fi - -echo; echo "$0 $(date): Starting VES kafka build process" if [[ ! -d /tmp/ves ]]; then echo; echo "$0 $(date): Cloning VES repo to /tmp/ves" git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves diff --git a/build/ves-agent/Dockerfile b/build/ves-agent/Dockerfile index 4c37197..293fcd5 100644 --- a/build/ves-agent/Dockerfile +++ b/build/ves-agent/Dockerfile @@ -21,26 +21,19 @@ FROM ubuntu:xenial MAINTAINER Bryan Sullivan -RUN apt-get update -RUN apt-get install -y apt-utils +RUN mkdir /opt/ves + +RUN apt-get update && apt-get install -y apt-utils RUN apt-get -y upgrade -RUN apt-get install -y git -# Required for kafka -RUN apt-get install -y default-jre -RUN apt-get install -y zookeeperd -RUN apt-get install -y python-pip +# Required for kafka: default-jre zookeeperd python-pip kafka-python +# Required for building librdkafka: git build-essential libpthread-stubs0-dev libssl-dev libsasl2-dev liblz4-dev +# Required for building collectd: pkg-config +RUN apt-get update && apt-get install -y default-jre \ +zookeeperd python-pip pkg-config \ +git build-essential libpthread-stubs0-dev libssl-dev libsasl2-dev liblz4-dev RUN pip install kafka-python -# Required for building collectd -RUN apt-get install -y pkg-config - -RUN mkdir /opt/ves # Build Kafka client -RUN apt-get install -y build-essential -RUN apt-get install -y libpthread-stubs0-dev -RUN apt-get install -y libssl-dev -RUN apt-get install -y libsasl2-dev -RUN apt-get install -y liblz4-dev RUN /bin/bash -c 'git clone --branch v0.9.5 \ https://github.com/edenhill/librdkafka.git /opt/ves/librdkafka; \ cd /opt/ves/librdkafka; ./configure --prefix=/usr; \ @@ -50,10 +43,6 @@ make; make install' RUN pip install pyaml RUN git clone https://gerrit.opnfv.org/gerrit/barometer /opt/ves/barometer -# Test patch -RUN /bin/bash -c 'cd /opt/ves/barometer; \ -git fetch https://gerrit.opnfv.org/gerrit/barometer \ -refs/changes/27/47427/1 && git checkout FETCH_HEAD' COPY start.sh /opt/ves/start.sh ENTRYPOINT ["/bin/bash", "/opt/ves/start.sh"] diff --git a/build/ves-barometer.sh b/build/ves-barometer.sh new file mode 100644 index 0000000..86e81f4 --- /dev/null +++ b/build/ves-barometer.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2018 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#. What this is: Build script for the OPNFV Barometer collectd agent docker image. +#. +#. Prerequisites: +#. Docker hub user logged in e.g. via "sudo docker login" +#. +#. Usage: +#. bash ves-barometer.sh [--no-cache] +#. hub-user: username for dockerhub +#. --no-cache +#. +#. NOTE: To allow patch testing, this script will not reclone the VES repo +#. if it exists under /tmp +#. +#. Status: this is a work in progress, under test. + +cache="$2" +echo; echo "$0 $(date): Update package repos" +sudo apt-get update + +echo; echo "$0 $(date): Starting VES agent build process" +if [[ ! -d /tmp/ves ]]; then + echo; echo "$0 $(date): Cloning VES repo to /tmp/ves" + git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves +fi + +echo; echo "$0 $(date): Building the image" +cd /tmp/ves/build/ves-barometer +sudo docker build $cache -t ves-barometer . + +echo; echo "$0 $(date): Tagging the image" +id=$(sudo docker images | grep ves-barometer | awk '{print $3}') +id=$(echo $id | cut -d ' ' -f 1) +sudo docker tag $id $1/ves-barometer:latest + +echo; echo "$0 $(date): Pushing the image to dockerhub as $1/ves-barometer" +sudo docker push $1/ves-barometer diff --git a/build/ves-barometer/Dockerfile b/build/ves-barometer/Dockerfile new file mode 100644 index 0000000..4bd4c51 --- /dev/null +++ b/build/ves-barometer/Dockerfile @@ -0,0 +1,38 @@ +# Copyright 2018 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# What this is: A Dockerfile for building an OPFNV VES Agent container image. +# +# Status: this is a work in progress, under test. +# +FROM centos:7 +RUN yum update -y && yum install -y which sudo git +ENV DOCKER y +ENV repos_dir /src +ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins +RUN git config --global http.sslVerify false + +WORKDIR ${repos_dir} +RUN git clone https://gerrit.opnfv.org/gerrit/barometer +WORKDIR ${repos_dir}/barometer/systems +RUN sh ./build_base_machine.sh + +RUN useradd -ms /bin/bash collectd_exec +RUN echo "collectd_exec ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +WORKDIR ${openstack_plugins} +RUN make + +COPY start.sh /opt/ves/start.sh +ENTRYPOINT ["/bin/bash", "/opt/ves/start.sh"] diff --git a/build/ves-barometer/start.sh b/build/ves-barometer/start.sh new file mode 100644 index 0000000..da452bf --- /dev/null +++ b/build/ves-barometer/start.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# Copyright 2018 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# What this is: Startup script for the OPNFV Barometer collectd agent running +# under docker. + +rm -f /opt/collectd/etc/collectd.conf.d/* + +if [[ "$ves_mode" == "node" ]]; then + cat </opt/collectd/etc/collectd.conf.d/collectd.conf +# for VES plugin +LoadPlugin logfile + + LogLevel debug + File STDOUT + Timestamp true + PrintSeverity false + + +LoadPlugin csv + + DataDir "/work-dir/collectd/install/var/lib/csv" + StoreRates false + + +LoadPlugin target_set +LoadPlugin match_regex + + + + Plugin "^memory$" + + + PluginInstance "host" + + + + +LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + + +LoadPlugin interface +LoadPlugin memory +LoadPlugin load +LoadPlugin disk +# TODO: how to set this option only to apply to VMs (not nodes) +LoadPlugin uuid + +LoadPlugin write_kafka + + Property "metadata.broker.list" "$ves_kafka_hostname:$ves_kafka_port" + + Format JSON + + +EOF + + if [[ -d /etc/nova ]]; then + cat <>~/collectd/collectd.conf +LoadPlugin virt + + Connection "qemu:///system" + RefreshInterval 60 + HostnameFormat uuid + PluginInstanceFormat name + ExtraStats "cpu_util" + +EOF + fi +else + cat </opt/collectd/etc/collectd.conf.d/collectd.conf +# for VES plugin +LoadPlugin logfile + + LogLevel debug + File STDOUT + Timestamp true + PrintSeverity false + + +LoadPlugin cpu + + ReportByCpu true + ReportByState true + ValuesPercentage true + + +LoadPlugin csv + + DataDir "/tmp" + + +LoadPlugin interface +LoadPlugin memory +LoadPlugin load +LoadPlugin disk +LoadPlugin uuid + +LoadPlugin write_kafka + + Property "metadata.broker.list" "$ves_kafka_hostname:$ves_kafka_port" + + Format JSON + + + +LoadPlugin target_set +LoadPlugin match_regex + + + + Plugin "^memory$" + + + PluginInstance "guest" + + + +EOF +fi + +echo; echo "cat /opt/collectd/etc/collectd.conf.d/collectd.conf" +cat /opt/collectd/etc/collectd.conf.d/collectd.conf + +#echo "Delete conf files causing collectd to fail" +#rm -f /opt/collectd/etc/collectd.conf.d/dpdk*.conf +#rm -f /opt/collectd/etc/collectd.conf.d/snmp*.conf +#rm -f /opt/collectd/etc/collectd.conf.d/virt.conf +#rm -f /opt/collectd/etc/collectd.conf.d/mcelog.conf +#rm -f /opt/collectd/etc/collectd.conf.d/rdt.conf +#sed -i -- 's/LoadPlugin cpufreq/#LoadPlugin cpufreq/' /opt/collectd/etc/collectd.conf.d/default_plugins.conf + +/opt/collectd/sbin/collectd -f +echo "collectd has exited. sleeping for an hour to enable debugging" +sleep 3600 diff --git a/build/ves-collector.sh b/build/ves-collector.sh index 58aa354..a09eeec 100644 --- a/build/ves-collector.sh +++ b/build/ves-collector.sh @@ -31,10 +31,10 @@ echo; echo "$0 $(date): Update package repos" sudo apt-get update echo; echo "$0 $(date): Starting VES collector build process" -if [[ -d /tmp/ves ]]; then rm -rf /tmp/ves; fi - -echo; echo "$0 $(date): Cloning VES repo to /tmp/ves" -git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves +if [[ ! -d /tmp/ves ]]; then + echo; echo "$0 $(date): Cloning VES repo to /tmp/ves" + git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves +fi echo; echo "$0 $(date): Building the image" cd /tmp/ves/build/ves-collector diff --git a/build/ves-collector/Dashboard.json b/build/ves-collector/Dashboard.json new file mode 100644 index 0000000..b88646c --- /dev/null +++ b/build/ves-collector/Dashboard.json @@ -0,0 +1,996 @@ +{ +"dashboard": { + "description": "This Dashboard provides a general overview of a host, with templating to select the hostname.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": 401, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 3, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + } + ], + "measurement": "load", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load-shortterm" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "host load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Percent", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 6, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "cpu" + ], + "type": "tag" + } + ], + "measurement": "cpuUsage", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"cpusystem\") FROM \"cpu\" WHERE $timeFilter GROUP BY time(1m) fill(null)", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cpuUsageUser" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "host CPU Usage User", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Percent", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 442, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 2, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "vnic" + ], + "type": "tag" + } + ], + "measurement": "vNicPerformance", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT derivative(mean(\"rxoctetsacc\"), 10s) FROM \"vnic\" WHERE \"system\" = 'computehost' AND $timeFilter GROUP BY time(1m) fill(null)", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "receivedTotalPacketsAccumulated" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Received Octets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Octets/Packets", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "grid": { + "leftLogBase": 1, + "leftMax": null, + "leftMin": null, + "rightLogBase": 1, + "rightMax": null, + "rightMin": null + }, + "id": 4, + "interval": "30s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "vnic" + ], + "type": "tag" + } + ], + "measurement": "vNicPerformance", + "orderByTime": "ASC", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "receivedOctetsAccumulated" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmitted Octets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "short", + "label": "Octets/Packets", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 362, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "disk" + ], + "type": "tag" + } + ], + "measurement": "diskUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "diskOpsWriteLast" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "disk", + "operator": "=", + "value": "sda" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage SDA", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + }, + { + "params": [ + "disk" + ], + "type": "tag" + } + ], + "measurement": "diskUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "diskOpsWriteLast" + ], + "type": "field" + }, + { + "params": [ + "5" + ], + "type": "moving_average" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + }, + { + "condition": "AND", + "key": "disk", + "operator": "=", + "value": "sdb" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Usage SDB", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "VESEvents", + "fill": 1, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "system" + ], + "type": "tag" + } + ], + "measurement": "memoryUsage", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "memoryUsed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "system", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "VESEvents", + "hide": 0, + "includeAll": true, + "label": "host", + "multi": true, + "name": "host", + "options": [], + "query": "SHOW TAG VALUES WITH KEY=system", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "10s", + "20s", + "30s", + "1m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "VES Demo", + "version": 4 +} +} diff --git a/build/ves-collector/Dockerfile b/build/ves-collector/Dockerfile index 9161871..4cd135f 100644 --- a/build/ves-collector/Dockerfile +++ b/build/ves-collector/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# What this is: A Dockerfile for building an OPFNV VES Agent container image. +# What this is: A Dockerfile for building an OPFNV VES Collector container image. # # Status: this is a work in progress, under test. # @@ -21,14 +21,9 @@ FROM ubuntu:xenial MAINTAINER Bryan Sullivan -RUN apt-get update -RUN apt-get install -y apt-utils +RUN apt-get update && apt-get install -y apt-utils RUN apt-get -y upgrade -RUN apt-get update -RUN apt-get install -y git - -# Required for VES collector -RUN apt-get install -y python-pip python-jsonschema +RUN apt-get update && apt-get install -y git python-pip python-jsonschema curl RUN pip install requests RUN mkdir /opt/ves @@ -37,5 +32,6 @@ RUN mkdir /opt/ves RUN git clone https://github.com/att/evel-test-collector.git /opt/ves/evel-test-collector COPY monitor.py /opt/ves/evel-test-collector/code/collector/monitor.py +COPY Dashboard.json /opt/ves/Dashboard.json COPY start.sh /opt/ves/start.sh ENTRYPOINT ["/bin/bash", "/opt/ves/start.sh"] diff --git a/build/ves-collector/start.sh b/build/ves-collector/start.sh index be30c9a..250af34 100644 --- a/build/ves-collector/start.sh +++ b/build/ves-collector/start.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2017 AT&T Intellectual Property, Inc +# Copyright 2017-2018 AT&T Intellectual Property, Inc # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,6 +23,8 @@ sed -i -- \ evel-test-collector/config/collector.conf sed -i -- "s/vel_domain = 127.0.0.1/vel_domain = $ves_host/g" \ evel-test-collector/config/collector.conf +sed -i -- "s/vel_port = 30000/vel_port = $ves_port/g" \ + evel-test-collector/config/collector.conf sed -i -- "s/vel_username =/vel_username = $ves_user/g" \ evel-test-collector/config/collector.conf sed -i -- "s/vel_password =/vel_password = $ves_pass/g" \ @@ -31,18 +33,65 @@ sed -i -- "s~vel_path = vendor_event_listener/~vel_path = $ves_path~g" \ evel-test-collector/config/collector.conf sed -i -- "s~vel_topic_name = example_vnf~vel_topic_name = $ves_topic~g" \ evel-test-collector/config/collector.conf -sed -i -- "/vel_topic_name = /a influxdb = $ves_influxdb_host" \ +sed -i -- "/vel_topic_name = /a influxdb = $ves_influxdb_host:$ves_influxdb_port" \ evel-test-collector/config/collector.conf +echo; echo "evel-test-collector/config/collector.conf" +cat evel-test-collector/config/collector.conf + +echo; echo "wait for InfluxDB API at $ves_influxdb_host:$ves_influxdb_port" +while ! curl http://$ves_influxdb_host:$ves_influxdb_port/ping ; do + echo "InfluxDB API is not yet responding... waiting 10 seconds" + sleep 10 +done + +echo; echo "setup veseventsdb in InfluxDB" +# TODO: check if pre-existing and skip +curl -X POST http://$ves_influxdb_host:$ves_influxdb_port/query \ + --data-urlencode "q=CREATE DATABASE veseventsdb" + +echo; echo "wait for Grafana API to be active" +while ! curl http://$ves_grafana_host:$ves_grafana_port ; do + echo "Grafana API is not yet responding... waiting 10 seconds" + sleep 10 +done + +echo; echo "add VESEvents datasource to Grafana" +# TODO: check if pre-existing and skip +cat </opt/ves/datasource.json +{ "name":"VESEvents", + "type":"influxdb", + "access":"direct", + "url":"http://$ves_influxdb_host:$ves_influxdb_port", + "password":"root", + "user":"root", + "database":"veseventsdb", + "basicAuth":false, + "basicAuthUser":"", + "basicAuthPassword":"", + "withCredentials":false, + "isDefault":false, + "jsonData":null +} +EOF + +curl -H "Accept: application/json" -H "Content-type: application/json" \ + -X POST -d @/opt/ves/datasource.json \ + http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/datasources + +echo; echo "add VES dashboard to Grafana" +curl -H "Accept: application/json" -H "Content-type: application/json" \ + -X POST -d @/opt/ves/Dashboard.json \ + http://$ves_grafana_auth@$ves_grafana_host:$ves_grafana_port/api/dashboards/db + if [[ "$ves_loglevel" != "" ]]; then python /opt/ves/evel-test-collector/code/collector/monitor.py \ --config /opt/ves/evel-test-collector/config/collector.conf \ - --influxdb $ves_influxdb_host \ + --influxdb $ves_influxdb_host:$ves_influxdb_port \ --section default > /opt/ves/monitor.log 2>&1 else python /opt/ves/evel-test-collector/code/collector/monitor.py \ --config /opt/ves/evel-test-collector/config/collector.conf \ - --influxdb $ves_influxdb_host \ + --influxdb $ves_influxdb_host:$ves_influxdb_port \ --section default fi - diff --git a/build/ves-kafka.sh b/build/ves-kafka.sh index 19a632b..c489535 100644 --- a/build/ves-kafka.sh +++ b/build/ves-kafka.sh @@ -19,7 +19,7 @@ #. Docker hub user logged in e.g. via "sudo docker login" #. #. Usage: -#. bash ves-kafka.sh +#. bash ves-kafka.sh #. hub-user: username for dockerhub #. #. NOTE: To allow patch testing, this script will not reclone the VES repo diff --git a/build/ves-kafka/start.sh b/build/ves-kafka/start.sh index ab4169b..37c36c2 100644 --- a/build/ves-kafka/start.sh +++ b/build/ves-kafka/start.sh @@ -16,14 +16,14 @@ #. What this is: Startup script for a kafka server as used by the OPNFV VES #. framework. -echo "$zookeeper $zookeeper_host" >>/etc/hosts +echo "$zookeeper_host $zookeeper_hostname" >>/etc/hosts cat /etc/hosts cd /opt/ves -sed -i "s/localhost:2181/$zookeeper_host:2181/" \ +sed -i "s/localhost:2181/$zookeeper_hostname:$zookeeper_port/" \ kafka_2.11-0.11.0.2/config/server.properties grep 2181 kafka_2.11-0.11.0.2/config/server.properties -sed -i "s~#advertised.listeners=PLAINTEXT://your.host.name:9092~advertised.listeners=PLAINTEXT://$kafka_hostname:9092~" \ +sed -i "s~#advertised.listeners=PLAINTEXT://your.host.name:9092~advertised.listeners=PLAINTEXT://$kafka_hostname:$kafka_port~" \ kafka_2.11-0.11.0.2/config/server.properties grep advertised.listeners kafka_2.11-0.11.0.2/config/server.properties -- cgit 1.2.3-korg