summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMaryam Tahhan <maryam.tahhan@intel.com>2017-11-29 11:48:19 +0000
committerMaryam Tahhan <maryam.tahhan@intel.com>2017-11-29 14:01:54 +0000
commitbfb1d7536e376244a48275afdbb5693c80571ed1 (patch)
tree77b8d6da9e1c3a14df6c1a636a6193f6e436ccb5
parent016ec072effd1638b52179eb056fda2891bd3777 (diff)
docker: add kafka support to collectd
Adds kafka support to collectd, cleaned up configuration files and updates documentation. Change-Id: Ie66ed22e8904e08e59aa7047d6afc9d176f3fb02 Signed-off-by: Maryam Tahhan <maryam.tahhan@intel.com>
-rw-r--r--docs/release/userguide/docker.userguide.rst11
-rw-r--r--src/Makefile1
-rw-r--r--src/collectd/Makefile1
-rw-r--r--src/collectd/collectd_sample_configs/csv.conf2
-rw-r--r--src/collectd/collectd_sample_configs/default_plugins.conf31
-rw-r--r--src/collectd/collectd_sample_configs/dpdkstat.conf2
-rw-r--r--src/collectd/collectd_sample_configs/exec.conf5
-rw-r--r--src/collectd/collectd_sample_configs/hugepages.conf2
-rw-r--r--src/collectd/collectd_sample_configs/kafka.conf21
-rw-r--r--src/collectd/collectd_sample_configs/ovs_events.conf3
-rw-r--r--src/collectd/collectd_sample_configs/rdt.conf2
-rwxr-xr-xsrc/collectd/collectd_sample_configs/write_notification.sh2
-rw-r--r--src/librdkafka/Makefile75
-rw-r--r--src/package-list.mk3
-rwxr-xr-xsystems/centos/7/build_base_machine.sh1
15 files changed, 146 insertions, 16 deletions
diff --git a/docs/release/userguide/docker.userguide.rst b/docs/release/userguide/docker.userguide.rst
index c6f56b2..46590eb 100644
--- a/docs/release/userguide/docker.userguide.rst
+++ b/docs/release/userguide/docker.userguide.rst
@@ -47,6 +47,7 @@ to the standard collectd plugins):
* RDT plugin
* virt
* SNMP Agent
+* Kafka_write plugin
Plugins and third party applications in Barometer repository that will be available in the
docker image:
@@ -251,13 +252,13 @@ Run the barometer docker image:
The docker barometer image contains configuration for all the collectd plugins. In the command
above we are overriding /opt/collectd/etc/collectd.conf.d by mounting a host directory
`pwd`/../src/collectd_sample_configs thta contains only the sample configurations we are interested
- in running.
+ in running. It's important to do this if you don't have DPDK, or RDT installed on the host.
To make some changes run:
.. code:: bash
- sudo docker exec -tid barometer_image /bin/bash
+ sudo docker exec -ti barometer_image /bin/bash
Check your docker image is running
@@ -267,8 +268,8 @@ Check your docker image is running
Build the influxdb + Grafana docker images
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Start by installing docker compose:
+On the node where you want to run influxdb + grafana:
+1. Start by installing docker compose
.. code:: bash
@@ -309,7 +310,7 @@ Start by installing docker compose:
.. code:: bash
- $ ./configure_grafana.sh
+ $ cd dashboards && ./configure_grafana.sh
8. Connect to <host_ip>:3000 with a browser and log into grafana: admin/admin
diff --git a/src/Makefile b/src/Makefile
index 4785e85..7c6c17c 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -40,6 +40,7 @@ SUBDIRS += dpdk
endif
SUBDIRS += libpqos
SUBDIRS += pmu-tools
+SUBDIRS += librdkafka
SUBDIRS += collectd
SUBDIRS += collectd-openstack-plugins
diff --git a/src/collectd/Makefile b/src/collectd/Makefile
index decb8e8..41fe89a 100644
--- a/src/collectd/Makefile
+++ b/src/collectd/Makefile
@@ -48,6 +48,7 @@ CONFIG_CMD += --enable-exec
CONFIG_CMD += --enable-python
CONFIG_CMD += --enable-threshold
CONFIG_CMD += --disable-perl
+CONFIG_CMD += --with-librdkafka=/usr
.PHONY: install force_install config force_make
diff --git a/src/collectd/collectd_sample_configs/csv.conf b/src/collectd/collectd_sample_configs/csv.conf
index e235003..99a8498 100644
--- a/src/collectd/collectd_sample_configs/csv.conf
+++ b/src/collectd/collectd_sample_configs/csv.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/default_plugins.conf b/src/collectd/collectd_sample_configs/default_plugins.conf
new file mode 100644
index 0000000..c96b007
--- /dev/null
+++ b/src/collectd/collectd_sample_configs/default_plugins.conf
@@ -0,0 +1,31 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#Hostname ""
+
+LoadPlugin cpufreq
+LoadPlugin disk
+LoadPlugin ethstat
+LoadPlugin ipc
+LoadPlugin ipmi
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin numa
+LoadPlugin processes
+LoadPlugin df
+LoadPlugin turbostat
+LoadPlugin uptime
+LoadPlugin contextswitch
+LoadPlugin irq
+LoadPlugin df
+LoadPlugin swap
diff --git a/src/collectd/collectd_sample_configs/dpdkstat.conf b/src/collectd/collectd_sample_configs/dpdkstat.conf
index 201f9d3..919e6e6 100644
--- a/src/collectd/collectd_sample_configs/dpdkstat.conf
+++ b/src/collectd/collectd_sample_configs/dpdkstat.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/exec.conf b/src/collectd/collectd_sample_configs/exec.conf
index e36409e..0a291bd 100644
--- a/src/collectd/collectd_sample_configs/exec.conf
+++ b/src/collectd/collectd_sample_configs/exec.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,9 +11,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-
LoadPlugin exec
+
<Plugin exec>
# For OVS PMD stats plugin
Exec "collectd_exec" "/opt/collectd/etc/collectd.conf.d/ovs_pmd_stats.sh"
diff --git a/src/collectd/collectd_sample_configs/hugepages.conf b/src/collectd/collectd_sample_configs/hugepages.conf
index 4e4d915..558b6f5 100644
--- a/src/collectd/collectd_sample_configs/hugepages.conf
+++ b/src/collectd/collectd_sample_configs/hugepages.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/kafka.conf b/src/collectd/collectd_sample_configs/kafka.conf
new file mode 100644
index 0000000..f81e87f
--- /dev/null
+++ b/src/collectd/collectd_sample_configs/kafka.conf
@@ -0,0 +1,21 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+LoadPlugin write_kafka
+
+<Plugin write_kafka>
+ Property "metadata.broker.list" "localhost:9092"
+ <Topic "collectd">
+ Format JSON
+ </Topic>
+</Plugin>
diff --git a/src/collectd/collectd_sample_configs/ovs_events.conf b/src/collectd/collectd_sample_configs/ovs_events.conf
index a52842c..250e1e4 100644
--- a/src/collectd/collectd_sample_configs/ovs_events.conf
+++ b/src/collectd/collectd_sample_configs/ovs_events.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
<LoadPlugin ovs_events>
Interval 1
</LoadPlugin>
diff --git a/src/collectd/collectd_sample_configs/rdt.conf b/src/collectd/collectd_sample_configs/rdt.conf
index 4ff0a77..ae983dc 100644
--- a/src/collectd/collectd_sample_configs/rdt.conf
+++ b/src/collectd/collectd_sample_configs/rdt.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/write_notification.sh b/src/collectd/collectd_sample_configs/write_notification.sh
index 28198f9..ed6ed9e 100755
--- a/src/collectd/collectd_sample_configs/write_notification.sh
+++ b/src/collectd/collectd_sample_configs/write_notification.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/src/librdkafka/Makefile b/src/librdkafka/Makefile
new file mode 100644
index 0000000..ea5e34e
--- /dev/null
+++ b/src/librdkafka/Makefile
@@ -0,0 +1,75 @@
+# makefile to manage collectd package
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Contributors:
+# Aihua Li, Huawei Technologies.
+# Maryam Tahhan, Intel Corporation.
+
+include ../mk/master.mk
+include ../package-list.mk
+
+WORK_DIR = librdkafka
+TAG_DONE_FLAG = $(WORK_DIR)/.$(KAFKA_TAG).done
+CONFIG_CMD =
+CONFIG_CMD += ./configure
+CONFIG_CMD += --prefix=/usr
+
+.PHONY: install force_install force_make
+
+# install depends on make
+force_install: force_make
+
+all: force_make
+ @echo "Finished making $(WORK_DIR) "
+
+INSTALL_TARGET = force_install force_make
+
+force_make: $(WORK_DIR)
+ $(AT)cd $(WORK_DIR); git checkout -b $(KAFKA_TAG) $(KAFKA_TAG); $(CONFIG_CMD); cd -
+ $(AT)$(MAKE) -C $(WORK_DIR)
+ @echo "Make done"
+
+force_install:
+ $(AT)sudo $(MAKE) -C $(WORK_DIR) install
+
+install: $(INSTALL_TARGET)
+
+# hard way to clean and clobber
+clean:
+ $(AT)cd $(WORK_DIR) && git clean -xfd *.o
+clobber:
+ $(AT)rm -rf $(WORK_DIR)
+
+# distclean is for developer who would like to keep the
+# clone git repo, saving time to fetch again from url
+distclean:
+ $(AT)cd $(WORK_DIR) && git clean -xfd && git checkout -f
+
+test:
+ @echo "Make test in $(WORK_DIR) (stub) "
+
+sanity:
+ @echo "Make sanity in $(WORK_DIR) (stub) "
+
+$(WORK_DIR):
+ $(AT)git clone $(KAFKA_URL)
+
+$(TAG_DONE_FLAG): $(WORK_DIR)
+ $(AT)cd $(WORK_DIR); git checkout $(KAFKA_TAG)
+ifneq ($(PATCH_FILE),)
+ $(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE)
+endif
+ $(AT)touch $@
diff --git a/src/package-list.mk b/src/package-list.mk
index a059bd5..3a92a1f 100644
--- a/src/package-list.mk
+++ b/src/package-list.mk
@@ -14,6 +14,9 @@ LIBPQOS_TAG ?= master
PMUTOOLS_URL ?= https://github.com/andikleen/pmu-tools
PMUTOOLS_TAG ?= master
+KAFKA_URL ?= https://github.com/edenhill/librdkafka.git
+KAFKA_TAG ?= v0.9.5
+
# collectd section
COLLECTD_URL ?= https://github.com/collectd/collectd
COLLECTD_TAG ?= collectd-5.8
diff --git a/systems/centos/7/build_base_machine.sh b/systems/centos/7/build_base_machine.sh
index 9c49d49..1853931 100755
--- a/systems/centos/7/build_base_machine.sh
+++ b/systems/centos/7/build_base_machine.sh
@@ -58,4 +58,3 @@ net-snmp-devel
# install epel release required for git-review
epel-release
" | grep -v ^#)
-