From bfb1d7536e376244a48275afdbb5693c80571ed1 Mon Sep 17 00:00:00 2001 From: Maryam Tahhan Date: Wed, 29 Nov 2017 11:48:19 +0000 Subject: docker: add kafka support to collectd Adds kafka support to collectd, cleaned up configuration files and updates documentation. Change-Id: Ie66ed22e8904e08e59aa7047d6afc9d176f3fb02 Signed-off-by: Maryam Tahhan --- src/Makefile | 1 + src/collectd/Makefile | 1 + src/collectd/collectd_sample_configs/csv.conf | 2 +- .../collectd_sample_configs/default_plugins.conf | 31 +++++++++ src/collectd/collectd_sample_configs/dpdkstat.conf | 2 +- src/collectd/collectd_sample_configs/exec.conf | 5 +- .../collectd_sample_configs/hugepages.conf | 2 +- src/collectd/collectd_sample_configs/kafka.conf | 21 ++++++ .../collectd_sample_configs/ovs_events.conf | 3 +- src/collectd/collectd_sample_configs/rdt.conf | 2 +- .../collectd_sample_configs/write_notification.sh | 2 +- src/librdkafka/Makefile | 75 ++++++++++++++++++++++ src/package-list.mk | 3 + 13 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 src/collectd/collectd_sample_configs/default_plugins.conf create mode 100644 src/collectd/collectd_sample_configs/kafka.conf create mode 100644 src/librdkafka/Makefile (limited to 'src') diff --git a/src/Makefile b/src/Makefile index 4785e854..7c6c17cc 100644 --- a/src/Makefile +++ b/src/Makefile @@ -40,6 +40,7 @@ SUBDIRS += dpdk endif SUBDIRS += libpqos SUBDIRS += pmu-tools +SUBDIRS += librdkafka SUBDIRS += collectd SUBDIRS += collectd-openstack-plugins diff --git a/src/collectd/Makefile b/src/collectd/Makefile index decb8e84..41fe89aa 100644 --- a/src/collectd/Makefile +++ b/src/collectd/Makefile @@ -48,6 +48,7 @@ CONFIG_CMD += --enable-exec CONFIG_CMD += --enable-python CONFIG_CMD += --enable-threshold CONFIG_CMD += --disable-perl +CONFIG_CMD += --with-librdkafka=/usr .PHONY: install force_install config force_make diff --git a/src/collectd/collectd_sample_configs/csv.conf b/src/collectd/collectd_sample_configs/csv.conf index e2350039..99a8498d 100644 --- a/src/collectd/collectd_sample_configs/csv.conf +++ b/src/collectd/collectd_sample_configs/csv.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/default_plugins.conf b/src/collectd/collectd_sample_configs/default_plugins.conf new file mode 100644 index 00000000..c96b0076 --- /dev/null +++ b/src/collectd/collectd_sample_configs/default_plugins.conf @@ -0,0 +1,31 @@ +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#Hostname "" + +LoadPlugin cpufreq +LoadPlugin disk +LoadPlugin ethstat +LoadPlugin ipc +LoadPlugin ipmi +LoadPlugin load +LoadPlugin memory +LoadPlugin numa +LoadPlugin processes +LoadPlugin df +LoadPlugin turbostat +LoadPlugin uptime +LoadPlugin contextswitch +LoadPlugin irq +LoadPlugin df +LoadPlugin swap diff --git a/src/collectd/collectd_sample_configs/dpdkstat.conf b/src/collectd/collectd_sample_configs/dpdkstat.conf index 201f9d37..919e6e6e 100644 --- a/src/collectd/collectd_sample_configs/dpdkstat.conf +++ b/src/collectd/collectd_sample_configs/dpdkstat.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/exec.conf b/src/collectd/collectd_sample_configs/exec.conf index e36409e5..0a291bdb 100644 --- a/src/collectd/collectd_sample_configs/exec.conf +++ b/src/collectd/collectd_sample_configs/exec.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,9 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - LoadPlugin exec + # For OVS PMD stats plugin Exec "collectd_exec" "/opt/collectd/etc/collectd.conf.d/ovs_pmd_stats.sh" diff --git a/src/collectd/collectd_sample_configs/hugepages.conf b/src/collectd/collectd_sample_configs/hugepages.conf index 4e4d9152..558b6f50 100644 --- a/src/collectd/collectd_sample_configs/hugepages.conf +++ b/src/collectd/collectd_sample_configs/hugepages.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/kafka.conf b/src/collectd/collectd_sample_configs/kafka.conf new file mode 100644 index 00000000..f81e87fd --- /dev/null +++ b/src/collectd/collectd_sample_configs/kafka.conf @@ -0,0 +1,21 @@ +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +LoadPlugin write_kafka + + + Property "metadata.broker.list" "localhost:9092" + + Format JSON + + diff --git a/src/collectd/collectd_sample_configs/ovs_events.conf b/src/collectd/collectd_sample_configs/ovs_events.conf index a52842ca..250e1e43 100644 --- a/src/collectd/collectd_sample_configs/ovs_events.conf +++ b/src/collectd/collectd_sample_configs/ovs_events.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - Interval 1 diff --git a/src/collectd/collectd_sample_configs/rdt.conf b/src/collectd/collectd_sample_configs/rdt.conf index 4ff0a77a..ae983dc0 100644 --- a/src/collectd/collectd_sample_configs/rdt.conf +++ b/src/collectd/collectd_sample_configs/rdt.conf @@ -1,4 +1,4 @@ -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/collectd/collectd_sample_configs/write_notification.sh b/src/collectd/collectd_sample_configs/write_notification.sh index 28198f9d..ed6ed9e2 100755 --- a/src/collectd/collectd_sample_configs/write_notification.sh +++ b/src/collectd/collectd_sample_configs/write_notification.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2016 OPNFV +# Copyright 2017 OPNFV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/src/librdkafka/Makefile b/src/librdkafka/Makefile new file mode 100644 index 00000000..ea5e34e2 --- /dev/null +++ b/src/librdkafka/Makefile @@ -0,0 +1,75 @@ +# makefile to manage collectd package +# Copyright 2017 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Contributors: +# Aihua Li, Huawei Technologies. +# Maryam Tahhan, Intel Corporation. + +include ../mk/master.mk +include ../package-list.mk + +WORK_DIR = librdkafka +TAG_DONE_FLAG = $(WORK_DIR)/.$(KAFKA_TAG).done +CONFIG_CMD = +CONFIG_CMD += ./configure +CONFIG_CMD += --prefix=/usr + +.PHONY: install force_install force_make + +# install depends on make +force_install: force_make + +all: force_make + @echo "Finished making $(WORK_DIR) " + +INSTALL_TARGET = force_install force_make + +force_make: $(WORK_DIR) + $(AT)cd $(WORK_DIR); git checkout -b $(KAFKA_TAG) $(KAFKA_TAG); $(CONFIG_CMD); cd - + $(AT)$(MAKE) -C $(WORK_DIR) + @echo "Make done" + +force_install: + $(AT)sudo $(MAKE) -C $(WORK_DIR) install + +install: $(INSTALL_TARGET) + +# hard way to clean and clobber +clean: + $(AT)cd $(WORK_DIR) && git clean -xfd *.o +clobber: + $(AT)rm -rf $(WORK_DIR) + +# distclean is for developer who would like to keep the +# clone git repo, saving time to fetch again from url +distclean: + $(AT)cd $(WORK_DIR) && git clean -xfd && git checkout -f + +test: + @echo "Make test in $(WORK_DIR) (stub) " + +sanity: + @echo "Make sanity in $(WORK_DIR) (stub) " + +$(WORK_DIR): + $(AT)git clone $(KAFKA_URL) + +$(TAG_DONE_FLAG): $(WORK_DIR) + $(AT)cd $(WORK_DIR); git checkout $(KAFKA_TAG) +ifneq ($(PATCH_FILE),) + $(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE) +endif + $(AT)touch $@ diff --git a/src/package-list.mk b/src/package-list.mk index a059bd55..3a92a1f7 100644 --- a/src/package-list.mk +++ b/src/package-list.mk @@ -14,6 +14,9 @@ LIBPQOS_TAG ?= master PMUTOOLS_URL ?= https://github.com/andikleen/pmu-tools PMUTOOLS_TAG ?= master +KAFKA_URL ?= https://github.com/edenhill/librdkafka.git +KAFKA_TAG ?= v0.9.5 + # collectd section COLLECTD_URL ?= https://github.com/collectd/collectd COLLECTD_TAG ?= collectd-5.8 -- cgit 1.2.3-korg