docker: add kafka support to collectd

Adds kafka support to collectd, cleaned up configuration files and updates
documentation.

Change-Id: Ie66ed22e8904e08e59aa7047d6afc9d176f3fb02
Signed-off-by: Maryam Tahhan <maryam.tahhan@intel.com>

13 files changed, 140 insertions, 10 deletions

diff --git a/src/Makefile b/src/Makefile
index 4785e854..7c6c17cc 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -40,6 +40,7 @@ SUBDIRS += dpdk
 endif
 SUBDIRS += libpqos
 SUBDIRS += pmu-tools
+SUBDIRS += librdkafka
 SUBDIRS += collectd
 SUBDIRS += collectd-openstack-plugins
 
diff --git a/src/collectd/Makefile b/src/collectd/Makefile
index decb8e84..41fe89aa 100644
--- a/src/collectd/Makefile
+++ b/src/collectd/Makefile
@@ -48,6 +48,7 @@ CONFIG_CMD += --enable-exec
 CONFIG_CMD += --enable-python
 CONFIG_CMD += --enable-threshold
 CONFIG_CMD += --disable-perl
+CONFIG_CMD += --with-librdkafka=/usr
 
 .PHONY: install force_install config force_make
 
diff --git a/src/collectd/collectd_sample_configs/csv.conf b/src/collectd/collectd_sample_configs/csv.conf
index e2350039..99a8498d 100644
--- a/src/collectd/collectd_sample_configs/csv.conf
+++ b/src/collectd/collectd_sample_configs/csv.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/default_plugins.conf b/src/collectd/collectd_sample_configs/default_plugins.conf
new file mode 100644
index 00000000..c96b0076
--- /dev/null
+++ b/src/collectd/collectd_sample_configs/default_plugins.conf
@@ -0,0 +1,31 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#Hostname ""
+
+LoadPlugin cpufreq
+LoadPlugin disk
+LoadPlugin ethstat
+LoadPlugin ipc
+LoadPlugin ipmi
+LoadPlugin load
+LoadPlugin memory
+LoadPlugin numa
+LoadPlugin processes
+LoadPlugin df
+LoadPlugin turbostat
+LoadPlugin uptime
+LoadPlugin contextswitch
+LoadPlugin irq
+LoadPlugin df
+LoadPlugin swap
diff --git a/src/collectd/collectd_sample_configs/dpdkstat.conf b/src/collectd/collectd_sample_configs/dpdkstat.conf
index 201f9d37..919e6e6e 100644
--- a/src/collectd/collectd_sample_configs/dpdkstat.conf
+++ b/src/collectd/collectd_sample_configs/dpdkstat.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/exec.conf b/src/collectd/collectd_sample_configs/exec.conf
index e36409e5..0a291bdb 100644
--- a/src/collectd/collectd_sample_configs/exec.conf
+++ b/src/collectd/collectd_sample_configs/exec.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,9 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
 LoadPlugin exec
+
 <Plugin exec>
      # For OVS PMD stats plugin
      Exec "collectd_exec" "/opt/collectd/etc/collectd.conf.d/ovs_pmd_stats.sh"
diff --git a/src/collectd/collectd_sample_configs/hugepages.conf b/src/collectd/collectd_sample_configs/hugepages.conf
index 4e4d9152..558b6f50 100644
--- a/src/collectd/collectd_sample_configs/hugepages.conf
+++ b/src/collectd/collectd_sample_configs/hugepages.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/kafka.conf b/src/collectd/collectd_sample_configs/kafka.conf
new file mode 100644
index 00000000..f81e87fd
--- /dev/null
+++ b/src/collectd/collectd_sample_configs/kafka.conf
@@ -0,0 +1,21 @@
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+LoadPlugin write_kafka
+
+<Plugin write_kafka>
+  Property "metadata.broker.list" "localhost:9092"
+  <Topic "collectd">
+    Format JSON
+  </Topic>
+</Plugin>
diff --git a/src/collectd/collectd_sample_configs/ovs_events.conf b/src/collectd/collectd_sample_configs/ovs_events.conf
index a52842ca..250e1e43 100644
--- a/src/collectd/collectd_sample_configs/ovs_events.conf
+++ b/src/collectd/collectd_sample_configs/ovs_events.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 <LoadPlugin ovs_events>
   Interval 1
 </LoadPlugin>
diff --git a/src/collectd/collectd_sample_configs/rdt.conf b/src/collectd/collectd_sample_configs/rdt.conf
index 4ff0a77a..ae983dc0 100644
--- a/src/collectd/collectd_sample_configs/rdt.conf
+++ b/src/collectd/collectd_sample_configs/rdt.conf
@@ -1,4 +1,4 @@
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/src/collectd/collectd_sample_configs/write_notification.sh b/src/collectd/collectd_sample_configs/write_notification.sh
index 28198f9d..ed6ed9e2 100755
--- a/src/collectd/collectd_sample_configs/write_notification.sh
+++ b/src/collectd/collectd_sample_configs/write_notification.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2016 OPNFV
+# Copyright 2017 OPNFV
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
diff --git a/src/librdkafka/Makefile b/src/librdkafka/Makefile
new file mode 100644
index 00000000..ea5e34e2
--- /dev/null
+++ b/src/librdkafka/Makefile
@@ -0,0 +1,75 @@
+# makefile to manage collectd package
+# Copyright 2017 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Contributors:
+#   Aihua Li, Huawei Technologies.
+#   Maryam Tahhan, Intel Corporation.
+
+include ../mk/master.mk
+include ../package-list.mk
+
+WORK_DIR = librdkafka
+TAG_DONE_FLAG = $(WORK_DIR)/.$(KAFKA_TAG).done
+CONFIG_CMD =
+CONFIG_CMD += ./configure
+CONFIG_CMD += --prefix=/usr
+
+.PHONY: install force_install force_make
+
+# install depends on make
+force_install: force_make
+
+all: force_make
+	@echo "Finished making $(WORK_DIR) "
+
+INSTALL_TARGET = force_install force_make
+
+force_make: $(WORK_DIR)
+	$(AT)cd $(WORK_DIR); git checkout -b $(KAFKA_TAG) $(KAFKA_TAG); $(CONFIG_CMD); cd -
+	$(AT)$(MAKE) -C $(WORK_DIR)
+	@echo "Make done"
+
+force_install:
+	$(AT)sudo $(MAKE) -C $(WORK_DIR) install
+
+install: $(INSTALL_TARGET)
+
+# hard way to clean and clobber
+clean:
+	$(AT)cd $(WORK_DIR) && git clean -xfd *.o
+clobber:
+	$(AT)rm -rf $(WORK_DIR)
+
+# distclean is for developer who would like to keep the
+# clone git repo, saving time to fetch again from url
+distclean:
+	$(AT)cd $(WORK_DIR) && git clean -xfd && git checkout -f
+
+test:
+	@echo "Make test in $(WORK_DIR) (stub) "
+
+sanity:
+	@echo "Make sanity in $(WORK_DIR) (stub) "
+
+$(WORK_DIR):
+	$(AT)git clone $(KAFKA_URL)
+
+$(TAG_DONE_FLAG): $(WORK_DIR)
+	$(AT)cd $(WORK_DIR); git checkout $(KAFKA_TAG)
+ifneq ($(PATCH_FILE),)
+	$(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE)
+endif
+	$(AT)touch $@
diff --git a/src/package-list.mk b/src/package-list.mk
index a059bd55..3a92a1f7 100644
--- a/src/package-list.mk
+++ b/src/package-list.mk
@@ -14,6 +14,9 @@ LIBPQOS_TAG ?= master
 PMUTOOLS_URL ?= https://github.com/andikleen/pmu-tools
 PMUTOOLS_TAG ?= master
 
+KAFKA_URL ?= https://github.com/edenhill/librdkafka.git
+KAFKA_TAG ?= v0.9.5
+
 # collectd section
 COLLECTD_URL ?= https://github.com/collectd/collectd
 COLLECTD_TAG ?= collectd-5.8