Merge "docker: Add support for experimental collectd container"

9 files changed, 192 insertions, 17 deletions

diff --git a/docker/barometer-collectd-experimental/Dockerfile b/docker/barometer-collectd-experimental/Dockerfile
new file mode 100644
index 00000000..e505d2e2
--- /dev/null
+++ b/docker/barometer-collectd-experimental/Dockerfile
@@ -0,0 +1,37 @@
+FROM centos:7
+RUN yum update -y && \
+        yum install -y which sudo git && \
+        yum clean all && \
+        git config --global http.sslVerify false
+
+ENV DOCKER y
+ENV COLLECTD_FLAVOR experimental
+ENV WITH_DPDK y
+ENV repos_dir /src
+ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins
+
+WORKDIR ${repos_dir}
+RUN mkdir -p ${repos_dir}/barometer
+COPY . ${repos_dir}/barometer
+COPY docker/barometer-collectd-experimental/collectd_apply_pull_request.sh \
+     ${repos_dir}/barometer/src/collectd/
+
+# copying additional experimental configs on top of configs for 'master'
+# branch release
+COPY docker/barometer-collectd-experimental/experimental-configs/* \
+     ${repos_dir}/barometer/src/collectd/collectd_sample_configs-master/
+
+WORKDIR ${repos_dir}/barometer/systems
+RUN sh ./build_base_machine.sh && \
+        useradd -ms /bin/bash collectd_exec && \
+        echo "collectd_exec ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+WORKDIR ${openstack_plugins}
+RUN make && \
+        pip install --upgrade pip && \
+        pip install -r ${openstack_plugins}/collectd-openstack-plugins/requirements.txt
+
+COPY docker/barometer-collectd-experimental/run_collectd.sh /run_collectd.sh
+RUN chmod +x /run_collectd.sh
+
+ENTRYPOINT ["/run_collectd.sh"]
diff --git a/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh b/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh
new file mode 100755
index 00000000..dcea33a3
--- /dev/null
+++ b/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh
@@ -0,0 +1,39 @@
+#! /bin/bash
+# Copyright 2019 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This files contains list of pull requests to be applied on top
+# of master branch before building collectd included in docker
+# collectd-experimental container
+
+# Space/newline separated list of pull requests IDs
+# for example:
+# PULL_REQUESTS=(3027 #reimplement delay rate
+#                3028 #other PR
+#                )
+
+PULL_REQUESTS=(
+    3045 #logparser
+    #insert another PR ID here
+              )
+
+# during rebasing/merging git requires email & name to be set
+git config user.email "barometer-experimental@container"
+git config user.name "BarometerExperimental"
+
+for PR_ID in "${PULL_REQUESTS[@]}"
+do
+    echo "Applying pull request $PR_ID"
+    git pull --rebase origin pull/$PR_ID/head
+done
diff --git a/docker/barometer-collectd-experimental/experimental-configs/README b/docker/barometer-collectd-experimental/experimental-configs/README
new file mode 100644
index 00000000..e4386358
--- /dev/null
+++ b/docker/barometer-collectd-experimental/experimental-configs/README
@@ -0,0 +1 @@
+Place here configuration files that are required by experimental pull requests
diff --git a/docker/barometer-collectd-experimental/experimental-configs/logparser.conf b/docker/barometer-collectd-experimental/experimental-configs/logparser.conf
new file mode 100644
index 00000000..3802768b
--- /dev/null
+++ b/docker/barometer-collectd-experimental/experimental-configs/logparser.conf
@@ -0,0 +1,74 @@
+# Copyright 2019 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+LoadPlugin logparser
+
+<Plugin logparser>
+  <Logfile "/var/log/syslog">
+    FirstFullRead false
+    <Message "pcie_errors">
+      DefaultType "pcie_error"
+      DefaultSeverity "warning"
+      <Match "aer error">
+        Regex "AER:.*error received"
+        SubmatchIdx -1
+      </Match>
+      <Match "incident time">
+        Regex "(... .. ..:..:..) .* pcieport.*AER"
+        SubmatchIdx 1
+        IsMandatory false
+      </Match>
+      <Match "root port">
+        Regex "pcieport (.*): AER:"
+        SubmatchIdx 1
+        IsMandatory true
+      </Match>
+      <Match "device">
+        PluginInstance true
+        Regex " ([0-9a-fA-F:\\.]*): PCIe Bus Error"
+        SubmatchIdx 1
+        IsMandatory false
+      </Match>
+      <Match "severity_mandatory">
+        Regex "severity="
+        SubMatchIdx -1
+      </Match>
+      <Match "nonfatal">
+        Regex "severity=.*\\([nN]on-[fF]atal"
+        TypeInstance "non_fatal"
+        IsMandatory false
+      </Match>
+      <Match "fatal">
+        Regex "severity=.*\\([fF]atal"
+        Severity "failure"
+        TypeInstance "fatal"
+        IsMandatory false
+      </Match>
+      <Match "corrected">
+        Regex "severity=Corrected"
+        TypeInstance "correctable"
+        IsMandatory false
+      </Match>
+      <Match "error type">
+        Regex "type=(.*),"
+        SubmatchIdx 1
+        IsMandatory false
+      </Match>
+     <Match "id">
+        Regex ", id=(.*)"
+        SubmatchIdx 1
+      </Match>
+    </Message>
+  </Logfile>
+</Plugin>
diff --git a/docker/barometer-collectd-experimental/run_collectd.sh b/docker/barometer-collectd-experimental/run_collectd.sh
new file mode 100644
index 00000000..b45711e1
--- /dev/null
+++ b/docker/barometer-collectd-experimental/run_collectd.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Copyright 2019 OPNFV
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+/opt/collectd/sbin/collectd -f
diff --git a/docker/barometer-collectd-master/Dockerfile b/docker/barometer-collectd-master/Dockerfile
index 0f890d08..da94888d 100644
--- a/docker/barometer-collectd-master/Dockerfile
+++ b/docker/barometer-collectd-master/Dockerfile
@@ -5,7 +5,7 @@ RUN yum update -y && \
         git config --global http.sslVerify false
 
 ENV DOCKER y
-ENV COLLECTD_USE_MASTER y
+ENV COLLECTD_FLAVOR master
 ENV WITH_DPDK y
 ENV repos_dir /src
 ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins
diff --git a/docker/barometer-collectd/Dockerfile b/docker/barometer-collectd/Dockerfile
index a7a8a077..2f2f57e5 100644
--- a/docker/barometer-collectd/Dockerfile
+++ b/docker/barometer-collectd/Dockerfile
@@ -6,6 +6,7 @@ RUN yum update -y && \
 
 ENV DOCKER y
 ENV WITH_DPDK y
+ENV COLLECTD_FLAVOR stable
 ENV repos_dir /src
 ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins
 
diff --git a/src/collectd/Makefile b/src/collectd/Makefile
index f6766439..8eea00fd 100644
--- a/src/collectd/Makefile
+++ b/src/collectd/Makefile
@@ -39,11 +39,6 @@ ifeq ($(XDG_CACHE_HOME_DIR),)
 XDG_CACHE_HOME_DIR = $(shell echo $$HOME)
 endif
 
-SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs
-ifdef COLLECTD_USE_MASTER
-SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs-master
-endif
-
 CONFIG_CMD += --enable-syslog
 CONFIG_CMD += --enable-logfile
 CONFIG_CMD += --with-libpqos=$(LIBPQOS_DIR)
@@ -73,8 +68,6 @@ config $(WORK_DIR)/Makefile: $(WORK_DIR)/configure
 	@echo "Configure done"
 
 force_make: $(WORK_DIR)/Makefile
-	$(AT)cd $(WORK_DIR) && git pull $(COLLECTD_URL) $(COLLECTD_TAG)
-	@echo "git pull done"
 	$(AT) $(MAKE) -C $(WORK_DIR) $(MORE_MAKE_FLAGS)
 	@echo "Make done"
 
@@ -117,13 +110,11 @@ sanity:
 
 .PHONY: boot
 # boot collectd is the process to produce the script 'configure'
-boot $(WORK_DIR)/configure:
+boot $(WORK_DIR)/configure: $(TAG_DONE_FLAG)
 	@echo "booting up collectd"
 	$(AT)cd $(WORK_DIR); $(BUILD_CMD)
 	@echo "done booting collectd"
 
-boot $(WORK_DIR)/configure: $(TAG_DONE_FLAG)
-
 $(WORK_DIR):
 	$(AT)git clone $(COLLECTD_URL)
 
@@ -133,4 +124,8 @@ $(TAG_DONE_FLAG): $(WORK_DIR)
 ifneq ($(PATCH_FILE),)
 	$(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE)
 endif
+ifeq ($(COLLECTD_USE_EXPERIMENTAL_PR), y)
+	@echo "Applying experimental pull requests"
+	$(AT)cd $(WORK_DIR); ../collectd_apply_pull_request.sh
+endif
 	$(AT)touch $@
diff --git a/src/package-list.mk b/src/package-list.mk
index b58e092a..47b90bde 100644
--- a/src/package-list.mk
+++ b/src/package-list.mk
@@ -19,13 +19,26 @@ KAFKA_TAG ?= v0.9.5
 
 # collectd section
 COLLECTD_URL ?= https://github.com/collectd/collectd
-# there are 2 collectd flavors:
-# -"collectd" - based on stable collect release
-# -"collectd-master" - development version, based on master branch
-ifdef COLLECTD_USE_MASTER
-	COLLECTD_TAG ?= master
-else
+
+# there are 3 collectd flavors:
+# -"stable" - based on stable collectd release
+# -"master" - development version, based on master branch
+# -"experimental" - it is based on master branch as above and includes
+#                   set pull requests with experimental features
+ifeq ($(COLLECTD_FLAVOR), stable)
+# using latest stable release
 	COLLECTD_TAG ?= collectd-5.8
+	SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs
+else
+# 'master' and 'experimental' collectd flavors are both using
+# code from master branch
+	COLLECTD_TAG ?= master
+	SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs-master
+ifeq ($(COLLECTD_FLAVOR), experimental)
+# 'experimental' flavor is using additional Pull Requests that
+# are put on top of master release
+	COLLECTD_USE_EXPERIMENTAL_PR ?= y
+endif #end of experimental-branch handling
 endif
 
 COLLECTD_OPENSTACK_URL ?= https://github.com/openstack/collectd-openstack-plugins