From 48bd4b945262edf8558abd9c0dacdd18fe8ef84b Mon Sep 17 00:00:00 2001 From: Kamil Wiatrowski Date: Fri, 11 Jan 2019 07:58:03 +0000 Subject: docker: Add support for experimental collectd container Experimental container is based on master branch and includes set of pull requests with experimental features. By default one addtional PR is included - logparser. Change-Id: I455122b23b6de382f6ef75a790f7b052784b59ac Signed-off-by: Radoslaw Jablonski Signed-off-by: Kamil Wiatrowski --- docker/barometer-collectd-experimental/Dockerfile | 37 +++++++++++ .../collectd_apply_pull_request.sh | 39 ++++++++++++ .../experimental-configs/README | 1 + .../experimental-configs/logparser.conf | 74 ++++++++++++++++++++++ .../run_collectd.sh | 15 +++++ docker/barometer-collectd-master/Dockerfile | 2 +- docker/barometer-collectd/Dockerfile | 1 + src/collectd/Makefile | 15 ++--- src/package-list.mk | 25 ++++++-- 9 files changed, 192 insertions(+), 17 deletions(-) create mode 100644 docker/barometer-collectd-experimental/Dockerfile create mode 100755 docker/barometer-collectd-experimental/collectd_apply_pull_request.sh create mode 100644 docker/barometer-collectd-experimental/experimental-configs/README create mode 100644 docker/barometer-collectd-experimental/experimental-configs/logparser.conf create mode 100644 docker/barometer-collectd-experimental/run_collectd.sh diff --git a/docker/barometer-collectd-experimental/Dockerfile b/docker/barometer-collectd-experimental/Dockerfile new file mode 100644 index 00000000..e505d2e2 --- /dev/null +++ b/docker/barometer-collectd-experimental/Dockerfile @@ -0,0 +1,37 @@ +FROM centos:7 +RUN yum update -y && \ + yum install -y which sudo git && \ + yum clean all && \ + git config --global http.sslVerify false + +ENV DOCKER y +ENV COLLECTD_FLAVOR experimental +ENV WITH_DPDK y +ENV repos_dir /src +ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins + +WORKDIR ${repos_dir} +RUN mkdir -p ${repos_dir}/barometer +COPY . ${repos_dir}/barometer +COPY docker/barometer-collectd-experimental/collectd_apply_pull_request.sh \ + ${repos_dir}/barometer/src/collectd/ + +# copying additional experimental configs on top of configs for 'master' +# branch release +COPY docker/barometer-collectd-experimental/experimental-configs/* \ + ${repos_dir}/barometer/src/collectd/collectd_sample_configs-master/ + +WORKDIR ${repos_dir}/barometer/systems +RUN sh ./build_base_machine.sh && \ + useradd -ms /bin/bash collectd_exec && \ + echo "collectd_exec ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + +WORKDIR ${openstack_plugins} +RUN make && \ + pip install --upgrade pip && \ + pip install -r ${openstack_plugins}/collectd-openstack-plugins/requirements.txt + +COPY docker/barometer-collectd-experimental/run_collectd.sh /run_collectd.sh +RUN chmod +x /run_collectd.sh + +ENTRYPOINT ["/run_collectd.sh"] diff --git a/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh b/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh new file mode 100755 index 00000000..dcea33a3 --- /dev/null +++ b/docker/barometer-collectd-experimental/collectd_apply_pull_request.sh @@ -0,0 +1,39 @@ +#! /bin/bash +# Copyright 2019 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This files contains list of pull requests to be applied on top +# of master branch before building collectd included in docker +# collectd-experimental container + +# Space/newline separated list of pull requests IDs +# for example: +# PULL_REQUESTS=(3027 #reimplement delay rate +# 3028 #other PR +# ) + +PULL_REQUESTS=( + 3045 #logparser + #insert another PR ID here + ) + +# during rebasing/merging git requires email & name to be set +git config user.email "barometer-experimental@container" +git config user.name "BarometerExperimental" + +for PR_ID in "${PULL_REQUESTS[@]}" +do + echo "Applying pull request $PR_ID" + git pull --rebase origin pull/$PR_ID/head +done diff --git a/docker/barometer-collectd-experimental/experimental-configs/README b/docker/barometer-collectd-experimental/experimental-configs/README new file mode 100644 index 00000000..e4386358 --- /dev/null +++ b/docker/barometer-collectd-experimental/experimental-configs/README @@ -0,0 +1 @@ +Place here configuration files that are required by experimental pull requests diff --git a/docker/barometer-collectd-experimental/experimental-configs/logparser.conf b/docker/barometer-collectd-experimental/experimental-configs/logparser.conf new file mode 100644 index 00000000..3802768b --- /dev/null +++ b/docker/barometer-collectd-experimental/experimental-configs/logparser.conf @@ -0,0 +1,74 @@ +# Copyright 2019 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LoadPlugin logparser + + + + FirstFullRead false + + DefaultType "pcie_error" + DefaultSeverity "warning" + + Regex "AER:.*error received" + SubmatchIdx -1 + + + Regex "(... .. ..:..:..) .* pcieport.*AER" + SubmatchIdx 1 + IsMandatory false + + + Regex "pcieport (.*): AER:" + SubmatchIdx 1 + IsMandatory true + + + PluginInstance true + Regex " ([0-9a-fA-F:\\.]*): PCIe Bus Error" + SubmatchIdx 1 + IsMandatory false + + + Regex "severity=" + SubMatchIdx -1 + + + Regex "severity=.*\\([nN]on-[fF]atal" + TypeInstance "non_fatal" + IsMandatory false + + + Regex "severity=.*\\([fF]atal" + Severity "failure" + TypeInstance "fatal" + IsMandatory false + + + Regex "severity=Corrected" + TypeInstance "correctable" + IsMandatory false + + + Regex "type=(.*)," + SubmatchIdx 1 + IsMandatory false + + + Regex ", id=(.*)" + SubmatchIdx 1 + + + + diff --git a/docker/barometer-collectd-experimental/run_collectd.sh b/docker/barometer-collectd-experimental/run_collectd.sh new file mode 100644 index 00000000..b45711e1 --- /dev/null +++ b/docker/barometer-collectd-experimental/run_collectd.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright 2019 OPNFV +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +/opt/collectd/sbin/collectd -f diff --git a/docker/barometer-collectd-master/Dockerfile b/docker/barometer-collectd-master/Dockerfile index 0f890d08..da94888d 100644 --- a/docker/barometer-collectd-master/Dockerfile +++ b/docker/barometer-collectd-master/Dockerfile @@ -5,7 +5,7 @@ RUN yum update -y && \ git config --global http.sslVerify false ENV DOCKER y -ENV COLLECTD_USE_MASTER y +ENV COLLECTD_FLAVOR master ENV WITH_DPDK y ENV repos_dir /src ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins diff --git a/docker/barometer-collectd/Dockerfile b/docker/barometer-collectd/Dockerfile index a7a8a077..2f2f57e5 100644 --- a/docker/barometer-collectd/Dockerfile +++ b/docker/barometer-collectd/Dockerfile @@ -6,6 +6,7 @@ RUN yum update -y && \ ENV DOCKER y ENV WITH_DPDK y +ENV COLLECTD_FLAVOR stable ENV repos_dir /src ENV openstack_plugins /src/barometer/src/collectd-openstack-plugins diff --git a/src/collectd/Makefile b/src/collectd/Makefile index f6766439..8eea00fd 100644 --- a/src/collectd/Makefile +++ b/src/collectd/Makefile @@ -39,11 +39,6 @@ ifeq ($(XDG_CACHE_HOME_DIR),) XDG_CACHE_HOME_DIR = $(shell echo $$HOME) endif -SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs -ifdef COLLECTD_USE_MASTER -SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs-master -endif - CONFIG_CMD += --enable-syslog CONFIG_CMD += --enable-logfile CONFIG_CMD += --with-libpqos=$(LIBPQOS_DIR) @@ -73,8 +68,6 @@ config $(WORK_DIR)/Makefile: $(WORK_DIR)/configure @echo "Configure done" force_make: $(WORK_DIR)/Makefile - $(AT)cd $(WORK_DIR) && git pull $(COLLECTD_URL) $(COLLECTD_TAG) - @echo "git pull done" $(AT) $(MAKE) -C $(WORK_DIR) $(MORE_MAKE_FLAGS) @echo "Make done" @@ -117,13 +110,11 @@ sanity: .PHONY: boot # boot collectd is the process to produce the script 'configure' -boot $(WORK_DIR)/configure: +boot $(WORK_DIR)/configure: $(TAG_DONE_FLAG) @echo "booting up collectd" $(AT)cd $(WORK_DIR); $(BUILD_CMD) @echo "done booting collectd" -boot $(WORK_DIR)/configure: $(TAG_DONE_FLAG) - $(WORK_DIR): $(AT)git clone $(COLLECTD_URL) @@ -132,5 +123,9 @@ $(TAG_DONE_FLAG): $(WORK_DIR) $(AT)cd collectd; git checkout $(COLLECTD_TAG) ifneq ($(PATCH_FILE),) $(AT)cd $(WORK_DIR); patch -p1 < ../$(PATCH_FILE) +endif +ifeq ($(COLLECTD_USE_EXPERIMENTAL_PR), y) + @echo "Applying experimental pull requests" + $(AT)cd $(WORK_DIR); ../collectd_apply_pull_request.sh endif $(AT)touch $@ diff --git a/src/package-list.mk b/src/package-list.mk index b58e092a..47b90bde 100644 --- a/src/package-list.mk +++ b/src/package-list.mk @@ -19,13 +19,26 @@ KAFKA_TAG ?= v0.9.5 # collectd section COLLECTD_URL ?= https://github.com/collectd/collectd -# there are 2 collectd flavors: -# -"collectd" - based on stable collect release -# -"collectd-master" - development version, based on master branch -ifdef COLLECTD_USE_MASTER - COLLECTD_TAG ?= master -else + +# there are 3 collectd flavors: +# -"stable" - based on stable collectd release +# -"master" - development version, based on master branch +# -"experimental" - it is based on master branch as above and includes +# set pull requests with experimental features +ifeq ($(COLLECTD_FLAVOR), stable) +# using latest stable release COLLECTD_TAG ?= collectd-5.8 + SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs +else +# 'master' and 'experimental' collectd flavors are both using +# code from master branch + COLLECTD_TAG ?= master + SAMPLE_CONF_VARIANT_NAME = collectd_sample_configs-master +ifeq ($(COLLECTD_FLAVOR), experimental) +# 'experimental' flavor is using additional Pull Requests that +# are put on top of master release + COLLECTD_USE_EXPERIMENTAL_PR ?= y +endif #end of experimental-branch handling endif COLLECTD_OPENSTACK_URL ?= https://github.com/openstack/collectd-openstack-plugins -- cgit 1.2.3-korg