Merge changes from topic "collectd_6_testing"

* changes:
  Add a playbook that tests collectd 6 vs collectd5
  [docker][ansible] Add a playbook for building the containers
  Create a flask app to show metrics from collectd

9 files changed, 326 insertions, 2 deletions

diff --git a/docker/ansible/collectd6_test.yml b/docker/ansible/collectd6_test.yml
new file mode 100644
index 00000000..c1a3a8b4
--- /dev/null
+++ b/docker/ansible/collectd6_test.yml
@@ -0,0 +1,143 @@
+---
+# ansible-playbook -e PR=<PRID> -e new_plugin=<plugin> collectd6_test.yml
+
+# As well as passing a PRID, a config command should be passable too since
+# a lot of the plugins have been explicitly disabled in the build.
+- hosts: localhost
+  become: true
+  tasks:
+    - name: Set names for containers to be used for testing
+      set_fact:
+        collectd5_container_name: "bar-collectd-latest"
+        collectd6_container_name: "bar-collectd-6{{ '-' + PR if PR is defined }}"
+        flask_container_name: "test-collectd-5-v-6"
+
+    - name: Remove existing containers
+      docker_container:
+        name: "{{ item }}"
+        state: absent
+        force_kill: yes
+      with_items:
+        - "{{ collectd5_container_name }}"
+        - "{{ collectd6_container_name }}"
+        - "{{ flask_container_name }}"
+
+    - name: Get a list of containers
+      command:
+        docker ps -a
+      register: output
+
+    - name: Confirm that existing test containers were removed
+      assert:
+        that:
+          - 'not "{{ collectd5_container_name }}" in output.stdout'
+          - 'not "{{ collectd6_container_name }}" in output.stdout'
+          - 'not "{{ flask_container_name }}" in output.stdout'
+
+    - name: Build collectd containers
+      include_role:
+        name: build_collectd
+      args:
+        apply:
+          tags:
+            - latest
+            - collectd-6
+            - flask_test
+      vars:
+        COLLECTD_PULL_REQUESTS: "{{ PR | default() }}"
+        COLLECTD_CONFIG_CMD_ARGS: "{{ '--enable-' + new_plugin if new_plugin is defined }}"
+
+    - name: "Set up config for write_http plugin"
+      set_fact:
+        collectd_plugins: "{{ collectd_plugins | default([]) | union(['write_http']) }}"
+        collectd_plugin_write_http_nodes:
+          flask:
+            url: http://localhost:5000
+            format: "Command"
+
+    - name: Generate collectd configs
+      include_role:
+        name: config_files
+
+    # Since I can't skip-tags here, I have to remove the plugins later
+    # TODO(efoley) Add a disable_plugins and enable_plugins list to
+    # roles/config_files, as an alternative to tags.
+    # This alternative is kinda needed anyway, so that it's easier to add extra
+    # plugins instead of using.
+    # ``{{ collectd_plugins | default([]) | union(['the_plugin_i_want_to_enable'])}}``
+    # Tags can stay, since they are convenient, and easier to pass to the
+    # command line than a list of plugins
+    - name: "Remove plugin configs"
+      file:
+        path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf"
+        state: absent
+      with_items:
+        - snmp_agent
+        - intel_pmu
+
+    # TODO(efoley): The path here should be parameterised, to a degree, I don't
+    # want it to be repeated. And I shouldn't assume that this is always going
+    # to be the value (unless it is in vars/main instead of defaults/main)
+    - name: "Remove plugin configs (collectd 6)"
+      file:
+        path: "/opt/collectd/etc/collectd.conf.d/{{ item }}.conf"
+        state: absent
+      with_items:
+        - csv
+        - network
+        - rrdtool
+        - write_kafka
+        - write_prometheus
+        - logfile
+
+    - debug:
+        var: PR
+
+    - name: Run the collectd-6 container
+      include_role:
+        name: run_collectd
+      vars:
+        collectd_image_name: "opnfv/barometer-collectd-6{{ '-' + PR if PR is defined }}"
+        collectd_container_name: "{{ collectd6_container_name }}"
+
+    - name: Run the collectd-latest container
+      include_role:
+        name: run_collectd
+      vars:
+        collectd_image_name: opnfv/barometer-collectd-latest
+        collectd_container_name: "{{ collectd5_container_name }}"
+
+    - name: Run the flask test container
+      docker_container:
+        name: "{{ flask_container_name }}"
+        image: test-collectd-write_http
+        detach: yes
+        state: started
+          #network_mode: host
+        published_ports: 5000:5000
+
+    - name: Check output for flask app
+      become: true
+      shell: |
+        docker logs {{ flask_container_name }} {{ '| grep "' + new_plugin + '"' if new_plugin is defined  }} | tail -200
+      register: output
+
+    - debug:
+        var: output.stdout_lines
+
+    - name: Get a list of running containers
+      become: true
+      command:
+        docker ps
+      register: output
+
+    - name: Make sure that the expected containers are running
+      assert:
+        that:
+          - '"{{ collectd6_container_name }}" in output.stdout'
+          - '"{{ collectd5_container_name }}" in output.stdout'
+          - '"{{ flask_container_name }}" in output.stdout'
+
+# Create a small report at the end for collectd versions...
+# Update Apply PRs to check out a branch when it is a single PR
+# OR update these playbooks to use the tag way of checking out a PR
diff --git a/docker/ansible/collectd_build.yml b/docker/ansible/collectd_build.yml
new file mode 100644
index 00000000..d5cad076
--- /dev/null
+++ b/docker/ansible/collectd_build.yml
@@ -0,0 +1,22 @@
+# Copyright 2021 Anuket and others
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# ansible-playbook collectd_build.yml
+#
+- hosts: localhost
+  become: true
+  become_user: root
+  gather_facts: true
+  roles:
+    - name: build_collectd
diff --git a/docker/ansible/roles/build_collectd/tasks/main.yml b/docker/ansible/roles/build_collectd/tasks/main.yml
new file mode 100644
index 00000000..2ab92296
--- /dev/null
+++ b/docker/ansible/roles/build_collectd/tasks/main.yml
@@ -0,0 +1,71 @@
+# Copyright 2021 Anuket and others
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Build stable container
+  docker_image:
+    name: opnfv/barometer-collectd
+    build:
+      path: "{{ playbook_dir }}/../barometer-collectd/"
+    source: build
+  tags:
+    - stable
+
+- name: Build the latest container
+  docker_image:
+    name: opnfv/barometer-collectd-latest
+    build:
+      path: "{{ playbook_dir }}/../../"
+      dockerfile: "docker/barometer-collectd-latest/Dockerfile"
+    source: build
+  tags:
+    - latest
+
+- name: Build collectd-experimental
+  docker_image:
+    name: opnfv/barometer-collectd-experimental
+    build:
+      path: "{{ playbook_dir }}/../../"
+      dockerfile: "docker/barometer-collectd-experimental/Dockerfile"
+      args:
+        COLLECTD_FLAVOR: experimental
+        COLLECTD_TAG: "{{ COLLECTD_TAG | default('main') }}"
+        COLLECTD_PULL_REQUESTS: "{{ COLLECTD_PULL_REQUESTS | default() }}"
+    source: build
+  tags:
+    - experimental
+
+- name: Build collectd-6
+  docker_image:
+    name: "opnfv/barometer-collectd-6{{ ( '-' + COLLECTD_PULL_REQUESTS ) if COLLECTD_PULL_REQUESTS is defined else '' }}"
+    build:
+      path: "{{ playbook_dir }}/../../"
+      dockerfile: "docker/barometer-collectd-experimental/Dockerfile"
+      args:
+        COLLECTD_FLAVOR: collectd-6
+        COLLECTD_TAG: "{{ COLLECTD_TAG | default('collectd-6.0') }}"
+        COLLECTD_CONFIG_CMD_ARGS: "{{ COLLECTD_CONFIG_CMD_ARGS if COLLECTD_CONFIG_CMD_ARGS is defined }}"
+    source: build
+  tags:
+    - collectd-6
+
+- name: Build test_app for write_http
+  docker_image:
+    name: test-collectd-write_http
+    build:
+      path: "{{ playbook_dir }}/../flask_app/"
+    source: build
+  tags:
+    - flask_test
+    - never
+
diff --git a/docker/ansible/roles/run_collectd/tasks/main.yml b/docker/ansible/roles/run_collectd/tasks/main.yml
index 2c5d0e67..bf5aabf5 100644
--- a/docker/ansible/roles/run_collectd/tasks/main.yml
+++ b/docker/ansible/roles/run_collectd/tasks/main.yml
@@ -15,7 +15,7 @@
 
 - name: remove bar-collectd container
   docker_container:
-    name: bar-collectd
+    name: "{{ collectd_container_name }}"
     state: absent
   tags:
     - rm_containers
@@ -52,7 +52,7 @@
 
 - name: launch collectd container
   docker_container:
-    name: bar-collectd
+    name: "{{ collectd_container_name }}"
     image: "{{ collectd_image_name }}"
     volumes: "{{ volumes_list }}"
     entrypoint: "/run_collectd.sh"
diff --git a/docker/ansible/roles/run_collectd/vars/main.yml b/docker/ansible/roles/run_collectd/vars/main.yml
index 96c9032e..26007ecf 100644
--- a/docker/ansible/roles/run_collectd/vars/main.yml
+++ b/docker/ansible/roles/run_collectd/vars/main.yml
@@ -13,6 +13,7 @@
 # limitations under the License.
 ---
 
+collectd_container_name: "bar-collectd"
 default_flavor: "{{ flavor|default('stable')|string }}"
 flavor_image_name: "{{
          'barometer-collectd-latest' if (default_flavor == 'master' or default_flavor == 'latest') else
diff --git a/docker/flask_app/Dockerfile b/docker/flask_app/Dockerfile
new file mode 100644
index 00000000..67e6d589
--- /dev/null
+++ b/docker/flask_app/Dockerfile
@@ -0,0 +1,25 @@
+# Copyright 2021 Anuket and others. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM python:3-alpine
+
+EXPOSE 5000
+
+WORKDIR /app
+COPY ./ /app
+
+RUN python -m pip  install -r requirements.txt
+
+ENTRYPOINT ["python", "flask_app.py"]
diff --git a/docker/flask_app/README b/docker/flask_app/README
new file mode 100644
index 00000000..03f8f3ff
--- /dev/null
+++ b/docker/flask_app/README
@@ -0,0 +1,45 @@
+To build this run:
+  sudo docker build -t my-flask-app .
+
+To run the app and see collectd metrics:
+
+  sudo docker run -d --net=host  my-flask-app
+  OR
+  sudo docker run -d -p 5000:5000  my-flask-app
+
+and configure collectd to use the write_http plugin:
+
+ LoadPlugin write_http
+
+ <Plugin "write_http">
+   <Node "example">
+     URL "http://127.0.0.1:5000"
+     Format Command
+     # Format JSON
+   </Node>
+ </Plugin>
+
+Format Command is used to make the output more readable for humans.
+You can also use JSON.
+
+Later the server will do something more useful.
+To view the metrics that are being sent by collectd, run::
+
+   sudo docker inspect <container_id>
+   #OR
+   sudo docker logs <container_id>
+
+Metrics from collectd-5.x will use PUTVAL
+Metrics from collectd-6.x will use PUTMETRIC
+
+Sample output::
+
+  127.0.0.1 - - [21/Apr/2021 19:31:49] "POST / HTTP/1.1" 200 -
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/gauge-TSC interval=10.000 1619029909.268:2112.02271161789
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-busy interval=10.000 1619029909.268:1613.51555288381
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/percent-c1 interval=10.000 1619029909.268:86.2353665532377
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu00/frequency-average interval=10.000 1619029909.268:222.094501460956
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature interval=10.000 1619029909.268:53
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-pkg00/temperature-tcc_activation interval=10.000 1619029909.268:100
+  PUTVAL fbae30cc-2f20-11b2-a85c-819293100691/turbostat-cpu04/frequency-average interval=10.000 1619029909.268:206.978572579757
+
diff --git a/docker/flask_app/flask_app.py b/docker/flask_app/flask_app.py
new file mode 100644
index 00000000..771a91bc
--- /dev/null
+++ b/docker/flask_app/flask_app.py
@@ -0,0 +1,16 @@
+from flask import Flask, request
+import json
+
+app = Flask(__name__)
+
+@app.route('/', methods=['GET', 'POST'])
+def get_data():
+    #print(request.data)
+    #print(type(request.data))
+    print(request.data.decode('utf-8'))
+    #print(json.loads(request.data.decode("utf-8")))
+
+    return 'This is working!'
+
+if __name__=='__main__':
+    app.run(debug=True, host='0.0.0.0')
diff --git a/docker/flask_app/requirements.txt b/docker/flask_app/requirements.txt
new file mode 100644
index 00000000..e3e9a71d
--- /dev/null
+++ b/docker/flask_app/requirements.txt
@@ -0,0 +1 @@
+Flask