Containerize kafka server

JIRA: VES-2

Change-Id: Idc2db4b99de7d2757a7907f3170e58ae9a38f8b5
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

6 files changed, 151 insertions, 47 deletions

diff --git a/build/ves-agent.sh b/build/ves-agent.sh
index b3d3616..87d4b07 100644
--- a/build/ves-agent.sh
+++ b/build/ves-agent.sh
@@ -15,10 +15,15 @@
 #
 #. What this is: Build script for the VES Agent docker image on Ubuntu.
 #.
+#. Prerequisites:
+#.   Docker hub user logged in e.g. via "sudo docker login"
+#.
 #. Usage:
 #.   bash ves-agent.sh <hub-user> <hub-pass>
 #.     hub-user: username for dockerhub
-#.     hub-pass: password for dockerhub
+#.
+#. NOTE: To allow patch testing, this script will not reclone the VES repo
+#. if it exists under /tmp
 #.
 #. Status: this is a work in progress, under test.
 
@@ -28,13 +33,15 @@ sudo apt-get update
 echo; echo "$0 $(date): Starting VES agent build process"
 if [[ -d /tmp/ves ]]; then rm -rf /tmp/ves; fi
 
-echo; echo "$0 $(date): Cloning VES repo to /tmp/ves"
-git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves
+echo; echo "$0 $(date): Starting VES kafka build process"
+if [[ ! -d /tmp/ves ]]; then
+  echo; echo "$0 $(date): Cloning VES repo to /tmp/ves"
+  git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves
+fi
 
 echo; echo "$0 $(date): Building the image"
 cd /tmp/ves/build/ves-agent
 sudo docker build -t ves-agent .
-sudo docker login -u $1 -p $2
 
 echo; echo "$0 $(date): Tagging the image"
 id=$(sudo docker images | grep ves-agent | awk '{print $3}')
diff --git a/build/ves-collector.sh b/build/ves-collector.sh
index 4b33f24..58aa354 100644
--- a/build/ves-collector.sh
+++ b/build/ves-collector.sh
@@ -19,9 +19,11 @@
 #.   Docker hub user logged in e.g. via "sudo docker login"
 #.
 #. Usage:
-#.   bash ves-collector.sh <hub-user> <hub-pass>
+#.   bash ves-collector.sh <hub-user>
 #.     hub-user: username for dockerhub
-#.     hub-pass: password for dockerhub
+#.
+#. NOTE: To allow patch testing, this script will not reclone the VES repo
+#. if it exists under /tmp
 #.
 #. Status: this is a work in progress, under test.
 
diff --git a/build/ves-kafka.sh b/build/ves-kafka.sh
new file mode 100644
index 0000000..19a632b
--- /dev/null
+++ b/build/ves-kafka.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: Build script for a kafka server as used by OPNFV VES.
+#.
+#. Prerequisites:
+#.   Docker hub user logged in e.g. via "sudo docker login"
+#.
+#. Usage:
+#.   bash ves-kafka.sh <hub-user> <hub-pass>
+#.     hub-user: username for dockerhub
+#.
+#. NOTE: To allow patch testing, this script will not reclone the VES repo
+#. if it exists under /tmp
+#.
+#. Status: this is a work in progress, under test.
+
+wd=$(pwd)
+echo; echo "$0 $(date): Update package repos"
+sudo apt-get update
+
+echo; echo "$0 $(date): Starting VES kafka build process"
+if [[ ! -d /tmp/ves ]]; then
+  echo; echo "$0 $(date): Cloning VES repo to /tmp/ves"
+  git clone https://gerrit.opnfv.org/gerrit/ves /tmp/ves
+fi
+
+echo; echo "$0 $(date): Building the image"
+cd /tmp/ves/build/ves-kafka
+sudo docker build -t ves-kafka .
+
+echo; echo "$0 $(date): Tagging the image"
+id=$(sudo docker images | grep ves-kafka | awk '{print $3}')
+id=$(echo $id | cut -d ' ' -f 1)
+sudo docker tag $id $1/ves-kafka:latest
+
+echo; echo "$0 $(date): Pushing the image to dockerhub as $1/ves-kafka"
+sudo docker push $1/ves-kafka
+cd $wd
diff --git a/build/ves-kafka/Dockerfile b/build/ves-kafka/Dockerfile
new file mode 100644
index 0000000..c2acc8a
--- /dev/null
+++ b/build/ves-kafka/Dockerfile
@@ -0,0 +1,44 @@
+# Copyright 2017 AT&T Intellectual Property, Inc
+#  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# What this is: A Dockerfile for building a kafka server for use by the OPFNV
+# VES framework.
+#
+# Status: this is a work in progress, under test.
+#
+
+FROM ubuntu:xenial
+
+MAINTAINER Bryan Sullivan
+
+RUN apt-get update
+RUN apt-get install -y apt-utils
+RUN apt-get upgrade -y
+RUN apt-get update
+# Required for kafka
+RUN apt-get install -y default-jre
+RUN apt-get install -y python-pip
+RUN pip install kafka-python
+RUN apt-get install -y wget
+
+RUN mkdir /opt/ves
+
+RUN cd /opt/ves; \
+wget http://www-eu.apache.org/dist/kafka/0.11.0.2/kafka_2.11-0.11.0.2.tgz; \
+tar -xvzf kafka_2.11-0.11.0.2.tgz; \
+sed -i -- 's/#delete.topic.enable=true/delete.topic.enable=true/' \
+  kafka_2.11-0.11.0.2/config/server.properties
+
+COPY start.sh /opt/ves/start.sh
+ENTRYPOINT ["/bin/bash", "/opt/ves/start.sh"]
diff --git a/build/ves-kafka/start.sh b/build/ves-kafka/start.sh
new file mode 100644
index 0000000..ab4169b
--- /dev/null
+++ b/build/ves-kafka/start.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: Startup script for a kafka server as used by the OPNFV VES
+#. framework.
+
+echo "$zookeeper $zookeeper_host" >>/etc/hosts
+cat /etc/hosts
+cd /opt/ves
+
+sed -i "s/localhost:2181/$zookeeper_host:2181/" \
+  kafka_2.11-0.11.0.2/config/server.properties
+grep 2181 kafka_2.11-0.11.0.2/config/server.properties
+sed -i "s~#advertised.listeners=PLAINTEXT://your.host.name:9092~advertised.listeners=PLAINTEXT://$kafka_hostname:9092~" \
+  kafka_2.11-0.11.0.2/config/server.properties
+grep advertised.listeners kafka_2.11-0.11.0.2/config/server.properties
+
+kafka_2.11-0.11.0.2/bin/kafka-server-start.sh \
+  kafka_2.11-0.11.0.2/config/server.properties
+
diff --git a/tools/ves-setup.sh b/tools/ves-setup.sh
index 10f6fd3..00d7db8 100644
--- a/tools/ves-setup.sh
+++ b/tools/ves-setup.sh
@@ -54,7 +54,7 @@
 #.
 #. Usage:
 #.   git clone https://gerrit.opnfv.org/gerrit/ves ~/ves
-#.   bash ~/ves/ves-setup.sh <collector|kafka|collectd|agent> [cloudify]
+#.   bash ~/ves/tools/ves-setup.sh <collector|kafka|collectd|agent> [cloudify]
 #.     collector: setup VES collector (test collector) 
 #.     kafka: setup kafka server for VES events from collect agent(s)
 #.     collectd: setup collectd with libvirt plugin, as a kafka publisher
@@ -140,45 +140,6 @@ EOF
   echo ~/ves/tools/ves_env.sh
 }
 
-function setup_kafka() {
-  log "setup kafka server"
-  common_prereqs
-
-  log "install kafka prerequisites"
-    if [[ "$dist" == "ubuntu" ]]; then
-    sudo apt-get install -y default-jre
-    sudo apt-get install -y zookeeperd
-    sudo apt-get install -y python-pip
-  else
-    # per http://aurora.apache.org/documentation/0.12.0/installing/#centos-7
-    sudo yum install -y https://archive.cloudera.com/cdh5/one-click-install/redhat/7/x86_64/cloudera-cdh-5-0.x86_64.rpm
-    # TODO: Barometer guide: Java 1.7 is needed for Kafka
-    sudo yum install -y java-1.7.0-openjdk
-    # TODO: Barometer guide: both packages and init needed
-    sudo yum install -y zookeeper zookeeper-server
-    sudo service zookeeper-server init
-    sudo zookeeper-server start
-    sudo yum install -y python-pip
-  fi
-  sudo pip install kafka-python
-
-  setup_env
-
-  cd ~
-  ver="0.11.0.2"
-  log "get and unpack kafka_2.11-$ver.tgz"
-  wget "http://www-eu.apache.org/dist/kafka/$ver/kafka_2.11-$ver.tgz"
-  tar -xvzf kafka_2.11-$ver.tgz
-
-  log "set delete.topic.enable=true"
-  sed -i -- 's/#delete.topic.enable=true/delete.topic.enable=true/' \
-    kafka_2.11-$ver/config/server.properties
-  grep delete.topic.enable kafka_2.11-$ver/config/server.properties
-  # TODO: Barometer VES guide to clarify hostname must be in /etc/hosts
-  sudo nohup kafka_2.11-$ver/bin/kafka-server-start.sh \
-    kafka_2.11-$ver/config/server.properties >kafka.log 2>&1 &
-}
-
 function setup_collectd() {
   log "setup collectd"
 
@@ -587,7 +548,14 @@ case "$1" in
     setup_collector
     ;;
   "kafka")
-    setup_kafka 
+    log "setup kafka server"
+    source ~/k8s_env.sh
+    sudo docker run -it -d -p 2181:2181 --name zookeeper zookeeper
+    sudo docker run -it -d -p 9092:9092 --name ves-kafka \
+      -e zookeeper_host=$k8s_master_host \
+      -e zookeeper=$k8s_master \
+      -e kafka_hostname=$ves_kafka_hostname \
+      blsaws/ves-kafka:latest
     ;;
   "verify")
     verify_veseventsdb "$1" "load" "load-shortterm"