Collectd working on VM version

JIRA: VES-1

Change-Id: If03d8ddaf70c9898b70248de98fbb2974d45dc81
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

4 files changed, 259 insertions, 218 deletions

diff --git a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
index 6221a02..ecd4961 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
+++ b/tests/blueprints/tosca-vnfd-hello-ves/evel_demo.c
@@ -181,48 +181,22 @@ void check_app_container_state() {
  * param[in]  none
  *****************************************************************************/
 
-double cpu(char *id) {
-  double a, b, loadavg;
+double cpu() {
+  double a[4], b[4], loadavg;
   FILE *fp;
   int status;
-  char str[100], save[100];
-  char *cpu;
-  a = 0;
-  b = 0;
 
-  fp = popen("cat /proc/stat", "r");
-  if (fp == NULL) {
-    EVEL_ERROR("popen failed to execute command");
-  }
+  fp = fopen("/proc/stat","r");
+  fscanf(fp,"%*s %lf %lf %lf %lf",&a[0],&a[1],&a[2],&a[3]);
+  fclose(fp);
+  sleep(1);
 
-  while (fgets(str, 100, fp) != NULL) {
-    strcpy(save, str);
-//    printf("line: %s\n", str);
-    cpu = strtok(str, " ");
-    if (strcmp(cpu, id) == 0) {
-      // Sum across user, nice, system, idle
-      strcpy(str, save);
-      a = atof(strtok(NULL, " "));
-      strcpy(str, save);
-      a += atof(strtok(NULL, " "));
-      strcpy(str, save);
-      a += atof(strtok(NULL, " "));
-      strcpy(str, save);
-      b = a + atof(strtok(NULL, " "));
-      loadavg = a/b;
-      printf("Load for %s found: %f\n", id, a/b);
-      break;
-    }
-    else if (strcmp(cpu, "intr") == 0) {
-      loadavg = -1;
-      break;
-    }
-  }
+  fp = fopen("/proc/stat","r");
+  fscanf(fp,"%*s %lf %lf %lf %lf",&b[0],&b[1],&b[2],&b[3]);
+  fclose(fp);
+
+  loadavg = ((b[0]+b[1]+b[2]) - (a[0]+a[1]+a[2])) / ((b[0]+b[1]+b[2]+b[3]) - (a[0]+a[1]+a[2]+a[3]));
 
-  status = pclose(fp);
-  if (status == -1) {
-    EVEL_ERROR("pclose returned an error");
-  }
   return(loadavg);
 }
 
@@ -276,22 +250,10 @@ void measure_traffic() {
       memory_configured, memory_used, request_rate);
 
     if (measurement != NULL) {
+      cpu();
       evel_measurement_type_set(measurement, "HTTP request rate");
-      if ((loadavg=cpu("cpu")) != -1) {
-        evel_measurement_agg_cpu_use_set(measurement, loadavg);
-      }
-      if ((loadavg=cpu("cpu0")) != -1) {
-        evel_measurement_cpu_use_add(measurement, "cpu0", loadavg);
-      }
-      if ((loadavg=cpu("cpu1")) != -1) {
-        evel_measurement_cpu_use_add(measurement, "cpu1", loadavg);
-      }
-      if ((loadavg=cpu("cpu2")) != -1) {
-        evel_measurement_cpu_use_add(measurement, "cpu2", loadavg);
-      }
-      if ((loadavg=cpu("cpu3")) != -1) {
-        evel_measurement_cpu_use_add(measurement, "cpu3", loadavg);
-      }
+      evel_measurement_agg_cpu_use_set(measurement, loadavg);
+      evel_measurement_cpu_use_add(measurement, "cpu0", loadavg);
       evel_measurement_fsys_use_add(measurement,"00-11-22",100.11, 100.22, 33,
                                                            200.11, 200.22, 44);
       evel_measurement_fsys_use_add(measurement,"33-44-55",300.11, 300.22, 55,
diff --git a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
index 5307435..5cc9a09 100644
--- a/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
+++ b/tests/blueprints/tosca-vnfd-hello-ves/monitor.py
@@ -34,13 +34,12 @@ import json
 import jsonschema
 import select
 
-report_time = ""
-requestRate = ""
-vfStatus = ""
+report_time = ''
+requestRate = ''
 monitor_mode = "f"
-summary = ""
-status = ""
-vfStatus = ""
+summary = ['***** Summary of key stats *****','','','']
+status = ['','unknown','unknown','unknown']
+vdu = 0
 base_url = ''
 template_404 = b'''POST {0}'''
 columns = 0
@@ -97,39 +96,39 @@ class PathDispatcher:
 def process_event(e):
   global status
   global summary
-  global vfStatus
 
   epoch = e.event.commonEventHeader.lastEpochMicrosec
 
   report_time = time.strftime('%Y-%m-%d %H:%M:%S', 
                   time.localtime(int(epoch)/1000000))
 
-  domain = e.event.commonEventHeader.domain
-
-  if domain == 'measurementsForVfScaling':
-
-    aggregateCpuUsage = e.event.measurementsForVfScaling.aggregateCpuUsage
-    requestRate = e.event.measurementsForVfScaling.requestRate
-    summary = report_time + " app state: " + vfStatus + ", request rate: " + str(requestRate)
-    if monitor_mode == "c": print '{0} *** app state: {1}\trequest rate: {2}'.format(
-      report_time, vfStatus, str(requestRate))
+  host = e.event.commonEventHeader.reportingEntityName
+  if 'VDU1' in host or 'vdu1' in host: vdu = 1
+  if 'VDU2' in host or 'vdu2' in host: vdu = 2
+  if 'VDU3' in host or 'vdu3' in host: vdu = 3
 
-  if domain == 'fault':
-
-    alarmCondition = e.event.faultFields.alarmCondition
-    specificProblem = e.event.faultFields.specificProblem
-#    vfStatus = e.event.faultFields.vfStatus
-    vfStatus = e.event.faultFields.specificProblem
+  domain = e.event.commonEventHeader.domain
 
-    status = report_time + " app state change: " + specificProblem
-    if monitor_mode == "c": print '{0} *** vfStatus change: {1}'.format(report_time,
-                      specificProblem)
+  if e.event.commonEventHeader.functionalRole == 'vHello_VES agent':
+    if domain == 'measurementsForVfScaling':
+      aggregateCpuUsage = e.event.measurementsForVfScaling.aggregateCpuUsage
+      requestRate = e.event.measurementsForVfScaling.requestRate
+      summary[vdu] = "VDU" + str(vdu) + " state=" + status[vdu] + ", tps=" + str(requestRate) + ", cpu=" + str(aggregateCpuUsage)
+      if monitor_mode == "c": print '{0} *** VDU{1} state={2}, tps={3}'.format(
+        report_time, vdu, status[vdu], str(requestRate))
+
+    if domain == 'fault':
+      alarmCondition = e.event.faultFields.alarmCondition
+      specificProblem = e.event.faultFields.specificProblem
+#    status[vdu] = e.event.faultFields.vfStatus
+      status[vdu] = e.event.faultFields.specificProblem
+      if monitor_mode == "c": print '{0} *** VDU{1} state: {2}'.format(
+        report_time, vdu, status[vdu])
 
 # print_there only works if SSH'd to the VM manually - need to investigate
 #  print_there(1,columns-56,summary)
-  print '{0}'.format(summary)
-#  print_there(2,columns-56,status)
-  print '{0}'.format(status)
+    for s in summary:
+      print '{0}'.format(s)
 
 #--------------------------------------------------------------------------
 # Main monitoring and logging procedure
diff --git a/tests/blueprints/tosca-vnfd-hello-ves/start.sh b/tests/blueprints/tosca-vnfd-hello-ves/start.sh
index 1aa01e8..23f6080 100755
--- a/tests/blueprints/tosca-vnfd-hello-ves/start.sh
+++ b/tests/blueprints/tosca-vnfd-hello-ves/start.sh
@@ -19,11 +19,17 @@
 # Status: this is a work in progress, under test.
 #
 # How to use:
-# $ bash start.sh ID IP username password
-#   ID: username:password to use in REST
-#   IP: IP address of the collector
+# Intended to be invoked from vHello_VES.sh
+# $ bash start.sh type params
+#   type: type of VNF component [webserver|lb|monitor]
+#     webserver params: ID CollectorIP username password
+#     lb params:        ID CollectorIP username password app1_ip app2_ip
+#     collector params: ID CollectorIP username password
+#   ID: VM ID
+#   CollectorIP: IP address of the collector
 #   username: Username for Collector RESTful API authentication
 #   password: Password for Collector RESTful API authentication
+#   app1_ip app2_ip: address of the web servers
 
 setup_agent () {
   echo "$0: Install prerequisites"
@@ -46,31 +52,60 @@ setup_agent () {
   sed -i -- "/api_secure,/{n;s/.*/                      \"$username\",/}" evel-library/code/evel_demo/evel_demo.c
   sed -i -- "/\"hello\",/{n;s/.*/                      \"$password\",/}" evel-library/code/evel_demo/evel_demo.c
 
-  echo "$0: Build agent demo"
+  echo "$0: Build evel_demo agent"
   cd evel-library/bldjobs
   make
   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/ubuntu/evel-library/libs/x86_64
   
-  nohup ../output/x86_64/evel_demo --id $agent_id --fqdn $collector_ip --port 30000 --username $username --password $password &
-
-#  echo "$0: Start agent demo, repeat every minute"
-#  crontab -l > /tmp/cron
-#  echo "* * * * 1-5 /home/ubuntu/evel-library/output/x86_64/evel_demo --id $agent_id --fqdn $collector_ip --port 30000 --username $username --password $password" >> /tmp/cron
-#  crontab /tmp/cron
-#  rm /tmp/cron
+  echo "$0: Start evel_demo agent"
+  nohup ../output/x86_64/evel_demo --id $vm_id --fqdn $collector_ip --port 30000 --username $username --password $password > /dev/null 2>&1 &
+
+  echo "$0: Install collectd plugin"
+  cd ~
+  git clone https://github.com/maryamtahhan/OpenStackBarcelonaDemo.git
+  cd OpenStackBarcelonaDemo/ves_plugin
+  host=$(hostname)
+  sed -i -- "s/23380d70-2c71-4e35-99e2-f43f97e4ec65/$vm_id/g" ves_plugin.py
+  sed -i -- "s/cscf0001vm001abc001/$host/g" ves_plugin.py
+  sed -i -- "s/cscf0001vm001oam001/$host/g" ves_plugin.py
+  sed -i -- "s/SGW/$type/" ves_plugin.py
+  sed -i -- "s/reporting_entity_id = \"\"/reporting_entity_id = \"$vm_id\"/g" ves_plugin.py
+  sed -i -- "s/reporting_entity_name = \"cscf0001vm001oam001\"/reporting_entity_name = \"$host\"/g" ves_plugin.py 
+  sed -i -- "s/self.__username = ''/self.__username = 'hello'/g" ves_plugin.py
+  sed -i -- "s/self.__password = ''/self.__password = 'world'/g" ves_plugin.py
+  sed -i -- "s/self.__domain = '127.0.0.1'/self.__domain = '$collector_ip'/g" ves_plugin.py
+  sed -i -- "s/measurementForVfScaling/measurementsForVfScaling/g" ves_plugin.py
+
+  sudo apt-get install -y collectd
+  sudo sed -i -- "s/FQDNLookup true/FQDNLookup false/" /etc/collectd/collectd.conf
+  cat <<EOF | sudo tee -a  /etc/collectd/collectd.conf
+<LoadPlugin python>
+  Globals true
+</LoadPlugin>
+
+<Plugin python>
+  ModulePath "/home/ubuntu/OpenStackBarcelonaDemo/ves_plugin/"
+  LogTraces true
+  Interactive false
+  Import "ves_plugin"
+</Plugin>
+EOF
+  sudo service collectd restart
 }
 
-echo "$0: Setup website and dockerfile"
-mkdir ~/www
-mkdir ~/www/html
+setup_webserver () {
+  echo "$0: Setup website and dockerfile"
+  mkdir ~/www
+  mkdir ~/www/html
 
-# ref: https://hub.docker.com/_/nginx/
-cat > ~/www/Dockerfile <<EOM
+  # ref: https://hub.docker.com/_/nginx/
+  cat > ~/www/Dockerfile <<EOM
 FROM nginx
 COPY html /usr/share/nginx/html
 EOM
 
-cat > ~/www/html/index.html <<EOM
+  host=$(hostname)
+  cat > ~/www/html/index.html <<EOM
 <!DOCTYPE html>
 <html>
 <head>
@@ -82,38 +117,84 @@ body { width: 100%; background-color: white; color: black; padding: 0px; margin:
 </head>
 <body>
 Hello World!<br>
+Welcome to OPNFV @ $host!</large><br/>
 <a href="http://wiki.opnfv.org"><img src="https://www.opnfv.org/sites/all/themes/opnfv/logo.png"></a>
 </body></html>
 EOM
 
-cp ~/ves/tests/blueprints/tosca-vnfd-hello-ves/favicon.ico ~/www/html/favicon.ico
-
-echo "$0: Install docker"
-# Per https://docs.docker.com/engine/installation/linux/ubuntulinux/
-# Per https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04
-sudo apt-get install apt-transport-https ca-certificates
-sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
-echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
-sudo apt-get update
-sudo apt-get purge lxc-docker
-sudo apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual
-sudo apt-get install -y docker-engine
-
-echo "$0: Get nginx container and start website in docker"
-# Per https://hub.docker.com/_/nginx/
-sudo docker pull nginx
-cd ~/www
-sudo docker build -t vhello .
-sudo docker run --name vHello -d -p 80:80 vhello
-
-echo "$0: setup VES event delivery for the nginx server"
-
-# id=$(sudo ls /var/lib/docker/containers)
-# sudo tail -f /var/lib/docker/containers/$id/$id-json.log 
-
-agent_id=$1
-collector_ip=$2
-username=$3
-password=$4
-
-setup_agent
+  wget https://git.opnfv.org/cgit/ves/plain/tests/blueprints/tosca-vnfd-hello-ves/favicon.ico -O  ~/www/html/favicon.ico
+
+  echo "$0: Install docker"
+  # Per https://docs.docker.com/engine/installation/linux/ubuntulinux/
+  # Per https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-16-04
+  sudo apt-get install apt-transport-https ca-certificates
+  sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+  echo "deb https://apt.dockerproject.org/repo ubuntu-xenial main" | sudo tee /etc/apt/sources.list.d/docker.list
+  sudo apt-get update
+  sudo apt-get purge lxc-docker
+  sudo apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual
+  sudo apt-get install -y docker-engine
+
+  echo "$0: Get nginx container and start website in docker"
+  # Per https://hub.docker.com/_/nginx/
+  sudo docker pull nginx
+  cd ~/www
+  sudo docker build -t vhello .
+  sudo docker run --name vHello -d -p 80:80 vhello
+
+  echo "$0: setup VES agents"
+  setup_agent
+
+  # Debug hints
+  # id=$(sudo ls /var/lib/docker/containers)
+  # sudo tail -f /var/lib/docker/containers/$id/$id-json.log \
+  }
+
+setup_lb () {
+  echo "$0: setup VES load balancer"
+  echo "$0: install dependencies"
+  sudo apt-get update
+
+  echo "$0: Setup iptables rules"
+  echo "1" | sudo tee /proc/sys/net/ipv4/ip_forward
+  sudo sysctl net.ipv4.ip_forward=1
+  sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -m state --state NEW -m statistic --mode nth --every 2 --packet 0 -j DNAT --to-destination $app1_ip:80
+  sudo iptables -t nat -A PREROUTING -p tcp --dport 80 -m state --state NEW -m statistic --mode nth --every 2 --packet 0 -j DNAT --to-destination $app2_ip:80
+  sudo iptables -t nat -A POSTROUTING -j MASQUERADE
+  # debug hints: list rules (sudo iptables -S -t nat), flush (sudo iptables -F -t nat)
+
+  echo "$0: setup VES agents"
+  setup_agent
+}
+
+setup_monitor () {
+  echo "$0: setup VES Monitor"
+  echo "$0: install dependencies"
+  # Note below: python (2.7) is required due to dependency on module 'ConfigParser'
+  sudo apt-get update
+  sudo apt-get upgrade -y
+  sudo apt-get install -y python python-jsonschema
+
+  echo "$0: setup VES Monitor config"
+  sudo mkdir /var/log/att
+  sudo chown ubuntu /var/log/att
+  touch /var/log/att/collector.log
+  sudo chown ubuntu /home/ubuntu/
+  cd /home/ubuntu/
+  git clone https://github.com/att/evel-test-collector.git
+  sed -i -- "s/vel_username = /vel_username = $username/" evel-test-collector/config/collector.conf
+  sed -i -- "s/vel_password = /vel_password = $password/" evel-test-collector/config/collector.conf
+
+  python monitor.py --config evel-test-collector/config/collector.conf --section default 
+}
+
+type=$1
+vm_id=$2
+collector_ip=$3
+username=$4
+password=$5
+app1_ip=$6
+app2_ip=$7
+
+setup_$type
+exit 0
diff --git a/tests/vHello_VES.sh b/tests/vHello_VES.sh
index b0d0d0f..ea05d7f 100644
--- a/tests/vHello_VES.sh
+++ b/tests/vHello_VES.sh
@@ -21,7 +21,7 @@
 # How to use:
 #   $ git clone https://gerrit.opnfv.org/gerrit/ves
 #   $ cd ves/tests
-#   $ bash vHello_VES.sh [setup|start|run|test|stop|clean] 
+#   $ bash vHello_VES.sh [setup|start|run|test|stop|clean] [VDU1|VDU2|VDU3]
 #        [monitor|traffic|pause|nic]
 #   setup: setup test environment
 #   start: install blueprint and run test
@@ -33,19 +33,18 @@
 #     monitor: attach to the collector VM and run the VES Monitor
 #     traffic: generate some traffic
 #     pause: pause the VNF (web server) for a minute to generate a state change
-#     nic: timed ifdown/ifup to generate a NIC fault report
+#       VDU1|VDU2
+#     nic: timed ifdown/ifup to generate a NIC fault report (not yet implemented)
 
 trap 'fail' ERR
 
 pass() {
-  echo "$0: Hooray!"
-  set +x #echo off
+  echo "$0: $(date) Hooray!"
   exit 0
 }
 
 fail() {
-  echo "$0: Test Failed!"
-  set +x
+  echo "$0: $(date) Test Failed!"
   exit 1
 }
 
@@ -57,7 +56,7 @@ get_floating_net () {
   if [[ $FLOATING_NETWORK_ID ]]; then
     FLOATING_NETWORK_NAME=$(openstack network show $FLOATING_NETWORK_ID | awk "/ name / { print \$4 }")
   else
-    echo "$0: Floating network not found"
+    echo "$0: $(date) Floating network not found"
     exit 1
   fi
 }
@@ -71,44 +70,45 @@ try () {
     let count=$count-1
     $3
   done
-  if [[ $count -eq 0 ]]; then echo "$0: Command \"$3\" was not successful after $1 tries"; fi
+  if [[ $count -eq 0 ]]; then echo "$0: $(date) Command \"$3\" was not successful after $1 tries"; fi
 }
 
 setup () {
-  echo "$0: Setup temp test folder /tmp/tacker and copy this script there"
+  echo "$0: $(date) Started"
+  echo "$0: $(date) Setup temp test folder /tmp/tacker and copy this script there"
   mkdir -p /tmp/tacker
   chmod 777 /tmp/tacker/
   cp $0 /tmp/tacker/.
   chmod 755 /tmp/tacker/*.sh
 
-  echo "$0: tacker-setup part 1"
+  echo "$0: $(date) tacker-setup part 1"
   wget https://git.opnfv.org/cgit/models/plain/tests/utils/tacker-setup.sh -O /tmp/tacker/tacker-setup.sh
   bash /tmp/tacker/tacker-setup.sh tacker-cli init
 
-  echo "$0: tacker-setup part 2"
+  echo "$0: $(date) tacker-setup part 2"
   CONTAINER=$(sudo docker ps -l | awk "/tacker/ { print \$1 }")
   dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`
   if [ "$dist" == "Ubuntu" ]; then
-    echo "$0: JOID workaround for Colorado - enable ML2 port security"
+    echo "$0: $(date) JOID workaround for Colorado - enable ML2 port security"
     juju set neutron-api enable-ml2-port-security=true
 
-    echo "$0: Execute tacker-setup.sh in the container"
+    echo "$0: $(date) Execute tacker-setup.sh in the container"
     sudo docker exec -it $CONTAINER /bin/bash /tmp/tacker/tacker-setup.sh tacker-cli setup
   else
-    echo "$0: Execute tacker-setup.sh in the container"
+    echo "$0: $(date) Execute tacker-setup.sh in the container"
     sudo docker exec -i -t $CONTAINER /bin/bash /tmp/tacker/tacker-setup.sh tacker-cli setup
   fi
 
-  echo "$0: reset blueprints folder"
+  echo "$0: $(date) reset blueprints folder"
   if [[ -d /tmp/tacker/blueprints/tosca-vnfd-hello-ves ]]; then rm -rf /tmp/tacker/blueprints/tosca-vnfd-hello-ves; fi
   mkdir -p /tmp/tacker/blueprints/tosca-vnfd-hello-ves
 
-  echo "$0: copy tosca-vnfd-hello-ves to blueprints folder"
+  echo "$0: $(date) copy tosca-vnfd-hello-ves to blueprints folder"
   cp -r blueprints/tosca-vnfd-hello-ves /tmp/tacker/blueprints
 
   # Following two steps are in testing still. The guestfish step needs work.
 
-  #  echo "$0: Create Nova key pair"
+  #  echo "$0: $(date) Create Nova key pair"
   #  mkdir -p ~/.ssh
   #  nova keypair-delete vHello
   #  nova keypair-add vHello > /tmp/tacker/vHello.pem
@@ -116,7 +116,7 @@ setup () {
   #  pubkey=$(nova keypair-show vHello | grep "Public key:" | sed -- 's/Public key: //g')
   #  nova keypair-show vHello | grep "Public key:" | sed -- 's/Public key: //g' >/tmp/tacker/vHello.pub
 
-  echo "$0: Inject key into xenial server image"
+  echo "$0: $(date) Inject key into xenial server image"
   #  wget http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
   #  sudo yum install -y libguestfs-tools
   #  guestfish <<EOF
@@ -139,58 +139,68 @@ setup () {
   cp blueprints/tosca-vnfd-hello-ves/vHello.pem /tmp/tacker
   chmod 600 /tmp/tacker/vHello.pem
 
-  echo "$0: setup OpenStack CLI environment"
+  echo "$0: $(date) setup OpenStack CLI environment"
   source /tmp/tacker/admin-openrc.sh
 
-  echo "$0: Setup image_id"
+  echo "$0: $(date) Setup image_id"
   image_id=$(openstack image list | awk "/ models-xenial-server / { print \$2 }")
-  if [[ -z "$image_id" ]]; then glance --os-image-api-version 1 image-create --name models-xenial-server --disk-format qcow2 --file /tmp/xenial-server-cloudimg-amd64-disk1.img --container-format bare; fi 
+  if [[ -z "$image_id" ]]; then glance --os-image-api-version 1 image-create --name models-xenial-server --disk-format qcow2 --file /tmp/xenial-server-cloudimg-amd64-disk1.img --container-format bare; fi
+
+  echo "$0: $(date) Completed"
 }
 
 start() {
-  echo "$0: setup OpenStack CLI environment"
+  echo "$0: $(date) Started"
+  echo "$0: $(date) setup OpenStack CLI environment"
   source /tmp/tacker/admin-openrc.sh
 
-  echo "$0: create VNFD"
+  echo "$0: $(date) create VNFD"
   cd /tmp/tacker/blueprints/tosca-vnfd-hello-ves
   tacker vnfd-create --vnfd-file blueprint.yaml --name hello-ves
   if [ $? -eq 1 ]; then fail; fi
 
-  echo "$0: create VNF"
+  echo "$0: $(date) create VNF"
   tacker vnf-create --vnfd-name hello-ves --name hello-ves
   if [ $? -eq 1 ]; then fail; fi
 
-  echo "$0: wait for hello-ves to go ACTIVE"
+  echo "$0: $(date) wait for hello-ves to go ACTIVE"
   active=""
   while [[ -z $active ]]
   do
     active=$(tacker vnf-show hello-ves | grep ACTIVE)
     if [ "$(tacker vnf-show hello-ves | grep -c ERROR)" == "1" ]; then 
-      echo "$0: hello-ves VNF creation failed with state ERROR"
+      echo "$0: $(date) hello-ves VNF creation failed with state ERROR"
       fail
     fi
     sleep 10
   done
 
-  echo "$0: directly set port security on ports (bug/unsupported in Mitaka Tacker?)"
+  echo "$0: $(date) directly set port security on ports (bug/unsupported in Mitaka Tacker?)"
   vdus="VDU1 VDU2 VDU3 VDU4"
   vdui="1 2 3 4"
   declare -a vdu_id=()
   declare -a vdu_ip=()
   declare -a vdu_url=()
   HEAT_ID=$(tacker vnf-show hello-ves | awk "/instance_id/ { print \$4 }")
-  for i in $vdui; do
-    vdu_id[$i]=$(openstack stack resource list $HEAT_ID | awk "/$vdu / { print \$4 }")
-  done
-
+  vdu_id[1]=$(openstack stack resource list $HEAT_ID | awk "/VDU1 / { print \$4 }")
+  vdu_id[2]=$(openstack stack resource list $HEAT_ID | awk "/VDU2 / { print \$4 }")
+  vdu_id[3]=$(openstack stack resource list $HEAT_ID | awk "/VDU3 / { print \$4 }")
+  vdu_id[4]=$(openstack stack resource list $HEAT_ID | awk "/VDU4 / { print \$4 }")
+
+cat >/tmp/grep <<EOF
+${vdu_id[1]}
+${vdu_id[2]}
+${vdu_id[3]}
+${vdu_id[4]}
+EOF
   id=($(neutron port-list|grep -v "+"|grep -v name|awk '{print $2}'))
   for id in ${id[@]}; do
-    if [[ $(neutron port-show $id|grep -e ${vdu_id[1]} -e ${vdu_id[2]} -e ${vdu_id[3]} -e ${vdu_id[4]}) ]]; then 
+    if [[ $(neutron port-show $id | grep -f /tmp/grep) ]]; then 
       neutron port-update ${id} --port-security-enabled=True
     fi
   done
 
-  echo "$0: directly assign security group (unsupported in Mitaka Tacker)"
+  echo "$0: $(date) directly assign security group (unsupported in Mitaka Tacker)"
   if [[ $(openstack security group list | awk "/ vHello / { print \$2 }") ]]; then openstack security group delete vHello; fi
   openstack security group create vHello
   openstack security group rule create --ingress --protocol TCP --dst-port 22:22 vHello
@@ -200,79 +210,65 @@ start() {
     openstack server add security group ${vdu_id[$i]} default
   done
 
-  echo "$0: associate floating IPs"
+  echo "$0: $(date) associate floating IPs"
   get_floating_net
   for i in $vdui; do
     vdu_ip[$i]=$(openstack floating ip create $FLOATING_NETWORK_NAME | awk "/floating_ip_address/ { print \$4 }")
     nova floating-ip-associate ${vdu_id[$i]} ${vdu_ip[$i]}
   done
 
-  echo "$0: get web server addresses"
+  echo "$0: $(date) get web server addresses"
   vdu_url[1]="http://${vdu_ip[1]}"
   vdu_url[2]="http://${vdu_ip[3]}"
   vdu_url[3]="http://${vdu_ip[3]}"
   vdu_url[4]="http://${vdu_ip[4]}:30000/eventListener/v1"
 
-  echo "$0: wait 30 seconds for server SSH to be available"
+  echo "$0: $(date) wait 30 seconds for server SSH to be available"
   sleep 30
 
-  echo "$0: Setup the VES Collector in VDU4"
-  chown root /tmp/tacker/vHello.pem
-  # Note below: python (2.7) is required due to dependency on module 'ConfigParser'
-  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} << EOF
-sudo apt-get update
-sudo apt-get upgrade -y
-sudo apt-get install -y python python-jsonschema
-sudo mkdir /var/log/att
-sudo chown ubuntu /var/log/att
-touch /var/log/att/collector.log
-sudo chown ubuntu /home/ubuntu/
-cd /home/ubuntu/
-git clone https://github.com/att/evel-test-collector.git
-sed -i -- 's/vel_username = /vel_username = hello/' evel-test-collector/config/collector.conf
-sed -i -- 's/vel_password = /vel_password = world/' evel-test-collector/config/collector.conf
-EOF
-  # Replacing the default collector with monitor.py which has processing logic as well
-  scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/monitor.py ubuntu@${vdu_ip[4]}:/home/ubuntu/monitor.py
-
-  echo "$0: start LB in VDU3"
-  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[3]} <<EOF
-sudo iptables -A PREROUTING -i eth0 -p tcp --dport 80 -m state --state NEW -m nth --counter 0 --every 2 --packet 0 -j DNAT --to-destination ${vdu_ip[1]}
-sudo iptables -A PREROUTING -i eth0 -p tcp --dport 80 -m state --state NEW -m nth --counter 0 --every 2 --packet 1 -j DNAT --to-destination ${vdu_ip[2]}
-EOF
-
-  echo "$0: start vHello web server in VDU1/VDU2"
-  vdu1="1 2"
+  echo "$0: $(date) Copy startup script to the VMs"
   for i in $vdui; do
-    echo "$0: start vHello web server at ${vdu_ip[$i]}"
     ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[$i]} "sudo chown ubuntu /home/ubuntu"
     scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/start.sh ubuntu@${vdu_ip[$i]}:/home/ubuntu/start.sh
-    ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[$i]} "bash /home/ubuntu/start.sh ${vdu_id[$i]} ${vdu_ip[$i]} hello world"
   done
 
-  echo "$0: verify vHello server is running at http://${vdu_ip[3]}"
-  apt-get install -y curl
-  count=10
-  while [[ $count -gt 0 ]] 
-  do 
-    sleep 60
-    let count=$count-1
-    if [[ $(curl http://${vdu_ip[3]} | grep -c "Hello World") > 0 ]]; then pass; fi
-  done
-  fail
+  echo "$0: $(date) start vHello webserver in VDU1 at ${vdu_ip[1]}"
+  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[1]} "bash /home/ubuntu/start.sh webserver ${vdu_id[1]} ${vdu_ip[4]} hello world; exit"
+
+  echo "$0: $(date) start vHello webserver in VDU2 at ${vdu_ip[2]}"
+  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[2]} "bash /home/ubuntu/start.sh webserver ${vdu_id[2]} ${vdu_ip[4]} hello world; exit"
+
+  echo "$0: $(date) start LB in VDU3 at ${vdu_ip[3]}"
+  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[3]} "bash /home/ubuntu/start.sh lb ${vdu_id[3]} ${vdu_ip[4]} hello world; exit"
+
+  echo "$0: $(date) start Monitor in VDU4 at ${vdu_ip[4]}"
+  # Replacing the default collector with monitor.py which has processing logic as well
+  scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/monitor.py ubuntu@${vdu_ip[4]}:/home/ubuntu/monitor.py
+  ssh -i /tmp/tacker/vHello.pem -t -t -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} "bash /home/ubuntu/start.sh monitor ${vdu_id[4]} ${vdu_ip[4]} hello world; exit"
+
+#  echo "$0: $(date) verify vHello server is running at http://${vdu_ip[3]}"
+#  apt-get install -y curl
+#  count=10
+#  while [[ $count -gt 0 ]] 
+#  do 
+#    sleep 60
+#    let count=$count-1
+#    if [[ $(curl http://${vdu_ip[3]} | grep -c "Hello World") > 0 ]]; then pass; fi
+#  done
+#  fail
 }
 
 stop() {
-  echo "$0: setup OpenStack CLI environment"
+  echo "$0: $(date) setup OpenStack CLI environment"
   source /tmp/tacker/admin-openrc.sh
 
-  echo "$0: uninstall vHello blueprint via CLI"
+  echo "$0: $(date) uninstall vHello blueprint via CLI"
   vid=($(tacker vnf-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnf-delete ${id};  done
   vid=($(tacker vnfd-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnfd-delete ${id};  done
 # Need to remove the floatingip deletion or make it specific to the vHello VM
-#  fip=($(neutron floatingip-list|grep -v "+"|grep -v id|awk '{print $2}')); for id in ${fip[@]}; do neutron floatingip-delete ${id};  done
+  fip=($(neutron floatingip-list|grep -v "+"|grep -v id|awk '{print $2}')); for id in ${fip[@]}; do neutron floatingip-delete ${id};  done
   sg=($(openstack security group list|grep vHello|awk '{print $2}'))
-  for id in ${sg[@]}; do try 5 5 "openstack security group delete ${id}";  done
+  for id in ${sg[@]}; do try 10 5 "openstack security group delete ${id}";  done
 }
 
 #
@@ -282,12 +278,12 @@ stop() {
 get_vdu_ip () {
   source /tmp/tacker/admin-openrc.sh
 
-  echo "$0: find VM IP for $1"
+  echo "$0: $(date) find VM IP for $1"
   ip=$(openstack server list | awk "/$1/ { print \$10 }")
 }
 
 monitor () {
-  echo "$0: Start the VES Monitor in VDU4 - Stop first if running"
+  echo "$0: $(date) Start the VES Monitor in VDU4 - Stop first if running"
   get_vdu_ip VDU4
   sudo cp /tmp/tacker/vHello.pem /tmp/vHello.pem
   sudo chown $USER:$USER /tmp/vHello.pem
@@ -299,26 +295,29 @@ EOF
 }
 
 traffic () {
-  echo "$0: Generate some traffic, somewhat randomly"
+  echo "$0: $(date) Generate some traffic, somewhat randomly"
   get_vdu_ip VDU3
+  ns="0 00 000"
   while true
   do
-    sleep .0$[ ( $RANDOM % 10 ) + 1 ]s
-    curl -s http://$ip > /dev/null
+    for n in $ns; do
+      sleep .$n$[ ( $RANDOM % 10 ) + 1 ]s
+      curl -s http://$ip > /dev/null
+    done
   done
 }
 
 pause () {
-  echo "$0: Pause the VNF (web server) in VDU1 for a minute to generate a state change fault report (Stopped)"
-  get_vdu_ip VDU1
+  echo "$0: $(date) Pause the VNF (web server) in $1 for a minute to generate a state change fault report (Stopped)"
+  get_vdu_ip $1
   ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker pause vHello"
   sleep 60
-  echo "$0: Unpausing the VNF to generate a state change fault report (Started)"
+  echo "$0: $(date) Unpausing the VNF to generate a state change fault report (Started)"
   ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker unpause vHello"
 }
 
 forward_to_container () {
-  echo "$0: pass $1 command to this script in the tacker container"
+  echo "$0: $(date) pass $1 command to this script in the tacker container"
   CONTAINER=$(sudo docker ps -a | awk "/tacker/ { print \$1 }")
   sudo docker exec $CONTAINER /bin/bash /tmp/tacker/vHello_VES.sh $1 $1
   if [ $? -eq 1 ]; then fail; fi
@@ -348,10 +347,10 @@ case "$1" in
     pass
     ;;
   test)
-    $2
+    $2 $3
     ;;
   clean)
-    echo "$0: Uninstall Tacker and test environment"
+    echo "$0: $(date) Uninstall Tacker and test environment"
     bash /tmp/tacker/tacker-setup.sh $1 clean
     pass
     ;;