Update for Danube (work in progress)

JIRA: VES-1

Removed preliminary versions of tests for now.
Aligned with the Models vHello_3Node_Tacker test.
Separated out actions for collectd and monitor install and
run them directly vs thru the container.
Added parameters as needed for the new design.
Updated with typical overview of the test/demo flow.
Add port 30000 ingress to vHello security group.
Remove juju workaround for port_security.
Remote use of id_rsa (replaced with notes on ssh assumptions).
Use VDU3 (LB) in traffic generation.

Change-Id: I42807449f7526efcd191c0b7b0b4754ffcfffa6e
Signed-off-by: bryan <bryan.sullivan@att.com>

1 files changed, 364 insertions, 233 deletions

diff --git a/tests/vHello_VES.sh b/tests/vHello_VES.sh
index cc85ebc..bbddfa5 100644
--- a/tests/vHello_VES.sh
+++ b/tests/vHello_VES.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2016 AT&T Intellectual Property, Inc
+# Copyright 2016-2017 AT&T Intellectual Property, Inc
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,27 +14,54 @@
 # limitations under the License.
 #
 # What this is: Deployment test for the VES agent and collector based 
-# upon the Tacker Hello World blueprint 
+# upon the Tacker Hello World blueprint, designed as a manual demo of the VES
+# concept and integration with the Barometer project collectd agent. Typical 
+# demo procedure is to execute the following actions from the OPNFV jumphost
+# or some host wth access to the OpenStack controller (see below for details):
+#  setup: install Tacker in a docker container. Note: only needs to be done
+#         once per session, and can be reused across OPNFV VES and Models tests,
+#         i.e. you can start another test at the "start" step below.
+#  start: install blueprint and start the VNF, including the app (load-balanced
+#         web server) and VES agents running on the VMs. Installs the VES 
+#         monitor code but does not start the monitor (see below).
+#  start_collectd: start the collectd daemon on bare metal hypervisor hosts
+#  monitor: start the VES monitor, typically run in a second shell session.
+#  pause: pause the app at one of the web server VDUs (VDU1 or VDU2)
+#  stop: stop the VNF and uninstall the blueprint
+#  start_collectd: start the collectd daemon on bare metal hypervisor hosts
+#  clean: remove the tacker container and service (if desired, when done)
 #
 # Status: this is a work in progress, under test.
 #
 # How to use:
 #   $ git clone https://gerrit.opnfv.org/gerrit/ves
 #   $ cd ves/tests
-#   $ bash vHello_VES.sh [setup|start|run|test|stop|clean] [VDU1|VDU2|VDU3]
-#        [monitor|traffic|pause|nic]
-#   setup: setup test environment
-#   start: install blueprint and run test
-#   run: setup test environment and run test
-#   test: run test tools/scenario - see below
-#   stop: stop test and uninstall blueprint
-#   clean: cleanup after test
-#   Test:
+#   $ bash vHello_VES.sh setup <openrc> [branch]
+#     setup: setup test environment
+#     <openrc>: location of OpenStack openrc file
+#     branch: OpenStack branch to install (default: master)
+#   $ bash vHello_VES.sh start
+#     start: install blueprint and run test
+#   $ bash vHello_VES.sh start_collectd|stop_collectd <ip> <user> <monitor_ip>
+#     start_collectd: install and start collectd daemon on hypervisor
+#     stop_collectd: stop and uninstall collectd daemon on hypervisor
+#     <ip>: hypervisor ip 
+#     <user>: username on hypervisor hosts, for ssh (user must be setup for 
+#       key-based auth on the hosts)
+#   $ bash vHello_VES.sh monitor
 #     monitor: attach to the collector VM and run the VES Monitor
+#   $ bash vHello_VES.sh traffic
 #     traffic: generate some traffic
+#   $ bash vHello_VES.sh pause VDU1|VDU2
 #     pause: pause the VNF (web server) for a minute to generate a state change
-#       VDU1|VDU2
-#     nic: timed ifdown/ifup to generate a NIC fault report (not yet implemented)
+#     VDU1: Pause VDU1
+#     VDU2: Pause VDU2
+#   $ bash vHello_VES.sh stop
+#     stop: stop test and uninstall blueprint
+#   $ bash vHello_VES.sh clean  <hpvuser> <hpvpw>
+#     clean: cleanup after test
+#     <hpvuser>: username on hypervisor
+#     <hpvpw>: password on hypervisor
 
 trap 'fail' ERR
 
@@ -48,13 +75,21 @@ fail() {
   exit 1
 }
 
+assert() {
+  if [[ $2 == true ]]; then echo "$0 test assertion passed: $1"
+  else 
+    echo "$0 test assertion failed: $1"
+    fail
+  fi
+}
+
 get_floating_net () {
   network_ids=($(neutron net-list|grep -v "+"|grep -v name|awk '{print $2}'))
   for id in ${network_ids[@]}; do
       [[ $(neutron net-show ${id}|grep 'router:external'|grep -i "true") != "" ]] && FLOATING_NETWORK_ID=${id}
   done
   if [[ $FLOATING_NETWORK_ID ]]; then
-    FLOATING_NETWORK_NAME=$(openstack network show $FLOATING_NETWORK_ID | awk "/ name / { print \$4 }")
+    FLOATING_NETWORK_NAME=$(neutron net-show $FLOATING_NETWORK_ID | awk "/ name / { print \$4 }")
   else
     echo "$0: $(date) Floating network not found"
     exit 1
@@ -64,8 +99,7 @@ get_floating_net () {
 try () {
   count=$1
   $3
-  while [[ $? -eq 1 && $count -gt 0 ]] 
-  do 
+  while [[ $? == 1 && $count > 0 ]]; do 
     sleep $2
     let count=$count-1
     $3
@@ -74,255 +108,289 @@ try () {
 }
 
 setup () {
-  echo "$0: $(date) Started"
-  echo "$0: $(date) Setup temp test folder /tmp/tacker and copy this script there"
-  mkdir -p /tmp/tacker
-  chmod 777 /tmp/tacker/
-  cp $0 /tmp/tacker/.
-  chmod 755 /tmp/tacker/*.sh
+  trap 'fail' ERR
+
+  echo "$0: $(date) Setup shared test folder /opt/tacker"
+  if [ -d /opt/tacker ]; then sudo rm -rf /opt/tacker; fi 
+  sudo mkdir -p /opt/tacker
+  sudo chown $USER /opt/tacker
+  chmod 777 /opt/tacker/
+
+  echo "$0: $(date) copy test script and openrc to /opt/tacker"
+  cp $0 /opt/tacker/.
+  cp $1 /opt/tacker/admin-openrc.sh
+
+  source /opt/tacker/admin-openrc.sh
+  chmod 755 /opt/tacker/*.sh
 
   echo "$0: $(date) tacker-setup part 1"
-  wget https://git.opnfv.org/cgit/models/plain/tests/utils/tacker-setup.sh -O /tmp/tacker/tacker-setup.sh
-  bash /tmp/tacker/tacker-setup.sh tacker-cli init
+  bash utils/tacker-setup.sh init
+  if [ $? -eq 1 ]; then fail; fi
 
   echo "$0: $(date) tacker-setup part 2"
-  CONTAINER=$(sudo docker ps -l | awk "/tacker/ { print \$1 }")
   dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`
   if [ "$dist" == "Ubuntu" ]; then
-    echo "$0: $(date) JOID workaround for Colorado - enable ML2 port security"
-    juju set neutron-api enable-ml2-port-security=true
-
     echo "$0: $(date) Execute tacker-setup.sh in the container"
-    sudo docker exec -it $CONTAINER /bin/bash /tmp/tacker/tacker-setup.sh tacker-cli setup
+    sudo docker exec -it tacker /bin/bash /opt/tacker/tacker-setup.sh setup $2
   else
-    echo "$0: $(date) Copy private key to the container (needed for later install steps)"
-    cp ~/.ssh/id_rsa /tmp/tacker/id_rsa
     echo "$0: $(date) Execute tacker-setup.sh in the container"
-    sudo docker exec -i -t $CONTAINER /bin/bash /tmp/tacker/tacker-setup.sh tacker-cli setup
+    sudo docker exec -i -t tacker /bin/bash /opt/tacker/tacker-setup.sh setup $2
   fi
 
-  echo "$0: $(date) reset blueprints folder"
-  if [[ -d /tmp/tacker/blueprints/tosca-vnfd-hello-ves ]]; then rm -rf /tmp/tacker/blueprints/tosca-vnfd-hello-ves; fi
-  mkdir -p /tmp/tacker/blueprints/tosca-vnfd-hello-ves
+  assert "models-tacker-001 (Tacker installation in a docker container on the jumphost)" true 
+}
 
-  echo "$0: $(date) copy tosca-vnfd-hello-ves to blueprints folder"
-  cp -r blueprints/tosca-vnfd-hello-ves /tmp/tacker/blueprints
-
-  # Following two steps are in testing still. The guestfish step needs work.
-
-  #  echo "$0: $(date) Create Nova key pair"
-  #  mkdir -p ~/.ssh
-  #  nova keypair-delete vHello
-  #  nova keypair-add vHello > /tmp/tacker/vHello.pem
-  #  chmod 600 /tmp/tacker/vHello.pem
-  #  pubkey=$(nova keypair-show vHello | grep "Public key:" | sed -- 's/Public key: //g')
-  #  nova keypair-show vHello | grep "Public key:" | sed -- 's/Public key: //g' >/tmp/tacker/vHello.pub
-
-  echo "$0: $(date) Inject key into xenial server image"
-  #  wget http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
-  #  sudo yum install -y libguestfs-tools
-  #  guestfish <<EOF
-#add xenial-server-cloudimg-amd64-disk1.img
-#run
-#mount /dev/sda1 /
-#mkdir /home/ubuntu
-#mkdir /home/ubuntu/.ssh
-#cat <<EOM >/home/ubuntu/.ssh/authorized_keys
-#$pubkey
-#EOM
-#exit
-#chown -R ubuntu /home/ubuntu
-#EOF
-
-  # Using pre-key-injected image for now, vHello.pem as provided in the blueprint
-  if [ ! -f /tmp/xenial-server-cloudimg-amd64-disk1.img ]; then 
-    wget -O /tmp/xenial-server-cloudimg-amd64-disk1.img  http://artifacts.opnfv.org/models/images/xenial-server-cloudimg-amd64-disk1.img
-  fi
-  cp blueprints/tosca-vnfd-hello-ves/vHello.pem /tmp/tacker
-  chmod 600 /tmp/tacker/vHello.pem
+say_hello() {
+  echo "$0: $(date) Testing $1"
+  pass=false
+  count=10
+  while [[ $count > 0 && $pass != true ]] 
+  do 
+    sleep 30
+    let count=$count-1
+    if [[ $(curl $1 | grep -c "Hello World") > 0 ]]; then
+      echo "$0: $(date) Hello World found at $1"
+      pass=true
+    fi
+  done
+  if [[ $pass != true ]]; then fail; fi
+}
 
-  echo "$0: $(date) setup OpenStack CLI environment"
-  source /tmp/tacker/admin-openrc.sh
+copy_blueprint() {
+  echo "$0: $(date) copy test script to /opt/tacker"
+  cp $0 /opt/tacker/.
 
-  echo "$0: $(date) Setup image_id"
-  image_id=$(openstack image list | awk "/ models-xenial-server / { print \$2 }")
-  if [[ -z "$image_id" ]]; then glance --os-image-api-version 1 image-create --name models-xenial-server --disk-format qcow2 --file /tmp/xenial-server-cloudimg-amd64-disk1.img --container-format bare; fi
+  echo "$0: $(date) reset blueprints folder"
+  if [[ -d /opt/tacker/blueprints/tosca-vnfd-hello-ves ]]; then 
+    rm -rf /opt/tacker/blueprints/tosca-vnfd-hello-ves
+  fi
 
-  echo "$0: $(date) Completed"
+  echo "$0: $(date) copy tosca-vnfd-hello-ves to blueprints folder"
+  cp -r blueprints/tosca-vnfd-hello-ves /opt/tacker/blueprints/tosca-vnfd-hello-ves
+  cp $0 /opt/tacker/.
 }
 
 start() {
-  echo "$0: $(date) Started"
+#  Disable trap for now, need to test to ensure premature fail does not occur
+#  trap 'fail' ERR
+
   echo "$0: $(date) setup OpenStack CLI environment"
-  source /tmp/tacker/admin-openrc.sh
+  source /opt/tacker/admin-openrc.sh
+
+  echo "$0: $(date) Create Nova key pair"
+  if [[ -f /opt/tacker/vHello ]]; then rm /opt/tacker/vHello; fi
+  ssh-keygen -t rsa -N "" -f /opt/tacker/vHello -C ubuntu@vHello
+  chmod 600 /opt/tacker/vHello
+  openstack keypair create --public-key /opt/tacker/vHello.pub vHello
+  assert "models-nova-001 (Keypair creation)" true 
+
+  echo "$0: $(date) Inject public key into blueprint"
+  pubkey=$(cat /opt/tacker/vHello.pub)
+  sed -i -- "s~<pubkey>~$pubkey~" /opt/tacker/blueprints/tosca-vnfd-hello-ves/blueprint.yaml
+
+  vdus="VDU1 VDU2 VDU3 VDU4"
+  vdui="1 2 3 4"
+  vnf_vdui="1 2 3"
+  declare -a vdu_id=()
+  declare -a vdu_ip=()
+  declare -a vdu_url=()
+
+  # Setup for workarounds
+  echo "$0: $(date) allocate floating IPs"
+  get_floating_net
+  for i in $vdui; do
+    vdu_ip[$i]=$(nova floating-ip-create $FLOATING_NETWORK_NAME | awk "/$FLOATING_NETWORK_NAME/ { print \$4 }")
+    echo "$0: $(date) Pre-allocated ${vdu_ip[$i]} to VDU$i"
+  done
+
+  echo "$0: $(date) Inject web server floating IPs into LB code in blueprint"
+  sed -i -- "s/<vdu1_ip>/${vdu_ip[1]}/" /opt/tacker/blueprints/tosca-vnfd-hello-ves/blueprint.yaml
+  sed -i -- "s/<vdu2_ip>/${vdu_ip[2]}/" /opt/tacker/blueprints/tosca-vnfd-hello-ves/blueprint.yaml
+  # End setup for workarounds
 
   echo "$0: $(date) create VNFD"
-  cd /tmp/tacker/blueprints/tosca-vnfd-hello-ves
-  tacker vnfd-create --vnfd-file blueprint.yaml --name hello-ves
-  if [ $? -eq 1 ]; then fail; fi
+  cd /opt/tacker/blueprints/tosca-vnfd-hello-ves
+  # newton: NAME (was "--name") is now a positional parameter
+  tacker vnfd-create --vnfd-file blueprint.yaml hello-ves
+  if [[ $? -eq 0 ]]; then 
+    assert "models-tacker-002 (VNFD creation)" true
+  else
+    assert "models-tacker-002 (VNFD creation)" false
+  fi
 
   echo "$0: $(date) create VNF"
-  tacker vnf-create --vnfd-name hello-ves --name hello-ves
+  # newton: NAME (was "--name") is now a positional parameter
+  tacker vnf-create --vnfd-name hello-ves hello-ves
   if [ $? -eq 1 ]; then fail; fi
 
   echo "$0: $(date) wait for hello-ves to go ACTIVE"
   active=""
-  while [[ -z $active ]]
+  count=24
+  while [[ -z $active && $count -gt 0 ]]
   do
     active=$(tacker vnf-show hello-ves | grep ACTIVE)
-    if [ "$(tacker vnf-show hello-ves | grep -c ERROR)" == "1" ]; then 
+    if [[ $(tacker vnf-show hello-ves | grep -c ERROR) > 0 ]]; then 
       echo "$0: $(date) hello-ves VNF creation failed with state ERROR"
-      fail
+      assert "models-tacker-002 (VNF creation)" false
     fi
-    sleep 10
+    let count=$count-1
+    sleep 30
+    echo "$0: $(date) wait for hello-ves to go ACTIVE"
   done
+  if [[ $count == 0 ]]; then 
+    echo "$0: $(date) hello-ves VNF creation failed - timed out"
+    assert "models-tacker-002 (VNF creation)" false
+  fi
 
-  echo "$0: $(date) directly set port security on ports (bug/unsupported in Mitaka Tacker?)"
-  vdus="VDU1 VDU2 VDU3 VDU4"
-  vdui="1 2 3 4"
-  declare -a vdu_id=()
-  declare -a vdu_ip=()
-  declare -a vdu_url=()
-  HEAT_ID=$(tacker vnf-show hello-ves | awk "/instance_id/ { print \$4 }")
-  vdu_id[1]=$(openstack stack resource list $HEAT_ID | awk "/VDU1 / { print \$4 }")
-  vdu_id[2]=$(openstack stack resource list $HEAT_ID | awk "/VDU2 / { print \$4 }")
-  vdu_id[3]=$(openstack stack resource list $HEAT_ID | awk "/VDU3 / { print \$4 }")
-  vdu_id[4]=$(openstack stack resource list $HEAT_ID | awk "/VDU4 / { print \$4 }")
-
-cat >/tmp/grep <<EOF
-${vdu_id[1]}
-${vdu_id[2]}
-${vdu_id[3]}
-${vdu_id[4]}
-EOF
-  id=($(neutron port-list|grep -v "+"|grep -v name|awk '{print $2}'))
-  for id in ${id[@]}; do
-    if [[ $(neutron port-show $id | grep -f /tmp/grep) ]]; then 
-      neutron port-update ${id} --port-security-enabled=True
-    fi
+  # Setup for workarounds
+  echo "$0: $(date) directly set port security on ports (unsupported in Mitaka Tacker)"
+  # Alternate method
+  #  HEAT_ID=$(tacker vnf-show hello-ves | awk "/instance_id/ { print \$4 }")
+  #  SERVER_ID=$(openstack stack resource list $HEAT_ID | awk "/VDU1 / { print \$4 }")
+  for vdu in $vdus; do
+    echo "$0: $(date) Setting port security on $vdu"  
+    SERVER_ID=$(openstack server list | awk "/$vdu/ { print \$2 }")
+    id=($(neutron port-list|grep -v "+"|grep -v name|awk '{print $2}'))
+    for id in ${id[@]}; do
+      if [[ $(neutron port-show $id|grep $SERVER_ID) ]]; then neutron port-update ${id} --port-security-enabled=True; fi
+    done
   done
 
   echo "$0: $(date) directly assign security group (unsupported in Mitaka Tacker)"
-  if [[ $(openstack security group list | awk "/ vHello / { print \$2 }") ]]; then openstack security group delete vHello; fi
-  openstack security group create vHello
-  openstack security group rule create --ingress --protocol TCP --dst-port 22:22 vHello
-  openstack security group rule create --ingress --protocol TCP --dst-port 80:80 vHello
+  if [[ $(neutron security-group-list | awk "/ vHello / { print \$2 }") ]]; then neutron security-group-delete vHello; fi
+  neutron security-group-create vHello
+  neutron security-group-rule-create --direction ingress --protocol TCP --port-range-min 22 --port-range-max 22 vHello
+  neutron security-group-rule-create --direction ingress --protocol TCP --port-range-min 80 --port-range-max 80 vHello
+  neutron security-group-rule-create --direction ingress --protocol TCP --port-range-min 30000 --port-range-max 30000 vHello
   for i in $vdui; do
+    vdu_id[$i]=$(openstack server list | awk "/VDU$i/ { print \$2 }")
+    echo "$0: $(date) Assigning security groups to VDU$i (${vdu_id[$i]})"    
     openstack server add security group ${vdu_id[$i]} vHello
     openstack server add security group ${vdu_id[$i]} default
   done
 
   echo "$0: $(date) associate floating IPs"
-  get_floating_net
   for i in $vdui; do
-    vdu_ip[$i]=$(openstack floating ip create $FLOATING_NETWORK_NAME | awk "/floating_ip_address/ { print \$4 }")
     nova floating-ip-associate ${vdu_id[$i]} ${vdu_ip[$i]}
   done
 
   echo "$0: $(date) get web server addresses"
   vdu_url[1]="http://${vdu_ip[1]}"
-  vdu_url[2]="http://${vdu_ip[3]}"
+  vdu_url[2]="http://${vdu_ip[2]}"
   vdu_url[3]="http://${vdu_ip[3]}"
   vdu_url[4]="http://${vdu_ip[4]}:30000/eventListener/v1"
 
-  if [[ -f /tmp/tacker/id_rsa ]]; then
-    echo "$0: $(date) setup private key for ssh to hypervisors"
-    cp -p /tmp/tacker/id_rsa ~/.ssh/id_rsa
-    chown root ~/.ssh/id_rsa
-    chmod 600 ~/.ssh/id_rsa
-  fi
-
-  echo "$0: $(date) start collectd agent on bare metal hypervisor hosts"
-  hosts=($(openstack hypervisor list | grep -v Hostname | grep -v "+" | awk '{print $4}'))
-  for host in ${hosts[@]}; do
-    ip=$(openstack hypervisor show $host | grep host_ip | awk '{print $4}')
-    if [[ "$OS_CLOUDNAME" == "overcloud" ]]; then 
-      u="heat-admin"
-      p=""
-    else 
-      u="ubuntu"
-      p=":ubuntu"
-    fi
-    scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/start.sh $u@$ip:/home/$u/start.sh
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $u$p@$ip \
-      "nohup bash /home/$u/start.sh collectd $ip ${vdu_ip[4]} hello world > /dev/null 2>&1 &"
+  apt-get install -y curl
+
+  echo "$0: $(date) wait for VNF web service to be ready"
+  count=0
+  resp=$(curl http://${vdu_ip[1]})
+  echo $resp
+  while [[ $count < 10 && "$resp" == "" ]]; do
+    echo "$0: $(date) waiting for HTTP response from LB"
+    sleep 60
+    let count=$count+1
+    resp=$(curl http://${vdu_ip[3]})
+    echo $resp
   done
-
-  echo "$0: $(date) wait 30 seconds for server SSH to be available"
-  sleep 30
-
-  echo "$0: $(date) Copy startup script to the VMs"
-  for i in $vdui; do
-    ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[$i]} "sudo chown ubuntu /home/ubuntu"
-    scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/start.sh ubuntu@${vdu_ip[$i]}:/home/ubuntu/start.sh
+     
+  echo "$0: $(date) verify vHello server is running at each web server and via the LB"
+  say_hello http://${vdu_ip[1]}
+  say_hello http://${vdu_ip[2]}
+  say_hello http://${vdu_ip[3]}
+
+  assert "models-vhello-001 (vHello VNF creation)" true
+  assert "models-tacker-003 (VNF creation)" true
+  assert "models-tacker-vnfd-002 (artifacts creation)" true
+  assert "models-tacker-vnfd-003 (user_data creation)" true
+
+  echo "$0: $(date) Execute agent startup script in the VNF VMs"
+  for i in $vnf_vdui; do
+    ssh -i /opt/tacker/vHello -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[$i]} "sudo chown ubuntu /home/ubuntu"
+    scp -i /opt/tacker/vHello -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /opt/tacker/blueprints/tosca-vnfd-hello-ves/start.sh ubuntu@${vdu_ip[$i]}:/home/ubuntu/start.sh
+#    ssh -i /opt/tacker/vHello -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+#    ubuntu@${vdu_ip[$i]} "nohup bash /home/ubuntu/start.sh agent ${vdu_id[$i]} ${vdu_ip[4]} hello world > /dev/null 2>&1 &"
   done
 
-  echo "$0: $(date) start vHello webserver in VDU1 at ${vdu_ip[1]}"
-  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-    ubuntu@${vdu_ip[1]} "nohup bash /home/ubuntu/start.sh webserver ${vdu_id[1]} ${vdu_ip[4]} hello world > /dev/null 2>&1 &"
-
-  echo "$0: $(date) start vHello webserver in VDU2 at ${vdu_ip[2]}"
-  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-    ubuntu@${vdu_ip[2]} "nohup bash /home/ubuntu/start.sh webserver ${vdu_id[2]} ${vdu_ip[4]} hello world > /dev/null 2>&1 &"
-
-  echo "$0: $(date) start LB in VDU3 at ${vdu_ip[3]}"
-  ssh -i /tmp/tacker/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-    ubuntu@${vdu_ip[3]} "nohup bash /home/ubuntu/start.sh lb ${vdu_id[3]} ${vdu_ip[4]} hello world ${vdu_ip[1]} ${vdu_ip[2]} > /dev/null 2>&1 &"
-
-  echo "$0: $(date) start Monitor in VDU4 at ${vdu_ip[4]}"
-  # Replacing the default collector with monitor.py which has processing logic as well
-  scp -i /tmp/tacker/vHello.pem -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/tacker/blueprints/tosca-vnfd-hello-ves/monitor.py ubuntu@${vdu_ip[4]}:/home/ubuntu/monitor.py
-  ssh -i /tmp/tacker/vHello.pem -t -t -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} "bash /home/ubuntu/start.sh monitor ${vdu_id[1]} ${vdu_id[2]} ${vdu_id[3]} hello world"
-
-#  echo "$0: $(date) verify vHello server is running at http://${vdu_ip[3]}"
-#  apt-get install -y curl
-#  count=10
-#  while [[ $count -gt 0 ]] 
-#  do 
-#    sleep 60
-#    let count=$count-1
-#    if [[ $(curl http://${vdu_ip[3]} | grep -c "Hello World") > 0 ]]; then pass; fi
-#  done
-#  fail
+  echo "$0: $(date) setup Monitor in VDU4 at ${vdu_ip[4]}"
+  scp -i /opt/tacker/vHello -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /opt/tacker/blueprints/tosca-vnfd-hello-ves/start.sh ubuntu@${vdu_ip[4]}:/home/ubuntu/start.sh
+  scp -i /opt/tacker/vHello -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /opt/tacker/blueprints/tosca-vnfd-hello-ves/monitor.py ubuntu@${vdu_ip[4]}:/home/ubuntu/monitor.py
+  ssh -i /opt/tacker/vHello -t -t -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@${vdu_ip[4]} "bash /home/ubuntu/start.sh monitor ${vdu_id[1]} ${vdu_id[2]} ${vdu_id[3]} hello world"
 }
 
 stop() {
+  trap 'fail' ERR
+
   echo "$0: $(date) setup OpenStack CLI environment"
-  source /tmp/tacker/admin-openrc.sh
+  source /opt/tacker/admin-openrc.sh
+
+  if [[ "$(tacker vnf-list|grep hello-ves|awk '{print $2}')" != '' ]]; then
+    echo "$0: $(date) uninstall vHello blueprint via CLI"
+    try 12 10 "tacker vnf-delete hello-ves"
+    # It can take some time to delete a VNF - thus wait 2 minutes
+    count=12
+    while [[ $count > 0 && "$(tacker vnf-list|grep hello-ves|awk '{print $2}')" != '' ]]; do 
+      echo "$0: $(date) waiting for hello-ves VNF delete to complete"
+      sleep 10
+      let count=$count-1
+    done 
+    if [[ "$(tacker vnf-list|grep hello-ves|awk '{print $2}')" == '' ]]; then
+      assert "models-tacker-004 (VNF deletion)" true
+    else
+      assert "models-tacker-004 (VNF deletion)" false
+    fi
+  fi
+
+  # It can take some time to delete a VNFD - thus wait 2 minutes
+  if [[ "$(tacker vnfd-list|grep hello-ves|awk '{print $2}')" != '' ]]; then
+    echo "$0: $(date) trying to delete the hello-ves VNFD"
+    try 12 10 "tacker vnfd-delete hello-ves"
+    if [[ "$(tacker vnfd-list|grep hello-ves|awk '{print $2}')" == '' ]]; then
+      assert "models-tacker-005 (VNFD deletion)" true
+    else
+      assert "models-tacker-005 (VNFD deletion)" false
+    fi
+  fi
+
+# This part will apply for tests that dynamically create the VDU base image
+#  iid=($(openstack image list|grep VNFImage|awk '{print $2}')); for id in ${iid[@]}; do openstack image delete ${id};  done
+#  if [[ "$(openstack image list|grep VNFImage|awk '{print $2}')" == '' ]]; then
+#    assert "models-tacker-vnfd-004 (artifacts deletion)" true
+#  else
+#    assert "models-tacker-vnfd-004 (artifacts deletion)" false
+#  fi
 
-  echo "$0: $(date) uninstall vHello blueprint via CLI"
-  vid=($(tacker vnf-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnf-delete ${id};  done
-  vid=($(tacker vnfd-list|grep hello-ves|awk '{print $2}')); for id in ${vid[@]}; do tacker vnfd-delete ${id};  done
-# Need to remove the floatingip deletion or make it specific to the vHello VM
+  # Cleanup for workarounds
   fip=($(neutron floatingip-list|grep -v "+"|grep -v id|awk '{print $2}')); for id in ${fip[@]}; do neutron floatingip-delete ${id};  done
   sg=($(openstack security group list|grep vHello|awk '{print $2}'))
-  for id in ${sg[@]}; do try 10 5 "openstack security group delete ${id}";  done
-	
+  for id in ${sg[@]}; do try 5 5 "openstack security group delete ${id}";  done
+  kid=($(openstack keypair list|grep vHello|awk '{print $2}')); for id in ${kid[@]}; do openstack keypair delete ${id};  done
+}
+
+start_collectd() {
+  # NOTE: ensure hypervisor hostname is resolvable e.g. thru /etc/hosts
+  echo "$0: $(date) update start.sh script in case it changed"
+  cp -r blueprints/tosca-vnfd-hello-ves/start.sh /opt/tacker/blueprints/tosca-vnfd-hello-ves
+  echo "$0: $(date) start collectd agent on bare metal hypervisor host"
+  get_vdu_ip VDU4
+  scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /opt/tacker/blueprints/tosca-vnfd-hello-ves/start.sh $2@$1:/home/$2/start.sh
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $2@$1 \
+    "nohup bash /home/$2/start.sh collectd $1 $ip hello world > /dev/null 2>&1 &"
+}
+
+stop_collectd() {
   echo "$0: $(date) remove collectd agent on bare metal hypervisor hosts"
-  hosts=($(openstack hypervisor list | grep -v Hostname | grep -v "+" | awk '{print $4}'))
-  for host in ${hosts[@]}; do
-    ip=$(openstack hypervisor show $host | grep host_ip | awk '{print $4}')
-    if [[ "$OS_CLOUDNAME" == "overcloud" ]]; then 
-      u="heat-admin"
-      p=""
-    else 
-      u="ubuntu"
-      p=":ubuntu"
-    fi
-    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $u$p@$ip <<'EOF'
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $2@$1 <<'EOF'
 dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`
 if [ "$dist" == "Ubuntu" ]; then 
   sudo service collectd stop
   sudo apt-get remove -y collectd
-  sudo rm /etc/collectd/collectd.conf
 else
   sudo service collectd stop
-  sudo yum remove -y collectd
-  sudo rm /etc/collectd.conf
+  sudo yum remove -y collectd collectd-virt
 fi
-rm -rf $HOME/OpenStackBarcelonaDemo
+rm -rf $HOME/barometer
 EOF
-  done
 }
 
 #
@@ -330,7 +398,7 @@ EOF
 #
 
 get_vdu_ip () {
-  source /tmp/tacker/admin-openrc.sh
+  source /opt/tacker/admin-openrc.sh
 
   echo "$0: $(date) find VM IP for $1"
   ip=$(openstack server list | awk "/$1/ { print \$10 }")
@@ -339,10 +407,10 @@ get_vdu_ip () {
 monitor () {
   echo "$0: $(date) Start the VES Monitor in VDU4 - Stop first if running"
   get_vdu_ip VDU4
-  sudo cp /tmp/tacker/vHello.pem /tmp/vHello.pem
-  sudo chown $USER:$USER /tmp/vHello.pem
-  chmod 600 /tmp/vHello.pem
-  ssh -t -t -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip << 'EOF'
+  sudo cp /opt/tacker/vHello /tmp/vHello
+  sudo chown $USER:$USER /tmp/vHello
+  chmod 600 /tmp/vHello
+  ssh -t -t -i /tmp/vHello -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip << 'EOF'
 sudo kill $(ps -ef | grep evel-test-collector | awk '{print $2}')
 python monitor.py --config evel-test-collector/config/collector.conf --section default 
 EOF
@@ -364,67 +432,130 @@ traffic () {
 pause () {
   echo "$0: $(date) Pause the VNF (web server) in $1 for 30 seconds to generate a state change fault report (Stopped)"
   get_vdu_ip $1
-  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker pause vHello"
+  ssh -i /tmp/vHello -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker pause vHello"
   sleep 20
   echo "$0: $(date) Unpausing the VNF to generate a state change fault report (Started)"
-  ssh -i /tmp/vHello.pem -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker unpause vHello"
+  ssh -i /tmp/vHello -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$ip "sudo docker unpause vHello"
 }
 
 forward_to_container () {
-  echo "$0: $(date) pass $1 command to this script in the tacker container"
-  CONTAINER=$(sudo docker ps -a | awk "/tacker/ { print \$1 }")
-  sudo docker exec $CONTAINER /bin/bash /tmp/tacker/vHello_VES.sh $1 $1
+  echo "$0: $(date) pass $1 command to vHello.sh in tacker container"
+  sudo docker exec tacker /bin/bash /opt/tacker/vHello_VES.sh $1
   if [ $? -eq 1 ]; then fail; fi
 }
 
 dist=`grep DISTRIB_ID /etc/*-release | awk -F '=' '{print $2}'`
 case "$1" in
   setup)
-    setup
+    setup $2 $3
+    if [ $? -eq 1 ]; then fail; fi
     pass
     ;;
   run)
-    setup
+    setup $2 $3
+    copy_blueprint
     forward_to_container start
+    if [ $? -eq 1 ]; then fail; fi
     pass
     ;;
-  start|stop)
-    if [[ $# -eq 1 ]]; then forward_to_container $1
+  start)
+    if [[ -f /.dockerenv ]]; then
+      start
     else
-      # running inside the tacker container, ready to go
-      $1
+      copy_blueprint
+      forward_to_container start
     fi
     pass
     ;;
+  start_collectd)
+    start_collectd $2 $3 $4
+    if [ $? -eq 1 ]; then fail; fi
+    pass
+    ;;
+  stop_collectd)
+    stop_collectd $2 $3
+    if [ $? -eq 1 ]; then fail; fi
+    pass
+    ;;
+  monitor)
+    monitor
+    pass
+    ;;
   traffic)
-    $1
+    traffic
     pass
     ;;
-  test)
-    $2 $3
+  pause)
+    pause $2
+    ;;
+  stop)
+    if [[ -f /.dockerenv ]]; then
+      stop
+    else
+      forward_to_container stop
+    fi
+    if [ $? -eq 1 ]; then fail; fi
+    pass
     ;;
   clean)
+    stop_collectd $1
     echo "$0: $(date) Uninstall Tacker and test environment"
-    bash /tmp/tacker/tacker-setup.sh $1 clean
+    sudo docker exec -it tacker /bin/bash /opt/tacker/tacker-setup.sh clean
+    sudo docker stop tacker
+    sudo docker rm -v tacker
+    sudo rm -rf /opt/tacker
     pass
     ;;
   *)
-    echo "usage: bash vHello_VES.sh [setup|start|run|clean]"
-    echo "setup: setup test environment"
-    echo "start: install blueprint and run test"
-    echo "run: setup test environment and run test"
-    echo "stop: stop test and uninstall blueprint"
-    echo "clean: cleanup after test"
-    echo "usage: bash vHello_VES.sh [setup|start|run|test|stop|clean] [monitor|traffic|pause|nic]"
-    echo "setup: setup test environment"
-    echo "start: install blueprint and run test"
-    echo "run: setup test environment and run test"
-    echo "test: run test tools/scenario - see below"
-    echo "stop: stop test and uninstall blueprint"
-    echo "clean: cleanup after test"
-    echo "Test:"
-    echo "  monitor: attach to the collector VM and run the VES Monitor"
-    echo "  traffic: generate some traffic"
-    echo "  pause: pause the VNF (web server) for a minute to generate a state change"
-    fail
+    cat <<EOF
+ What this is: Deployment test for the VES agent and collector based 
+ upon the Tacker Hello World blueprint, designed as a manual demo of the VES
+ concept and integration with the Barometer project collectd agent. Typical 
+ demo procedure is to execute the following actions from the OPNFV jumphost
+ or some host wth access to the OpenStack controller (see below for details):
+  setup: install Tacker in a docker container. Note: only needs to be done
+         once per session, and can be reused across OPNFV VES and Models tests,
+         i.e. you can start another test at the "start" step below.
+  start: install blueprint and start the VNF, including the app (load-balanced
+         web server) and VES agents running on the VMs. Installs the VES 
+         monitor code but does not start the monitor (see below).
+  start_collectd: start the collectd daemon on bare metal hypervisor hosts
+  monitor: start the VES monitor, typically run in a second shell session.
+  pause: pause the app at one of the web server VDUs (VDU1 or VDU2)
+  stop: stop the VNF and uninstall the blueprint
+  start_collectd: start the collectd daemon on bare metal hypervisor hosts
+  clean: remove the tacker container and service (if desired, when done)
+
+ How to use:
+   $ git clone https://gerrit.opnfv.org/gerrit/ves
+   $ cd ves/tests
+   $ bash vHello_VES.sh <setup> <openrc> [branch]
+     setup: setup test environment
+     <openrc>: location of OpenStack openrc file
+     branch: OpenStack branch to install (default: master)
+   $ bash vHello_VES.sh start
+     start: install blueprint and run test
+     <user>: username on hypervisor hosts, for ssh (user must be setup for 
+       key-based auth on the hosts)
+   $ bash vHello_VES.sh start_collectd|stop_collectd <ip> <user> <monitor_ip>
+     start_collectd: install and start collectd daemon on hypervisor
+     stop_collectd: stop and uninstall collectd daemon on hypervisor
+     <ip>: hypervisor ip 
+     <user>: username on hypervisor hosts, for ssh (user must be setup for 
+       key-based auth on the hosts)
+   $ bash vHello_VES.sh monitor
+     monitor: attach to the collector VM and run the VES Monitor
+   $ bash vHello_VES.sh traffic
+     traffic: generate some traffic
+   $ bash vHello_VES.sh pause VDU1|VDU2
+     pause: pause the VNF (web server) for a minute to generate a state change
+     VDU1: Pause VDU1
+     VDU2: Pause VDU2
+   $ bash vHello_VES.sh stop
+     stop: stop test and uninstall blueprint
+   $ bash vHello_VES.sh clean <user>
+     clean: cleanup after test
+     <user>: username on hypervisor hosts, for ssh (user must be setup for 
+       key-based auth on the hosts)
+EOF
 esac