Initial implementation of ONAP scenario

Patch introduces an installation script, which is used by
OPNFV os-nosdn-onap-noha scenario for automatic ONAP deployment.

JIRA: AUTO-71
Change-Id: I427d7594e6799250809d04c3e47ac3c072fdb64f
Signed-off-by: Martin Klozik <martin.klozik@tieto.com>

2 files changed, 300 insertions, 70 deletions

diff --git a/ci/deploy-onap-fuel.sh b/ci/deploy-onap-fuel.sh
new file mode 100755
index 0000000..cc4dfb1
--- /dev/null
+++ b/ci/deploy-onap-fuel.sh
@@ -0,0 +1,193 @@
+#!/bin/bash
+#
+# Copyright 2018 Tieto
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script for automated deployment of ONAP on top of OPNFV Fuel/MCP installation
+# In the future both OOM and heat install methods should be supported.
+# At the beginning OOM will be used for simplification.
+
+# TODO:
+#   Configure ONAP to be able to control underlying OpenStack
+
+# Configuration to be passed to ci/deploy-onap.sh
+export SSH_USER="ubuntu"
+export SSH_IDENTITY="/root/.ssh/onap_key"
+
+# Use default values if compute configuration was not set by FUEL installer
+AUTO_INSTALL_DIR=${AUTO_INSTALL_DIR:-"."}
+AUTO_IMAGE_DIR="${AUTO_INSTALL_DIR}/images"
+CMP_COUNT=${CMP_COUNT:-2}           # number of compute nodes
+CMP_MIN_MEM=${CMP_MIN_MEM:-64000}   # MB RAM of the weakest compute node
+CMP_MIN_CPUS=${CMP_MIN_CPUS:-36}    # CPU count of the weakest compute node
+# size of storage for instances
+CMP_STORAGE_TOTAL=${CMP_STORAGE_TOTAL:-$((80*$CMP_COUNT))}
+VM_COUNT=${VM_COUNT:-2}             # number of VMs available for k8s cluster
+
+#
+# Functions
+#
+# function minimum accepts two numbers and prints smaller one
+function minimum(){
+    echo $(($1<$2?$1:$2))
+}
+
+# function remove_openstack_setup removes OS configuration performed by this
+#   script; So previously created configuration and deployed VMs will be
+#   removed before new ONAP deployment will be started.
+function remove_openstack_setup(){
+    # flavor is created 1st but removed last, so...
+    if ( ! openstack flavor list | grep 'onap.large' &> /dev/null ) ; then
+        #...no flavor means nothing to be removed
+        return
+    fi
+    echo -e "\nRemoving ONAP specific OpenStack configuration"
+    for a in $(openstack server list --name onap_vm -f value -c ID) ; do
+        openstack server delete $a
+    done
+    RULES=$(openstack security group rule list onap_security_group -f value -c ID)
+    for a in $RULES; do
+        openstack security group rule delete $a
+    done
+    openstack security group delete onap_security_group
+    for a in $(openstack floating ip list -f value -c ID) ; do
+        openstack floating ip delete $a
+    done
+    PORTS=$(openstack port list --network onap_private_network -f value -c ID)
+    for a in $PORTS ; do
+        openstack router remove port onap_router $a
+    done
+    PORTS=$(openstack port list --network onap_private_network -f value -c ID)
+    for a in $PORTS ; do
+        openstack port delete $a
+    done
+    openstack router delete onap_router
+    openstack subnet delete onap_private_subnet
+    openstack network delete onap_private_network
+    openstack image delete xenial
+    rm -rf $AUTO_IMAGE_DIR
+    openstack keypair delete onap_key
+    rm $SSH_IDENTITY
+    openstack flavor delete onap.large
+    echo
+}
+
+#
+# Script Main
+#
+
+# remove OpenStack configuration if it exists
+remove_openstack_setup
+
+echo -e "\nOpenStack configuration\n"
+
+# Calculate VM resources, so that flavor can be created
+echo "Configuration of compute node:"
+echo "Number of computes:    CMP_COUNT=$CMP_COUNT"
+echo "Minimal RAM:           CMP_MIN_MEM=$CMP_MIN_MEM"
+echo "Minimal CPUs count:    CMP_MIN_CPUS=$CMP_MIN_CPUS"
+echo "Storage for instances: CMP_STORAGE_TOTAL=$CMP_STORAGE_TOTAL"
+echo "Number of VMs:         VM_COUNT=$VM_COUNT"
+# Calculate VM parameters; there will be up to 1 VM per Compute node
+# to maximize resources available for VMs
+PER=90                      # % of compute resources will be consumed by VMs
+VM_DISK_MAX=100             # GB - max VM disk size
+VM_MEM_MAX=81920            # MB - max VM RAM size
+VM_CPUS_MAX=56              # max count of VM CPUs
+VM_MEM=$(minimum $(($CMP_MIN_MEM*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_MEM_MAX)
+VM_CPUS=$(minimum $(($CMP_MIN_CPUS*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_CPUS_MAX)
+VM_DISK=$(minimum $(($CMP_STORAGE_TOTAL*$PER/100/$VM_COUNT)) $VM_DISK_MAX)
+
+echo -e "\nFlavor configuration:"
+echo "CPUs      : $VM_CPUS"
+echo "RAM [MB]  : $VM_MEM"
+echo "DISK [GB] : $VM_DISK"
+
+# Create onap flavor
+openstack flavor create --ram $VM_MEM  --vcpus $VM_CPUS --disk $VM_DISK \
+    onap.large
+
+# Generate a keypair and store private key
+openstack keypair create onap_key > $SSH_IDENTITY
+chmod 600 $SSH_IDENTITY
+
+# Download and import VM image(s)
+mkdir $AUTO_IMAGE_DIR
+wget -P $AUTO_IMAGE_DIR https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
+openstack image create --disk-format qcow2 --container-format bare --public \
+    --file $AUTO_IMAGE_DIR/xenial-server-cloudimg-amd64-disk1.img xenial
+
+# Modify quotas (add 10% to required VM resources)
+openstack quota set --ram $(($VM_MEM*$VM_COUNT*110/100)) admin
+openstack quota set --cores $(($VM_CPUS*$VM_COUNT*110/100)) admin
+
+# Configure networking with DNS for access to the internet
+openstack network create onap_private_network --provider-network-type vxlan
+openstack subnet create onap_private_subnet --network onap_private_network \
+    --subnet-range 192.168.33.0/24 --ip-version 4 --dhcp --dns-nameserver "8.8.8.8"
+openstack router create onap_router
+openstack router add subnet onap_router onap_private_subnet
+openstack router set onap_router --external-gateway floating_net
+
+# Allow selected ports and protocols
+openstack security group create onap_security_group
+openstack security group rule create --protocol icmp onap_security_group
+openstack security group rule create  --proto tcp \
+    --dst-port 22:22 onap_security_group
+openstack security group rule create  --proto tcp \
+    --dst-port 8080:8080 onap_security_group            # rancher
+openstack security group rule create  --proto tcp \
+    --dst-port 8078:8078 onap_security_group            # horizon
+openstack security group rule create  --proto tcp \
+    --dst-port 8879:8879 onap_security_group            # helm
+openstack security group rule create  --proto tcp \
+    --dst-port 80:80 onap_security_group
+openstack security group rule create  --proto tcp \
+    --dst-port 443:443 onap_security_group
+
+# Allow communication between k8s cluster nodes
+PUBLIC_NET=`openstack subnet list --name floating_subnet -f value -c Subnet`
+openstack security group rule create --remote-ip $PUBLIC_NET --proto tcp \
+    --dst-port 1:65535 onap_security_group
+openstack security group rule create --remote-ip $PUBLIC_NET --proto udp \
+    --dst-port 1:65535 onap_security_group
+
+# Create VMs and assign floating IPs to them
+VM_ITER=1
+while [ $VM_ITER -le $VM_COUNT ] ; do
+    openstack floating ip create floating_net
+    VM_NAME[$VM_ITER]="onap_vm${VM_ITER}"
+    VM_IP[$VM_ITER]=$(openstack floating ip list -c "Floating IP Address" \
+        -c "Port" -f value | grep None | cut -f1 -d " " | head -n1)
+    openstack server create --flavor onap.large --image xenial \
+        --nic net-id=onap_private_network --security-group onap_security_group \
+        --key-name onap_key ${VM_NAME[$VM_ITER]}
+    sleep 5 # wait for VM init before floating IP can be assigned
+    openstack server add floating ip ${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]}
+    VM_ITER=$(($VM_ITER+1))
+done
+
+openstack server list
+
+echo "Waiting for VMs to start up for 5 minutes at $(date)"
+sleep 5m
+
+openstack server list
+
+# Start ONAP installation
+DATE_START=$(date)
+echo -e "\nONAP Installation Started at $DATE_START\n"
+$AUTO_INSTALL_DIR/ci/deploy-onap.sh ${VM_IP[@]}
+echo -e "\nONAP Installation Started at $DATE_START"
+echo -e "ONAP Installation Finished at $(date)\n"
diff --git a/ci/deploy-onap.sh b/ci/deploy-onap.sh
index e886492..4d7e3b3 100755
--- a/ci/deploy-onap.sh
+++ b/ci/deploy-onap.sh
@@ -26,8 +26,8 @@
 # NOTE: Following must be assured for all MASTER and SLAVE servers before
 #       onap-deploy.sh execution:
 #       1) ssh access without a password
-#       2) an "opnfv" user account with password-less sudo access must be
-#          available
+#       2) an user account with password-less sudo access must be
+#          available - default user is "opnfv"
 
 #
 # Configuration
@@ -40,10 +40,24 @@ HELM_VERSION=2.8.2
 
 MASTER=$1
 SERVERS=$*
+shift
+SLAVES=$*
 
-BRANCH='master'
+BRANCH='beijing'
 ENVIRON='onap'
 
+SSH_USER=${SSH_USER:-"opnfv"}
+SSH_OPTIONS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+# by defalult install full ONAP installation
+ONAP_COMPONENT_DISABLE=${ONAP_COMPONENT_DISABLE:-""}
+# example of minimal ONAP installation
+#ONAP_COMPONENT_DISABLE="clamp cli consul dcaegen2 esr log msb multicloud nbi oof policy uui vfc vnfsdk"
+
+# use identity file from the environment SSH_IDENTITY
+if [ -n "$SSH_IDENTITY" ] ; then
+    SSH_OPTIONS="-i $SSH_IDENTITY $SSH_OPTIONS"
+fi
+
 #
 # Installation
 #
@@ -52,9 +66,10 @@ echo "$SERVERS"
 
 for MACHINE in $SERVERS;
 do
-ssh opnfv@"$MACHINE" "bash -s" <<DOCKERINSTALL &
-    sudo su
-    apt-get update
+ssh $SSH_OPTIONS $SSH_USER@"$MACHINE" "bash -s" <<DOCKERINSTALL &
+    sudo -i
+    sysctl -w vm.max_map_count=262144
+    apt-get update -y
     curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh
 
     mkdir -p /etc/systemd/system/docker.service.d/
@@ -83,17 +98,15 @@ wait
 echo "INSTALLING RANCHER ON MASTER"
 echo "$MASTER"
 
-ssh opnfv@"$MASTER" "bash -s" <<RANCHERINSTALL &
-sudo su
-apt install jq -y
+ssh $SSH_OPTIONS $SSH_USER@"$MASTER" "bash -s" <<RANCHERINSTALL
+sudo -i
+echo "INSTALL STARTS"
+apt-get install -y jq make htop
 echo "Waiting for 30 seconds at \$(date)"
 sleep 30
 
 docker login -u docker -p docker nexus3.onap.org:10001
 
-echo "INSTALL STARTS"
-apt-get install make -y
-
 docker run -d --restart=unless-stopped -p 8080:8080\
  --name rancher_server rancher/server:v$RANCHER_VERSION
 curl -LO https://storage.googleapis.com/kubernetes-release/\
@@ -116,7 +129,7 @@ echo "/dockerdata-nfs *(rw,no_root_squash,no_subtree_check)">>/etc/exports
 service nfs-kernel-server restart
 
 echo "Waiting 10 minutes for Rancher to setup at \$(date)"
-sleep 600
+sleep 10m
 echo "Installing RANCHER CLI, KUBERNETES ENV on RANCHER"
 wget https://github.com/rancher/cli/releases/download/v${RANCHER_CLI_VER}-rc2\
 /rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
@@ -142,6 +155,10 @@ echo "Creating kubernetes environment named ${ENVIRON}"
 ./rancher env create -t kubernetes $ENVIRON > kube_env_id.json
 PROJECT_ID=\$(<kube_env_id.json)
 echo "env id: \$PROJECT_ID"
+
+echo "Waiting for ${ENVIRON} creation - 1 min at \$(date)"
+sleep 1m
+
 export RANCHER_HOST_URL=http://${MASTER}:8080/v1/projects/\$PROJECT_ID
 echo "you should see an additional kubernetes environment"
 ./rancher env ls
@@ -153,7 +170,7 @@ REG_URL_RESPONSE=\`curl -X POST -u \$KEY_PUBLIC:\$KEY_SECRET\
  "http://$MASTER:8080/v1/projects/\$PROJECT_ID/registrationtokens"\`
 echo "REG_URL_RESPONSE: \$REG_URL_RESPONSE"
 echo "Waiting for the server to finish url configuration - 1 min at \$(date)"
-sleep 60
+sleep 1m
 # see registrationUrl in
 REGISTRATION_TOKENS=\`curl http://$MASTER:8080/v2-beta/registrationtokens\`
 echo "REGISTRATION_TOKENS: \$REGISTRATION_TOKENS"
@@ -197,13 +214,12 @@ echo "docker run --rm --privileged\
  \$REGISTRATION_DOCKER\
  \$RANCHER_URL/v1/scripts/\$REGISTRATION_TOKEN"\
  > /tmp/rancher_register_host
-chown opnfv /tmp/rancher_register_host
+chown $SSH_USER /tmp/rancher_register_host
 
 RANCHERINSTALL
-wait
 
 echo "REGISTER TOKEN"
-HOSTREGTOKEN=$(ssh opnfv@"$MASTER" cat /tmp/rancher_register_host)
+HOSTREGTOKEN=$(ssh $SSH_OPTIONS $SSH_USER@"$MASTER" cat /tmp/rancher_register_host)
 echo "$HOSTREGTOKEN"
 
 echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'"
@@ -211,22 +227,38 @@ echo "$SERVERS"
 
 for MACHINE in $SERVERS;
 do
-ssh opnfv@"$MACHINE" "bash -s" <<REGISTERHOST &
-    sudo su
+ssh $SSH_OPTIONS $SSH_USER@"$MACHINE" "bash -s" <<REGISTERHOST &
+    sudo -i
     $HOSTREGTOKEN
     sleep 5
     echo "Host $MACHINE waiting for host registration 5 min at \$(date)"
-    sleep 300
+    sleep 5m
 REGISTERHOST
 done
 wait
 
+echo "CONFIGURING NFS ON SLAVES"
+echo "$SLAVES"
+
+for SLAVE in $SLAVES;
+do
+ssh $SSH_OPTIONS $SSH_USER@"$SLAVE" "bash -s" <<CONFIGURENFS &
+    sudo -i
+    apt-get install nfs-common -y
+    mkdir /dockerdata-nfs
+    chmod 777 /dockerdata-nfs
+    echo "$MASTER:/dockerdata-nfs /dockerdata-nfs   nfs    auto  0  0" >> /etc/fstab
+    mount -a
+    mount | grep dockerdata-nfs
+CONFIGURENFS
+done
+wait
+
 echo "DEPLOYING OOM ON RANCHER WITH MASTER"
 echo "$MASTER"
 
-ssh opnfv@"$MASTER" "bash -s" <<OOMDEPLOY &
-sudo su
-sysctl -w vm.max_map_count=262144
+ssh $SSH_OPTIONS $SSH_USER@"$MASTER" "bash -s" <<OOMDEPLOY
+sudo -i
 rm -rf oom
 echo "pulling new oom"
 git clone -b $BRANCH http://gerrit.onap.org/r/oom
@@ -242,10 +274,22 @@ chmod 777 prepull_docker.sh
 ./prepull_docker.sh
 echo "starting onap pods"
 cd oom/kubernetes/
+
+# Disable ONAP components
+if [ -n "$ONAP_COMPONENT_DISABLE" ] ; then
+    echo -n "Disable following ONAP components:"
+    for COMPONENT in $ONAP_COMPONENT_DISABLE; do
+        echo -n " \$COMPONENT"
+        sed -i '/^'\${COMPONENT}':$/!b;n;s/enabled: *true/enabled: false/' onap/values.yaml
+    done
+    echo
+fi
+
 helm init --upgrade
-helm serve &
+# run helm server on the background and detached from current shell
+nohup helm serve  0<&- &>/dev/null &
 echo "Waiting for helm setup for 5 min at \$(date)"
-sleep 300
+sleep 5m
 helm version
 helm repo add local http://127.0.0.1:8879
 helm repo list
@@ -254,40 +298,43 @@ helm install local/onap -n dev --namespace $ENVIRON
 cd ../../
 
 echo "Waiting for all pods to be up for 15-80 min at \$(date)"
-FAILED_PODS_LIMIT=0
-MAX_WAIT_PERIODS=480 # 120 MIN
+echo "Ignore failure of sdnc-ansible-server, see SDNC-443"
+TMP_POD_LIST='/tmp/onap_pod_list.txt'
+function get_onap_pods() {
+    kubectl get pods --namespace $ENVIRON > \$TMP_POD_LIST
+    return \$(cat \$TMP_POD_LIST | wc -l)
+}
+FAILED_PODS_LIMIT=1 # maximal number of falied ONAP PODs
+ALL_PODS_LIMIT=20   # minimum ONAP PODs to be up & running
+MAX_WAIT_PERIODS=500 # over 2 hours
 COUNTER=0
-PENDING_PODS=0
-while [  \$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) \
--gt \$FAILED_PODS_LIMIT ]; do
-  PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
-  PENDING_PODS=\$PENDING
+get_onap_pods
+ALL_PODS=\$?
+PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
+while [ \$PENDING -gt \$FAILED_PODS_LIMIT -o \$ALL_PODS -lt \$ALL_PODS_LIMIT ]; do
+  # print header every 20th lines
+  if [ \$COUNTER -eq \$((\$COUNTER/20*20)) ] ; then
+    printf "%-3s %-29s %-3s/%s\n" "Nr." "Datetime of check" "Err" "Total PODs"
+  fi
+  COUNTER=\$((\$COUNTER+1))
+  printf "%3s %-29s %3s/%-3s\n" \$COUNTER "\$(date)" \$PENDING \$ALL_PODS
   sleep 15
-  LIST_PENDING=\$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' )
-  echo "\${LIST_PENDING}"
-  echo "\${PENDING} pending > \${FAILED_PODS_LIMIT} at the \${COUNTER}th"\
-       " 15 sec interval out of \${MAX_WAIT_PERIODS}"
-  echo ""
-  COUNTER=\$((\$COUNTER + 1 ))
   if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then
     FAILED_PODS_LIMIT=800
+    ALL_PODS_LIMIT=0
   fi
+  get_onap_pods
+  ALL_PODS=\$?
+  PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
 done
 
 echo "Report on non-running containers"
-PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2')
-PENDING_COUNT=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
-PENDING_COUNT_AAI=\$(kubectl get pods -n $ENVIRON | grep aai- \
-| grep -E '0/|1/2' | wc -l)
-
-echo "Check filebeat 2/2 count for ELK stack logging consumption"
-FILEBEAT=\$(kubectl get pods --all-namespaces -a | grep 2/)
-echo "\${FILEBEAT}"
+get_onap_pods
+grep -E '0/|1/2' \$TMP_POD_LIST
+echo
+
 echo "sleep 5 min - to allow rest frameworks to finish at \$(date)"
-sleep 300
-echo "List of ONAP Modules"
-LIST_ALL=\$(kubectl get pods --all-namespaces -a  --show-all )
-echo "\${LIST_ALL}"
+sleep 5m
 echo "run healthcheck 2 times to warm caches and frameworks"\
      "so rest endpoints report properly - see OOM-447"
 
@@ -314,29 +361,19 @@ cloud-regions/ \
 
 # OOM-484 - robot scripts moved
 cd oom/kubernetes/robot
-echo "run healthcheck prep 1"
+echo -e "\nrun healthcheck prep 1"
 # OOM-722 adds namespace parameter
-if [ "$BRANCH" == "amsterdam" ]; then
-  ./ete-k8s.sh health > ~/health1.out
-else
-  ./ete-k8s.sh $ENVIRON health > ~/health1.out
-fi
+./ete-k8s.sh $ENVIRON health > ~/health1.out
 echo "sleep 5 min at \$(date)"
-sleep 300
+sleep 5m
+
 echo "run healthcheck prep 2"
-if [ "$BRANCH" == "amsterdam" ]; then
-  ./ete-k8s.sh health > ~/health2.out
-else
-  ./ete-k8s.sh $ENVIRON health > ~/health2.out
-fi
+./ete-k8s.sh $ENVIRON health > ~/health2.out
+
 echo "run healthcheck for real - wait a further 5 min at \$(date)"
-sleep 300
-if [ "$BRANCH" == "amsterdam" ]; then
-  ./ete-k8s.sh health
-else
-  ./ete-k8s.sh $ENVIRON health
-fi
+sleep 5m
+./ete-k8s.sh $ENVIRON health
 OOMDEPLOY
-wait
-echo "Finished install, ruturned from Master"
+
+echo "Finished install, ruturned from Master at $(date)"
 exit 0