From 613dc317021ac59c40bf7a3cdba694de0a009fa6 Mon Sep 17 00:00:00 2001 From: Martin Klozik Date: Thu, 1 Nov 2018 10:27:01 +0100 Subject: Initial implementation of ONAP scenario Patch introduces an installation script, which is used by OPNFV os-nosdn-onap-noha scenario for automatic ONAP deployment. JIRA: AUTO-71 Change-Id: I427d7594e6799250809d04c3e47ac3c072fdb64f Signed-off-by: Martin Klozik --- ci/deploy-onap-fuel.sh | 193 +++++++++++++++++++++++++++++++++++++++++++++++++ ci/deploy-onap.sh | 177 +++++++++++++++++++++++++++------------------ 2 files changed, 300 insertions(+), 70 deletions(-) create mode 100755 ci/deploy-onap-fuel.sh diff --git a/ci/deploy-onap-fuel.sh b/ci/deploy-onap-fuel.sh new file mode 100755 index 0000000..cc4dfb1 --- /dev/null +++ b/ci/deploy-onap-fuel.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# +# Copyright 2018 Tieto +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for automated deployment of ONAP on top of OPNFV Fuel/MCP installation +# In the future both OOM and heat install methods should be supported. +# At the beginning OOM will be used for simplification. + +# TODO: +# Configure ONAP to be able to control underlying OpenStack + +# Configuration to be passed to ci/deploy-onap.sh +export SSH_USER="ubuntu" +export SSH_IDENTITY="/root/.ssh/onap_key" + +# Use default values if compute configuration was not set by FUEL installer +AUTO_INSTALL_DIR=${AUTO_INSTALL_DIR:-"."} +AUTO_IMAGE_DIR="${AUTO_INSTALL_DIR}/images" +CMP_COUNT=${CMP_COUNT:-2} # number of compute nodes +CMP_MIN_MEM=${CMP_MIN_MEM:-64000} # MB RAM of the weakest compute node +CMP_MIN_CPUS=${CMP_MIN_CPUS:-36} # CPU count of the weakest compute node +# size of storage for instances +CMP_STORAGE_TOTAL=${CMP_STORAGE_TOTAL:-$((80*$CMP_COUNT))} +VM_COUNT=${VM_COUNT:-2} # number of VMs available for k8s cluster + +# +# Functions +# +# function minimum accepts two numbers and prints smaller one +function minimum(){ + echo $(($1<$2?$1:$2)) +} + +# function remove_openstack_setup removes OS configuration performed by this +# script; So previously created configuration and deployed VMs will be +# removed before new ONAP deployment will be started. +function remove_openstack_setup(){ + # flavor is created 1st but removed last, so... + if ( ! openstack flavor list | grep 'onap.large' &> /dev/null ) ; then + #...no flavor means nothing to be removed + return + fi + echo -e "\nRemoving ONAP specific OpenStack configuration" + for a in $(openstack server list --name onap_vm -f value -c ID) ; do + openstack server delete $a + done + RULES=$(openstack security group rule list onap_security_group -f value -c ID) + for a in $RULES; do + openstack security group rule delete $a + done + openstack security group delete onap_security_group + for a in $(openstack floating ip list -f value -c ID) ; do + openstack floating ip delete $a + done + PORTS=$(openstack port list --network onap_private_network -f value -c ID) + for a in $PORTS ; do + openstack router remove port onap_router $a + done + PORTS=$(openstack port list --network onap_private_network -f value -c ID) + for a in $PORTS ; do + openstack port delete $a + done + openstack router delete onap_router + openstack subnet delete onap_private_subnet + openstack network delete onap_private_network + openstack image delete xenial + rm -rf $AUTO_IMAGE_DIR + openstack keypair delete onap_key + rm $SSH_IDENTITY + openstack flavor delete onap.large + echo +} + +# +# Script Main +# + +# remove OpenStack configuration if it exists +remove_openstack_setup + +echo -e "\nOpenStack configuration\n" + +# Calculate VM resources, so that flavor can be created +echo "Configuration of compute node:" +echo "Number of computes: CMP_COUNT=$CMP_COUNT" +echo "Minimal RAM: CMP_MIN_MEM=$CMP_MIN_MEM" +echo "Minimal CPUs count: CMP_MIN_CPUS=$CMP_MIN_CPUS" +echo "Storage for instances: CMP_STORAGE_TOTAL=$CMP_STORAGE_TOTAL" +echo "Number of VMs: VM_COUNT=$VM_COUNT" +# Calculate VM parameters; there will be up to 1 VM per Compute node +# to maximize resources available for VMs +PER=90 # % of compute resources will be consumed by VMs +VM_DISK_MAX=100 # GB - max VM disk size +VM_MEM_MAX=81920 # MB - max VM RAM size +VM_CPUS_MAX=56 # max count of VM CPUs +VM_MEM=$(minimum $(($CMP_MIN_MEM*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_MEM_MAX) +VM_CPUS=$(minimum $(($CMP_MIN_CPUS*$CMP_COUNT*$PER/100/$VM_COUNT)) $VM_CPUS_MAX) +VM_DISK=$(minimum $(($CMP_STORAGE_TOTAL*$PER/100/$VM_COUNT)) $VM_DISK_MAX) + +echo -e "\nFlavor configuration:" +echo "CPUs : $VM_CPUS" +echo "RAM [MB] : $VM_MEM" +echo "DISK [GB] : $VM_DISK" + +# Create onap flavor +openstack flavor create --ram $VM_MEM --vcpus $VM_CPUS --disk $VM_DISK \ + onap.large + +# Generate a keypair and store private key +openstack keypair create onap_key > $SSH_IDENTITY +chmod 600 $SSH_IDENTITY + +# Download and import VM image(s) +mkdir $AUTO_IMAGE_DIR +wget -P $AUTO_IMAGE_DIR https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img +openstack image create --disk-format qcow2 --container-format bare --public \ + --file $AUTO_IMAGE_DIR/xenial-server-cloudimg-amd64-disk1.img xenial + +# Modify quotas (add 10% to required VM resources) +openstack quota set --ram $(($VM_MEM*$VM_COUNT*110/100)) admin +openstack quota set --cores $(($VM_CPUS*$VM_COUNT*110/100)) admin + +# Configure networking with DNS for access to the internet +openstack network create onap_private_network --provider-network-type vxlan +openstack subnet create onap_private_subnet --network onap_private_network \ + --subnet-range 192.168.33.0/24 --ip-version 4 --dhcp --dns-nameserver "8.8.8.8" +openstack router create onap_router +openstack router add subnet onap_router onap_private_subnet +openstack router set onap_router --external-gateway floating_net + +# Allow selected ports and protocols +openstack security group create onap_security_group +openstack security group rule create --protocol icmp onap_security_group +openstack security group rule create --proto tcp \ + --dst-port 22:22 onap_security_group +openstack security group rule create --proto tcp \ + --dst-port 8080:8080 onap_security_group # rancher +openstack security group rule create --proto tcp \ + --dst-port 8078:8078 onap_security_group # horizon +openstack security group rule create --proto tcp \ + --dst-port 8879:8879 onap_security_group # helm +openstack security group rule create --proto tcp \ + --dst-port 80:80 onap_security_group +openstack security group rule create --proto tcp \ + --dst-port 443:443 onap_security_group + +# Allow communication between k8s cluster nodes +PUBLIC_NET=`openstack subnet list --name floating_subnet -f value -c Subnet` +openstack security group rule create --remote-ip $PUBLIC_NET --proto tcp \ + --dst-port 1:65535 onap_security_group +openstack security group rule create --remote-ip $PUBLIC_NET --proto udp \ + --dst-port 1:65535 onap_security_group + +# Create VMs and assign floating IPs to them +VM_ITER=1 +while [ $VM_ITER -le $VM_COUNT ] ; do + openstack floating ip create floating_net + VM_NAME[$VM_ITER]="onap_vm${VM_ITER}" + VM_IP[$VM_ITER]=$(openstack floating ip list -c "Floating IP Address" \ + -c "Port" -f value | grep None | cut -f1 -d " " | head -n1) + openstack server create --flavor onap.large --image xenial \ + --nic net-id=onap_private_network --security-group onap_security_group \ + --key-name onap_key ${VM_NAME[$VM_ITER]} + sleep 5 # wait for VM init before floating IP can be assigned + openstack server add floating ip ${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]} + VM_ITER=$(($VM_ITER+1)) +done + +openstack server list + +echo "Waiting for VMs to start up for 5 minutes at $(date)" +sleep 5m + +openstack server list + +# Start ONAP installation +DATE_START=$(date) +echo -e "\nONAP Installation Started at $DATE_START\n" +$AUTO_INSTALL_DIR/ci/deploy-onap.sh ${VM_IP[@]} +echo -e "\nONAP Installation Started at $DATE_START" +echo -e "ONAP Installation Finished at $(date)\n" diff --git a/ci/deploy-onap.sh b/ci/deploy-onap.sh index e886492..4d7e3b3 100755 --- a/ci/deploy-onap.sh +++ b/ci/deploy-onap.sh @@ -26,8 +26,8 @@ # NOTE: Following must be assured for all MASTER and SLAVE servers before # onap-deploy.sh execution: # 1) ssh access without a password -# 2) an "opnfv" user account with password-less sudo access must be -# available +# 2) an user account with password-less sudo access must be +# available - default user is "opnfv" # # Configuration @@ -40,10 +40,24 @@ HELM_VERSION=2.8.2 MASTER=$1 SERVERS=$* +shift +SLAVES=$* -BRANCH='master' +BRANCH='beijing' ENVIRON='onap' +SSH_USER=${SSH_USER:-"opnfv"} +SSH_OPTIONS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' +# by defalult install full ONAP installation +ONAP_COMPONENT_DISABLE=${ONAP_COMPONENT_DISABLE:-""} +# example of minimal ONAP installation +#ONAP_COMPONENT_DISABLE="clamp cli consul dcaegen2 esr log msb multicloud nbi oof policy uui vfc vnfsdk" + +# use identity file from the environment SSH_IDENTITY +if [ -n "$SSH_IDENTITY" ] ; then + SSH_OPTIONS="-i $SSH_IDENTITY $SSH_OPTIONS" +fi + # # Installation # @@ -52,9 +66,10 @@ echo "$SERVERS" for MACHINE in $SERVERS; do -ssh opnfv@"$MACHINE" "bash -s" <>/etc/exports service nfs-kernel-server restart echo "Waiting 10 minutes for Rancher to setup at \$(date)" -sleep 600 +sleep 10m echo "Installing RANCHER CLI, KUBERNETES ENV on RANCHER" wget https://github.com/rancher/cli/releases/download/v${RANCHER_CLI_VER}-rc2\ /rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz @@ -142,6 +155,10 @@ echo "Creating kubernetes environment named ${ENVIRON}" ./rancher env create -t kubernetes $ENVIRON > kube_env_id.json PROJECT_ID=\$( /tmp/rancher_register_host -chown opnfv /tmp/rancher_register_host +chown $SSH_USER /tmp/rancher_register_host RANCHERINSTALL -wait echo "REGISTER TOKEN" -HOSTREGTOKEN=$(ssh opnfv@"$MASTER" cat /tmp/rancher_register_host) +HOSTREGTOKEN=$(ssh $SSH_OPTIONS $SSH_USER@"$MASTER" cat /tmp/rancher_register_host) echo "$HOSTREGTOKEN" echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'" @@ -211,22 +227,38 @@ echo "$SERVERS" for MACHINE in $SERVERS; do -ssh opnfv@"$MACHINE" "bash -s" <> /etc/fstab + mount -a + mount | grep dockerdata-nfs +CONFIGURENFS +done +wait + echo "DEPLOYING OOM ON RANCHER WITH MASTER" echo "$MASTER" -ssh opnfv@"$MASTER" "bash -s" </dev/null & echo "Waiting for helm setup for 5 min at \$(date)" -sleep 300 +sleep 5m helm version helm repo add local http://127.0.0.1:8879 helm repo list @@ -254,40 +298,43 @@ helm install local/onap -n dev --namespace $ENVIRON cd ../../ echo "Waiting for all pods to be up for 15-80 min at \$(date)" -FAILED_PODS_LIMIT=0 -MAX_WAIT_PERIODS=480 # 120 MIN +echo "Ignore failure of sdnc-ansible-server, see SDNC-443" +TMP_POD_LIST='/tmp/onap_pod_list.txt' +function get_onap_pods() { + kubectl get pods --namespace $ENVIRON > \$TMP_POD_LIST + return \$(cat \$TMP_POD_LIST | wc -l) +} +FAILED_PODS_LIMIT=1 # maximal number of falied ONAP PODs +ALL_PODS_LIMIT=20 # minimum ONAP PODs to be up & running +MAX_WAIT_PERIODS=500 # over 2 hours COUNTER=0 -PENDING_PODS=0 -while [ \$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) \ --gt \$FAILED_PODS_LIMIT ]; do - PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) - PENDING_PODS=\$PENDING +get_onap_pods +ALL_PODS=\$? +PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l) +while [ \$PENDING -gt \$FAILED_PODS_LIMIT -o \$ALL_PODS -lt \$ALL_PODS_LIMIT ]; do + # print header every 20th lines + if [ \$COUNTER -eq \$((\$COUNTER/20*20)) ] ; then + printf "%-3s %-29s %-3s/%s\n" "Nr." "Datetime of check" "Err" "Total PODs" + fi + COUNTER=\$((\$COUNTER+1)) + printf "%3s %-29s %3s/%-3s\n" \$COUNTER "\$(date)" \$PENDING \$ALL_PODS sleep 15 - LIST_PENDING=\$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' ) - echo "\${LIST_PENDING}" - echo "\${PENDING} pending > \${FAILED_PODS_LIMIT} at the \${COUNTER}th"\ - " 15 sec interval out of \${MAX_WAIT_PERIODS}" - echo "" - COUNTER=\$((\$COUNTER + 1 )) if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then FAILED_PODS_LIMIT=800 + ALL_PODS_LIMIT=0 fi + get_onap_pods + ALL_PODS=\$? + PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l) done echo "Report on non-running containers" -PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2') -PENDING_COUNT=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) -PENDING_COUNT_AAI=\$(kubectl get pods -n $ENVIRON | grep aai- \ -| grep -E '0/|1/2' | wc -l) - -echo "Check filebeat 2/2 count for ELK stack logging consumption" -FILEBEAT=\$(kubectl get pods --all-namespaces -a | grep 2/) -echo "\${FILEBEAT}" +get_onap_pods +grep -E '0/|1/2' \$TMP_POD_LIST +echo + echo "sleep 5 min - to allow rest frameworks to finish at \$(date)" -sleep 300 -echo "List of ONAP Modules" -LIST_ALL=\$(kubectl get pods --all-namespaces -a --show-all ) -echo "\${LIST_ALL}" +sleep 5m echo "run healthcheck 2 times to warm caches and frameworks"\ "so rest endpoints report properly - see OOM-447" @@ -314,29 +361,19 @@ cloud-regions/ \ # OOM-484 - robot scripts moved cd oom/kubernetes/robot -echo "run healthcheck prep 1" +echo -e "\nrun healthcheck prep 1" # OOM-722 adds namespace parameter -if [ "$BRANCH" == "amsterdam" ]; then - ./ete-k8s.sh health > ~/health1.out -else - ./ete-k8s.sh $ENVIRON health > ~/health1.out -fi +./ete-k8s.sh $ENVIRON health > ~/health1.out echo "sleep 5 min at \$(date)" -sleep 300 +sleep 5m + echo "run healthcheck prep 2" -if [ "$BRANCH" == "amsterdam" ]; then - ./ete-k8s.sh health > ~/health2.out -else - ./ete-k8s.sh $ENVIRON health > ~/health2.out -fi +./ete-k8s.sh $ENVIRON health > ~/health2.out + echo "run healthcheck for real - wait a further 5 min at \$(date)" -sleep 300 -if [ "$BRANCH" == "amsterdam" ]; then - ./ete-k8s.sh health -else - ./ete-k8s.sh $ENVIRON health -fi +sleep 5m +./ete-k8s.sh $ENVIRON health OOMDEPLOY -wait -echo "Finished install, ruturned from Master" + +echo "Finished install, ruturned from Master at $(date)" exit 0 -- cgit 1.2.3-korg