aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xci/deploy-onap-fuel.sh49
-rwxr-xr-xci/deploy-onap.sh109
2 files changed, 96 insertions, 62 deletions
diff --git a/ci/deploy-onap-fuel.sh b/ci/deploy-onap-fuel.sh
index cc4dfb1..2e8c3ad 100755
--- a/ci/deploy-onap-fuel.sh
+++ b/ci/deploy-onap-fuel.sh
@@ -33,7 +33,7 @@ CMP_MIN_MEM=${CMP_MIN_MEM:-64000} # MB RAM of the weakest compute node
CMP_MIN_CPUS=${CMP_MIN_CPUS:-36} # CPU count of the weakest compute node
# size of storage for instances
CMP_STORAGE_TOTAL=${CMP_STORAGE_TOTAL:-$((80*$CMP_COUNT))}
-VM_COUNT=${VM_COUNT:-2} # number of VMs available for k8s cluster
+VM_COUNT=${VM_COUNT:-6} # number of VMs available for k8s cluster
#
# Functions
@@ -163,27 +163,64 @@ openstack security group rule create --remote-ip $PUBLIC_NET --proto tcp \
openstack security group rule create --remote-ip $PUBLIC_NET --proto udp \
--dst-port 1:65535 onap_security_group
+# Get list of hypervisors and their zone
+HOST_ZONE=$(openstack host list -f value | grep compute | head -n1 | cut -d' ' -f3)
+HOST_NAME=($(openstack host list -f value | grep compute | cut -d' ' -f1))
+HOST_COUNT=$(echo ${HOST_NAME[@]} | wc -w)
# Create VMs and assign floating IPs to them
VM_ITER=1
+HOST_ITER=0
while [ $VM_ITER -le $VM_COUNT ] ; do
openstack floating ip create floating_net
VM_NAME[$VM_ITER]="onap_vm${VM_ITER}"
VM_IP[$VM_ITER]=$(openstack floating ip list -c "Floating IP Address" \
-c "Port" -f value | grep None | cut -f1 -d " " | head -n1)
+ # dispatch new VMs among compute nodes in round robin fashion
openstack server create --flavor onap.large --image xenial \
--nic net-id=onap_private_network --security-group onap_security_group \
- --key-name onap_key ${VM_NAME[$VM_ITER]}
+ --key-name onap_key ${VM_NAME[$VM_ITER]} \
+ --availability-zone ${HOST_ZONE}:${HOST_NAME[$HOST_ITER]}
sleep 5 # wait for VM init before floating IP can be assigned
openstack server add floating ip ${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]}
VM_ITER=$(($VM_ITER+1))
+ HOST_ITER=$(($HOST_ITER+1))
+ [ $HOST_ITER -ge $HOST_COUNT ] && HOST_ITER=0
done
-openstack server list
+echo "Waiting for VMs to start up for 2m at $(date)"
+sleep 2m
+
+openstack server list -c ID -c Name -c Status -c Networks -c Host --long
+
+# check that SSH to all VMs is working
+SSH_OPTIONS="-i $SSH_IDENTITY -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+COUNTER=1
+while [ $COUNTER -le 10 ] ; do
+ VM_UP=0
+ VM_ITER=1
+ while [ $VM_ITER -le $VM_COUNT ] ; do
+ if ssh $SSH_OPTIONS -l $SSH_USER ${VM_IP[$VM_ITER]} exit &>/dev/null ; then
+ VM_UP=$(($VM_UP+1))
+ echo "${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]}: up"
+ else
+ echo "${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]}: down"
+ fi
+ VM_ITER=$(($VM_ITER+1))
+ done
+ COUNTER=$(($COUNTER+1))
+ if [ $VM_UP -eq $VM_COUNT ] ; then
+ break
+ fi
+ echo "Waiting for VMs to be accessible via ssh for 2m at $(date)"
+ sleep 2m
+done
-echo "Waiting for VMs to start up for 5 minutes at $(date)"
-sleep 5m
+openstack server list -c ID -c Name -c Status -c Networks -c Host --long
-openstack server list
+if [ $VM_UP -ne $VM_COUNT ] ; then
+ echo "Only $VM_UP from $VM_COUNT VMs are accessible via ssh. Installation will be terminated."
+ exit 1
+fi
# Start ONAP installation
DATE_START=$(date)
diff --git a/ci/deploy-onap.sh b/ci/deploy-onap.sh
index 4d7e3b3..c34eb56 100755
--- a/ci/deploy-onap.sh
+++ b/ci/deploy-onap.sh
@@ -61,6 +61,19 @@ fi
#
# Installation
#
+
+# use standalone K8S master if there are enough VMs available for the K8S cluster
+SERVERS_COUNT=$(echo $SERVERS | wc -w)
+if [ $SERVERS_COUNT -gt 2 ] ; then
+ RANCHER_SLAVES=$SLAVES
+else
+ RANCHER_SLAVES=$SERVERS
+fi
+
+echo "INSTALLATION TOPOLOGY:"
+echo "Rancher Master: $MASTER"
+echo "Rancher Slaves: $RANCHER_SLAVES"
+echo
echo "INSTALLING DOCKER ON ALL MACHINES"
echo "$SERVERS"
@@ -223,9 +236,9 @@ HOSTREGTOKEN=$(ssh $SSH_OPTIONS $SSH_USER@"$MASTER" cat /tmp/rancher_register_ho
echo "$HOSTREGTOKEN"
echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'"
-echo "$SERVERS"
+echo "$RANCHER_SLAVES"
-for MACHINE in $SERVERS;
+for MACHINE in $RANCHER_SLAVES;
do
ssh $SSH_OPTIONS $SSH_USER@"$MACHINE" "bash -s" <<REGISTERHOST &
sudo -i
@@ -256,6 +269,7 @@ wait
echo "DEPLOYING OOM ON RANCHER WITH MASTER"
echo "$MASTER"
+TMP_POD_LIST='/tmp/onap_pod_list.txt'
ssh $SSH_OPTIONS $SSH_USER@"$MASTER" "bash -s" <<OOMDEPLOY
sudo -i
@@ -294,85 +308,68 @@ helm version
helm repo add local http://127.0.0.1:8879
helm repo list
make all
-helm install local/onap -n dev --namespace $ENVIRON
+if ( ! helm install local/onap -n dev --namespace $ENVIRON) ; then
+ echo "ONAP installation has failed at \$(date)"
+ exit 1
+fi
+
cd ../../
-echo "Waiting for all pods to be up for 15-80 min at \$(date)"
+echo "Waiting for ONAP pods to be up \$(date)"
echo "Ignore failure of sdnc-ansible-server, see SDNC-443"
-TMP_POD_LIST='/tmp/onap_pod_list.txt'
function get_onap_pods() {
- kubectl get pods --namespace $ENVIRON > \$TMP_POD_LIST
- return \$(cat \$TMP_POD_LIST | wc -l)
+ kubectl get pods --namespace $ENVIRON > $TMP_POD_LIST
+ return \$(cat $TMP_POD_LIST | wc -l)
}
-FAILED_PODS_LIMIT=1 # maximal number of falied ONAP PODs
-ALL_PODS_LIMIT=20 # minimum ONAP PODs to be up & running
-MAX_WAIT_PERIODS=500 # over 2 hours
+FAILED_PODS_LIMIT=1 # maximal number of failed ONAP PODs
+ALL_PODS_LIMIT=20 # minimum ONAP PODs to be up & running
+WAIT_PERIOD=60 # wait period in seconds
+MAX_WAIT_TIME=\$((3600*3)) # max wait time in seconds
+MAX_WAIT_PERIODS=\$((\$MAX_WAIT_TIME/\$WAIT_PERIOD))
COUNTER=0
get_onap_pods
ALL_PODS=\$?
-PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
+PENDING=\$(grep -E '0/|1/2' $TMP_POD_LIST | wc -l)
while [ \$PENDING -gt \$FAILED_PODS_LIMIT -o \$ALL_PODS -lt \$ALL_PODS_LIMIT ]; do
- # print header every 20th lines
+ # print header every 20th line
if [ \$COUNTER -eq \$((\$COUNTER/20*20)) ] ; then
printf "%-3s %-29s %-3s/%s\n" "Nr." "Datetime of check" "Err" "Total PODs"
fi
COUNTER=\$((\$COUNTER+1))
printf "%3s %-29s %3s/%-3s\n" \$COUNTER "\$(date)" \$PENDING \$ALL_PODS
- sleep 15
+ sleep \$WAIT_PERIOD
if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then
FAILED_PODS_LIMIT=800
ALL_PODS_LIMIT=0
fi
get_onap_pods
ALL_PODS=\$?
- PENDING=\$(grep -E '0/|1/2' \$TMP_POD_LIST | wc -l)
+ PENDING=\$(grep -E '0/|1/2' $TMP_POD_LIST | wc -l)
done
-echo "Report on non-running containers"
get_onap_pods
-grep -E '0/|1/2' \$TMP_POD_LIST
+cp $TMP_POD_LIST ~/onap_all_pods.txt
echo
-
-echo "sleep 5 min - to allow rest frameworks to finish at \$(date)"
-sleep 5m
-echo "run healthcheck 2 times to warm caches and frameworks"\
- "so rest endpoints report properly - see OOM-447"
-
-echo "curl with aai cert to cloud-region PUT"
-curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
-cloud-regions/cloud-region/CloudOwner/RegionOne \
---data "@aai-cloud-region-put.json" \
--H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
--H "X-TransactionId:jimmy-postman" \
--H "X-FromAppId:AAI" \
--H "Content-Type:application/json" \
--H "Accept:application/json" \
---cacert aaiapisimpledemoopenecomporg_20171003.crt -k
-
-echo "get the cloud region back"
-curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
-cloud-regions/ \
--H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
--H "X-TransactionId:jimmy-postman" \
--H "X-FromAppId:AAI" \
--H "Content-Type:application/json" \
--H "Accept:application/json" \
---cacert aaiapisimpledemoopenecomporg_20171003.crt -k
-
-# OOM-484 - robot scripts moved
+echo "========================"
+echo "ONAP INSTALLATION REPORT"
+echo "========================"
+echo
+echo "List of Failed PODs"
+echo "-------------------"
+grep -E '0/|1/2' $TMP_POD_LIST | tee ~/onap_failed_pods.txt
+echo
+echo "Summary:"
+echo "--------"
+echo " PODs Failed: \$(cat ~/onap_failed_pods.txt | wc -l)"
+echo " PODs Total: \$(cat ~/onap_all_pods.txt | wc -l)"
+echo
+echo "ONAP health TC results"
+echo "----------------------"
cd oom/kubernetes/robot
-echo -e "\nrun healthcheck prep 1"
-# OOM-722 adds namespace parameter
-./ete-k8s.sh $ENVIRON health > ~/health1.out
-echo "sleep 5 min at \$(date)"
-sleep 5m
-
-echo "run healthcheck prep 2"
-./ete-k8s.sh $ENVIRON health > ~/health2.out
-
-echo "run healthcheck for real - wait a further 5 min at \$(date)"
-sleep 5m
-./ete-k8s.sh $ENVIRON health
+./ete-k8s.sh $ENVIRON health | tee ~/onap_health.txt
+echo "==============================="
+echo "END OF ONAP INSTALLATION REPORT"
+echo "==============================="
OOMDEPLOY
echo "Finished install, ruturned from Master at $(date)"