ci: ONAP automated installation

Script for automated ONAP installation at LAAS servers.

JIRA: AUTO-49

Change-Id: I3b2c87d431924a81b86dca7a17226613b422a216
Signed-off-by: Richard Elias <richard.elias@tieto.com>
Signed-off-by: Martin Klozik <martin.klozik@tieto.com>

1 files changed, 342 insertions, 0 deletions

diff --git a/ci/deploy-onap.sh b/ci/deploy-onap.sh
new file mode 100755
index 0000000..e886492
--- /dev/null
+++ b/ci/deploy-onap.sh
@@ -0,0 +1,342 @@
+#!/bin/bash
+#
+# Copyright 2018 Tieto
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script for automated deployment of ONAP with Kubernetes at OPNFV LAAS
+# environment.
+#
+# Usage:
+#       onap-deploy.sh <MASTER> <SLAVE1> <SLAVE2>
+#
+#   where <MASTER> and <SLAVE_IPx> are IP addresses of servers to be used
+#   for ONAP installation.
+#
+# NOTE: Following must be assured for all MASTER and SLAVE servers before
+#       onap-deploy.sh execution:
+#       1) ssh access without a password
+#       2) an "opnfv" user account with password-less sudo access must be
+#          available
+
+#
+# Configuration
+#
+DOCKER_VERSION=17.03
+RANCHER_VERSION=1.6.14
+RANCHER_CLI_VER=0.6.11
+KUBECTL_VERSION=1.8.10
+HELM_VERSION=2.8.2
+
+MASTER=$1
+SERVERS=$*
+
+BRANCH='master'
+ENVIRON='onap'
+
+#
+# Installation
+#
+echo "INSTALLING DOCKER ON ALL MACHINES"
+echo "$SERVERS"
+
+for MACHINE in $SERVERS;
+do
+ssh opnfv@"$MACHINE" "bash -s" <<DOCKERINSTALL &
+    sudo su
+    apt-get update
+    curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh
+
+    mkdir -p /etc/systemd/system/docker.service.d/
+    echo "[Service]
+    ExecStart=
+    ExecStart=/usr/bin/dockerd -H fd:// \
+    --insecure-registry=nexus3.onap.org:10001"\
+     > /etc/systemd/system/docker.service.d/docker.conf
+
+    systemctl daemon-reload
+    systemctl restart docker
+    apt-mark hold docker-ce
+
+    for SERVER in $SERVERS;
+    do
+    echo "\$SERVER $ENVIRON\$(echo \$SERVER | cut -d. -f 4 )" >> /etc/hosts
+    done
+
+    hostname $ENVIRON\$(echo $MACHINE | cut -d. -f 4 )
+
+    echo "DOCKER INSTALLED ON $MACHINE"
+DOCKERINSTALL
+done
+wait
+
+echo "INSTALLING RANCHER ON MASTER"
+echo "$MASTER"
+
+ssh opnfv@"$MASTER" "bash -s" <<RANCHERINSTALL &
+sudo su
+apt install jq -y
+echo "Waiting for 30 seconds at \$(date)"
+sleep 30
+
+docker login -u docker -p docker nexus3.onap.org:10001
+
+echo "INSTALL STARTS"
+apt-get install make -y
+
+docker run -d --restart=unless-stopped -p 8080:8080\
+ --name rancher_server rancher/server:v$RANCHER_VERSION
+curl -LO https://storage.googleapis.com/kubernetes-release/\
+release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl
+chmod +x ./kubectl
+mv ./kubectl /usr/local/bin/kubectl
+mkdir ~/.kube
+wget http://storage.googleapis.com/kubernetes-helm\
+/helm-v${HELM_VERSION}-linux-amd64.tar.gz
+tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz
+mv linux-amd64/helm /usr/local/bin/helm
+
+echo "Installing nfs server"
+# changed from nfs_share to dockerdata-nfs
+apt-get install nfs-kernel-server -y
+
+mkdir -p /dockerdata-nfs
+chmod 777 /dockerdata-nfs
+echo "/dockerdata-nfs *(rw,no_root_squash,no_subtree_check)">>/etc/exports
+service nfs-kernel-server restart
+
+echo "Waiting 10 minutes for Rancher to setup at \$(date)"
+sleep 600
+echo "Installing RANCHER CLI, KUBERNETES ENV on RANCHER"
+wget https://github.com/rancher/cli/releases/download/v${RANCHER_CLI_VER}-rc2\
+/rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
+tar -zxvf rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
+cp rancher-v${RANCHER_CLI_VER}-rc2/rancher .
+
+API_RESPONSE=\`curl -s 'http://127.0.0.1:8080/v2-beta/apikey'\
+ -d '{"type":"apikey","accountId":"1a1","name":"autoinstall",\
+ "description":"autoinstall","created":null,"kind":null,\
+ "removeTime":null,"removed":null,"uuid":null}'\`
+# Extract and store token
+echo "API_RESPONSE: \${API_RESPONSE}"
+KEY_PUBLIC=\`echo \${API_RESPONSE} | jq -r .publicValue\`
+KEY_SECRET=\`echo \${API_RESPONSE} | jq -r .secretValue\`
+echo "publicValue: \$KEY_PUBLIC secretValue: \$KEY_SECRET"
+
+export RANCHER_URL=http://${MASTER}:8080
+export RANCHER_ACCESS_KEY=\$KEY_PUBLIC
+export RANCHER_SECRET_KEY=\$KEY_SECRET
+
+./rancher env ls
+echo "Creating kubernetes environment named ${ENVIRON}"
+./rancher env create -t kubernetes $ENVIRON > kube_env_id.json
+PROJECT_ID=\$(<kube_env_id.json)
+echo "env id: \$PROJECT_ID"
+export RANCHER_HOST_URL=http://${MASTER}:8080/v1/projects/\$PROJECT_ID
+echo "you should see an additional kubernetes environment"
+./rancher env ls
+
+REG_URL_RESPONSE=\`curl -X POST -u \$KEY_PUBLIC:\$KEY_SECRET\
+ -H 'Accept: application/json'\
+ -H 'ContentType: application/json'\
+ -d '{"name":"$MASTER"}'\
+ "http://$MASTER:8080/v1/projects/\$PROJECT_ID/registrationtokens"\`
+echo "REG_URL_RESPONSE: \$REG_URL_RESPONSE"
+echo "Waiting for the server to finish url configuration - 1 min at \$(date)"
+sleep 60
+# see registrationUrl in
+REGISTRATION_TOKENS=\`curl http://$MASTER:8080/v2-beta/registrationtokens\`
+echo "REGISTRATION_TOKENS: \$REGISTRATION_TOKENS"
+REGISTRATION_URL=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].registrationUrl\`
+REGISTRATION_DOCKER=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].image\`
+REGISTRATION_TOKEN=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].token\`
+echo "Registering host for image: \$REGISTRATION_DOCKER\
+ url: \$REGISTRATION_URL registrationToken: \$REGISTRATION_TOKEN"
+HOST_REG_COMMAND=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].command\`
+
+# base64 encode the kubectl token from the auth pair
+# generate this after the host is registered
+KUBECTL_TOKEN=\$(echo -n 'Basic '\$(echo\
+ -n "\$RANCHER_ACCESS_KEY:\$RANCHER_SECRET_KEY" | base64 -w 0) | base64 -w 0)
+echo "KUBECTL_TOKEN base64 encoded: \${KUBECTL_TOKEN}"
+
+# add kubectl config - NOTE: the following spacing has to be "exact"
+# or kubectl will not connect - with a localhost:8080 error
+echo 'apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    api-version: v1
+    insecure-skip-tls-verify: true
+    server: "https://$MASTER:8080/r/projects/'\$PROJECT_ID'/kubernetes:6443"
+  name: "${ENVIRON}"
+contexts:
+- context:
+    cluster: "${ENVIRON}"
+    user: "${ENVIRON}"
+  name: "${ENVIRON}"
+current-context: "${ENVIRON}"
+users:
+- name: "${ENVIRON}"
+  user:
+    token: "'\${KUBECTL_TOKEN}'" ' > ~/.kube/config
+
+echo "docker run --rm --privileged\
+ -v /var/run/docker.sock:/var/run/docker.sock\
+ -v /var/lib/rancher:/var/lib/rancher\
+ \$REGISTRATION_DOCKER\
+ \$RANCHER_URL/v1/scripts/\$REGISTRATION_TOKEN"\
+ > /tmp/rancher_register_host
+chown opnfv /tmp/rancher_register_host
+
+RANCHERINSTALL
+wait
+
+echo "REGISTER TOKEN"
+HOSTREGTOKEN=$(ssh opnfv@"$MASTER" cat /tmp/rancher_register_host)
+echo "$HOSTREGTOKEN"
+
+echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'"
+echo "$SERVERS"
+
+for MACHINE in $SERVERS;
+do
+ssh opnfv@"$MACHINE" "bash -s" <<REGISTERHOST &
+    sudo su
+    $HOSTREGTOKEN
+    sleep 5
+    echo "Host $MACHINE waiting for host registration 5 min at \$(date)"
+    sleep 300
+REGISTERHOST
+done
+wait
+
+echo "DEPLOYING OOM ON RANCHER WITH MASTER"
+echo "$MASTER"
+
+ssh opnfv@"$MASTER" "bash -s" <<OOMDEPLOY &
+sudo su
+sysctl -w vm.max_map_count=262144
+rm -rf oom
+echo "pulling new oom"
+git clone -b $BRANCH http://gerrit.onap.org/r/oom
+
+# NFS FIX for aaf-locate
+sed -i '/persistence:/s/^#//' ./oom/kubernetes/aaf/charts/aaf-locate/values.yaml
+sed -i '/mountPath: \/dockerdata/c\    mountPath: \/dockerdata-nfs'\
+ ./oom/kubernetes/aaf/charts/aaf-locate/values.yaml
+
+echo "Pre-pulling docker images at \$(date)"
+wget https://jira.onap.org/secure/attachment/11261/prepull_docker.sh
+chmod 777 prepull_docker.sh
+./prepull_docker.sh
+echo "starting onap pods"
+cd oom/kubernetes/
+helm init --upgrade
+helm serve &
+echo "Waiting for helm setup for 5 min at \$(date)"
+sleep 300
+helm version
+helm repo add local http://127.0.0.1:8879
+helm repo list
+make all
+helm install local/onap -n dev --namespace $ENVIRON
+cd ../../
+
+echo "Waiting for all pods to be up for 15-80 min at \$(date)"
+FAILED_PODS_LIMIT=0
+MAX_WAIT_PERIODS=480 # 120 MIN
+COUNTER=0
+PENDING_PODS=0
+while [  \$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) \
+-gt \$FAILED_PODS_LIMIT ]; do
+  PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
+  PENDING_PODS=\$PENDING
+  sleep 15
+  LIST_PENDING=\$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' )
+  echo "\${LIST_PENDING}"
+  echo "\${PENDING} pending > \${FAILED_PODS_LIMIT} at the \${COUNTER}th"\
+       " 15 sec interval out of \${MAX_WAIT_PERIODS}"
+  echo ""
+  COUNTER=\$((\$COUNTER + 1 ))
+  if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then
+    FAILED_PODS_LIMIT=800
+  fi
+done
+
+echo "Report on non-running containers"
+PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2')
+PENDING_COUNT=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
+PENDING_COUNT_AAI=\$(kubectl get pods -n $ENVIRON | grep aai- \
+| grep -E '0/|1/2' | wc -l)
+
+echo "Check filebeat 2/2 count for ELK stack logging consumption"
+FILEBEAT=\$(kubectl get pods --all-namespaces -a | grep 2/)
+echo "\${FILEBEAT}"
+echo "sleep 5 min - to allow rest frameworks to finish at \$(date)"
+sleep 300
+echo "List of ONAP Modules"
+LIST_ALL=\$(kubectl get pods --all-namespaces -a  --show-all )
+echo "\${LIST_ALL}"
+echo "run healthcheck 2 times to warm caches and frameworks"\
+     "so rest endpoints report properly - see OOM-447"
+
+echo "curl with aai cert to cloud-region PUT"
+curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
+cloud-regions/cloud-region/CloudOwner/RegionOne \
+--data "@aai-cloud-region-put.json" \
+-H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
+-H "X-TransactionId:jimmy-postman" \
+-H "X-FromAppId:AAI" \
+-H "Content-Type:application/json" \
+-H "Accept:application/json" \
+--cacert aaiapisimpledemoopenecomporg_20171003.crt -k
+
+echo "get the cloud region back"
+curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
+cloud-regions/ \
+-H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
+-H "X-TransactionId:jimmy-postman" \
+-H "X-FromAppId:AAI" \
+-H "Content-Type:application/json" \
+-H "Accept:application/json" \
+--cacert aaiapisimpledemoopenecomporg_20171003.crt -k
+
+# OOM-484 - robot scripts moved
+cd oom/kubernetes/robot
+echo "run healthcheck prep 1"
+# OOM-722 adds namespace parameter
+if [ "$BRANCH" == "amsterdam" ]; then
+  ./ete-k8s.sh health > ~/health1.out
+else
+  ./ete-k8s.sh $ENVIRON health > ~/health1.out
+fi
+echo "sleep 5 min at \$(date)"
+sleep 300
+echo "run healthcheck prep 2"
+if [ "$BRANCH" == "amsterdam" ]; then
+  ./ete-k8s.sh health > ~/health2.out
+else
+  ./ete-k8s.sh $ENVIRON health > ~/health2.out
+fi
+echo "run healthcheck for real - wait a further 5 min at \$(date)"
+sleep 300
+if [ "$BRANCH" == "amsterdam" ]; then
+  ./ete-k8s.sh health
+else
+  ./ete-k8s.sh $ENVIRON health
+fi
+OOMDEPLOY
+wait
+echo "Finished install, ruturned from Master"
+exit 0