path: root/ci/deploy-onap.sh

#!/bin/bash
#
# Copyright 2018 Tieto
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script for automated deployment of ONAP with Kubernetes at OPNFV LAAS
# environment.
#
# Usage:
#       onap-deploy.sh <MASTER> <SLAVE1> <SLAVE2>
#
#   where <MASTER> and <SLAVE_IPx> are IP addresses of servers to be used
#   for ONAP installation.
#
# NOTE: Following must be assured for all MASTER and SLAVE servers before
#       onap-deploy.sh execution:
#       1) ssh access without a password
#       2) an "opnfv" user account with password-less sudo access must be
#          available

#
# Configuration
#
DOCKER_VERSION=17.03
RANCHER_VERSION=1.6.14
RANCHER_CLI_VER=0.6.11
KUBECTL_VERSION=1.8.10
HELM_VERSION=2.8.2

MASTER=$1
SERVERS=$*

BRANCH='master'
ENVIRON='onap'

#
# Installation
#
echo "INSTALLING DOCKER ON ALL MACHINES"
echo "$SERVERS"

for MACHINE in $SERVERS;
do
ssh opnfv@"$MACHINE" "bash -s" <<DOCKERINSTALL &
    sudo su
    apt-get update
    curl https://releases.rancher.com/install-docker/$DOCKER_VERSION.sh | sh

    mkdir -p /etc/systemd/system/docker.service.d/
    echo "[Service]
    ExecStart=
    ExecStart=/usr/bin/dockerd -H fd:// \
    --insecure-registry=nexus3.onap.org:10001"\
     > /etc/systemd/system/docker.service.d/docker.conf

    systemctl daemon-reload
    systemctl restart docker
    apt-mark hold docker-ce

    for SERVER in $SERVERS;
    do
    echo "\$SERVER $ENVIRON\$(echo \$SERVER | cut -d. -f 4 )" >> /etc/hosts
    done

    hostname $ENVIRON\$(echo $MACHINE | cut -d. -f 4 )

    echo "DOCKER INSTALLED ON $MACHINE"
DOCKERINSTALL
done
wait

echo "INSTALLING RANCHER ON MASTER"
echo "$MASTER"

ssh opnfv@"$MASTER" "bash -s" <<RANCHERINSTALL &
sudo su
apt install jq -y
echo "Waiting for 30 seconds at \$(date)"
sleep 30

docker login -u docker -p docker nexus3.onap.org:10001

echo "INSTALL STARTS"
apt-get install make -y

docker run -d --restart=unless-stopped -p 8080:8080\
 --name rancher_server rancher/server:v$RANCHER_VERSION
curl -LO https://storage.googleapis.com/kubernetes-release/\
release/v$KUBECTL_VERSION/bin/linux/amd64/kubectl
chmod +x ./kubectl
mv ./kubectl /usr/local/bin/kubectl
mkdir ~/.kube
wget http://storage.googleapis.com/kubernetes-helm\
/helm-v${HELM_VERSION}-linux-amd64.tar.gz
tar -zxvf helm-v${HELM_VERSION}-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/helm

echo "Installing nfs server"
# changed from nfs_share to dockerdata-nfs
apt-get install nfs-kernel-server -y

mkdir -p /dockerdata-nfs
chmod 777 /dockerdata-nfs
echo "/dockerdata-nfs *(rw,no_root_squash,no_subtree_check)">>/etc/exports
service nfs-kernel-server restart

echo "Waiting 10 minutes for Rancher to setup at \$(date)"
sleep 600
echo "Installing RANCHER CLI, KUBERNETES ENV on RANCHER"
wget https://github.com/rancher/cli/releases/download/v${RANCHER_CLI_VER}-rc2\
/rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
tar -zxvf rancher-linux-amd64-v${RANCHER_CLI_VER}-rc2.tar.gz
cp rancher-v${RANCHER_CLI_VER}-rc2/rancher .

API_RESPONSE=\`curl -s 'http://127.0.0.1:8080/v2-beta/apikey'\
 -d '{"type":"apikey","accountId":"1a1","name":"autoinstall",\
 "description":"autoinstall","created":null,"kind":null,\
 "removeTime":null,"removed":null,"uuid":null}'\`
# Extract and store token
echo "API_RESPONSE: \${API_RESPONSE}"
KEY_PUBLIC=\`echo \${API_RESPONSE} | jq -r .publicValue\`
KEY_SECRET=\`echo \${API_RESPONSE} | jq -r .secretValue\`
echo "publicValue: \$KEY_PUBLIC secretValue: \$KEY_SECRET"

export RANCHER_URL=http://${MASTER}:8080
export RANCHER_ACCESS_KEY=\$KEY_PUBLIC
export RANCHER_SECRET_KEY=\$KEY_SECRET

./rancher env ls
echo "Creating kubernetes environment named ${ENVIRON}"
./rancher env create -t kubernetes $ENVIRON > kube_env_id.json
PROJECT_ID=\$(<kube_env_id.json)
echo "env id: \$PROJECT_ID"
export RANCHER_HOST_URL=http://${MASTER}:8080/v1/projects/\$PROJECT_ID
echo "you should see an additional kubernetes environment"
./rancher env ls

REG_URL_RESPONSE=\`curl -X POST -u \$KEY_PUBLIC:\$KEY_SECRET\
 -H 'Accept: application/json'\
 -H 'ContentType: application/json'\
 -d '{"name":"$MASTER"}'\
 "http://$MASTER:8080/v1/projects/\$PROJECT_ID/registrationtokens"\`
echo "REG_URL_RESPONSE: \$REG_URL_RESPONSE"
echo "Waiting for the server to finish url configuration - 1 min at \$(date)"
sleep 60
# see registrationUrl in
REGISTRATION_TOKENS=\`curl http://$MASTER:8080/v2-beta/registrationtokens\`
echo "REGISTRATION_TOKENS: \$REGISTRATION_TOKENS"
REGISTRATION_URL=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].registrationUrl\`
REGISTRATION_DOCKER=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].image\`
REGISTRATION_TOKEN=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].token\`
echo "Registering host for image: \$REGISTRATION_DOCKER\
 url: \$REGISTRATION_URL registrationToken: \$REGISTRATION_TOKEN"
HOST_REG_COMMAND=\`echo \$REGISTRATION_TOKENS | jq -r .data[0].command\`

# base64 encode the kubectl token from the auth pair
# generate this after the host is registered
KUBECTL_TOKEN=\$(echo -n 'Basic '\$(echo\
 -n "\$RANCHER_ACCESS_KEY:\$RANCHER_SECRET_KEY" | base64 -w 0) | base64 -w 0)
echo "KUBECTL_TOKEN base64 encoded: \${KUBECTL_TOKEN}"

# add kubectl config - NOTE: the following spacing has to be "exact"
# or kubectl will not connect - with a localhost:8080 error
echo 'apiVersion: v1
kind: Config
clusters:
- cluster:
    api-version: v1
    insecure-skip-tls-verify: true
    server: "https://$MASTER:8080/r/projects/'\$PROJECT_ID'/kubernetes:6443"
  name: "${ENVIRON}"
contexts:
- context:
    cluster: "${ENVIRON}"
    user: "${ENVIRON}"
  name: "${ENVIRON}"
current-context: "${ENVIRON}"
users:
- name: "${ENVIRON}"
  user:
    token: "'\${KUBECTL_TOKEN}'" ' > ~/.kube/config

echo "docker run --rm --privileged\
 -v /var/run/docker.sock:/var/run/docker.sock\
 -v /var/lib/rancher:/var/lib/rancher\
 \$REGISTRATION_DOCKER\
 \$RANCHER_URL/v1/scripts/\$REGISTRATION_TOKEN"\
 > /tmp/rancher_register_host
chown opnfv /tmp/rancher_register_host

RANCHERINSTALL
wait

echo "REGISTER TOKEN"
HOSTREGTOKEN=$(ssh opnfv@"$MASTER" cat /tmp/rancher_register_host)
echo "$HOSTREGTOKEN"

echo "REGISTERING HOSTS WITH RANCHER ENVIRONMENT '$ENVIRON'"
echo "$SERVERS"

for MACHINE in $SERVERS;
do
ssh opnfv@"$MACHINE" "bash -s" <<REGISTERHOST &
    sudo su
    $HOSTREGTOKEN
    sleep 5
    echo "Host $MACHINE waiting for host registration 5 min at \$(date)"
    sleep 300
REGISTERHOST
done
wait

echo "DEPLOYING OOM ON RANCHER WITH MASTER"
echo "$MASTER"

ssh opnfv@"$MASTER" "bash -s" <<OOMDEPLOY &
sudo su
sysctl -w vm.max_map_count=262144
rm -rf oom
echo "pulling new oom"
git clone -b $BRANCH http://gerrit.onap.org/r/oom

# NFS FIX for aaf-locate
sed -i '/persistence:/s/^#//' ./oom/kubernetes/aaf/charts/aaf-locate/values.yaml
sed -i '/mountPath: \/dockerdata/c\    mountPath: \/dockerdata-nfs'\
 ./oom/kubernetes/aaf/charts/aaf-locate/values.yaml

echo "Pre-pulling docker images at \$(date)"
wget https://jira.onap.org/secure/attachment/11261/prepull_docker.sh
chmod 777 prepull_docker.sh
./prepull_docker.sh
echo "starting onap pods"
cd oom/kubernetes/
helm init --upgrade
helm serve &
echo "Waiting for helm setup for 5 min at \$(date)"
sleep 300
helm version
helm repo add local http://127.0.0.1:8879
helm repo list
make all
helm install local/onap -n dev --namespace $ENVIRON
cd ../../

echo "Waiting for all pods to be up for 15-80 min at \$(date)"
FAILED_PODS_LIMIT=0
MAX_WAIT_PERIODS=480 # 120 MIN
COUNTER=0
PENDING_PODS=0
while [  \$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l) \
-gt \$FAILED_PODS_LIMIT ]; do
  PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
  PENDING_PODS=\$PENDING
  sleep 15
  LIST_PENDING=\$(kubectl get pods --all-namespaces -o wide | grep -E '0/|1/2' )
  echo "\${LIST_PENDING}"
  echo "\${PENDING} pending > \${FAILED_PODS_LIMIT} at the \${COUNTER}th"\
       " 15 sec interval out of \${MAX_WAIT_PERIODS}"
  echo ""
  COUNTER=\$((\$COUNTER + 1 ))
  if [ "\$MAX_WAIT_PERIODS" -eq \$COUNTER ]; then
    FAILED_PODS_LIMIT=800
  fi
done

echo "Report on non-running containers"
PENDING=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2')
PENDING_COUNT=\$(kubectl get pods --all-namespaces | grep -E '0/|1/2' | wc -l)
PENDING_COUNT_AAI=\$(kubectl get pods -n $ENVIRON | grep aai- \
| grep -E '0/|1/2' | wc -l)

echo "Check filebeat 2/2 count for ELK stack logging consumption"
FILEBEAT=\$(kubectl get pods --all-namespaces -a | grep 2/)
echo "\${FILEBEAT}"
echo "sleep 5 min - to allow rest frameworks to finish at \$(date)"
sleep 300
echo "List of ONAP Modules"
LIST_ALL=\$(kubectl get pods --all-namespaces -a  --show-all )
echo "\${LIST_ALL}"
echo "run healthcheck 2 times to warm caches and frameworks"\
     "so rest endpoints report properly - see OOM-447"

echo "curl with aai cert to cloud-region PUT"
curl -X PUT https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
cloud-regions/cloud-region/CloudOwner/RegionOne \
--data "@aai-cloud-region-put.json" \
-H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
-H "X-TransactionId:jimmy-postman" \
-H "X-FromAppId:AAI" \
-H "Content-Type:application/json" \
-H "Accept:application/json" \
--cacert aaiapisimpledemoopenecomporg_20171003.crt -k

echo "get the cloud region back"
curl -X GET https://127.0.0.1:30233/aai/v11/cloud-infrastructure/\
cloud-regions/ \
-H "authorization: Basic TW9kZWxMb2FkZXI6TW9kZWxMb2FkZXI=" \
-H "X-TransactionId:jimmy-postman" \
-H "X-FromAppId:AAI" \
-H "Content-Type:application/json" \
-H "Accept:application/json" \
--cacert aaiapisimpledemoopenecomporg_20171003.crt -k

# OOM-484 - robot scripts moved
cd oom/kubernetes/robot
echo "run healthcheck prep 1"
# OOM-722 adds namespace parameter
if [ "$BRANCH" == "amsterdam" ]; then
  ./ete-k8s.sh health > ~/health1.out
else
  ./ete-k8s.sh $ENVIRON health > ~/health1.out
fi
echo "sleep 5 min at \$(date)"
sleep 300
echo "run healthcheck prep 2"
if [ "$BRANCH" == "amsterdam" ]; then
  ./ete-k8s.sh health > ~/health2.out
else
  ./ete-k8s.sh $ENVIRON health > ~/health2.out
fi
echo "run healthcheck for real - wait a further 5 min at \$(date)"
sleep 300
if [ "$BRANCH" == "amsterdam" ]; then
  ./ete-k8s.sh health
else
  ./ete-k8s.sh $ENVIRON health
fi
OOMDEPLOY
wait
echo "Finished install, ruturned from Master"
exit 0