Merge AT&T WIP on modeled cloud-native stacks into Models

Change-Id: I646825bf7d1a9c1be9c00475028084f920c9d399
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

2 files changed, 594 insertions, 0 deletions

diff --git a/tools/rancher/demo_deploy.sh b/tools/rancher/demo_deploy.sh
new file mode 100644
index 0000000..981b421
--- /dev/null
+++ b/tools/rancher/demo_deploy.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: Complete scripted deployment of an experimental Rancher-based
+#. cloud-native application platform. When complete, Rancher and the following
+#. will be installed:
+#. - nginx and dokuwiki as demo applications
+#. - prometheus + grafana for cluster monitoring/stats
+#.   Prometheus dashboard: http://<master_public_ip>:9090
+#.   Grafana dashboard: http://<master_public_ip>:3000
+#. 
+#. Prerequisites:
+#. - Ubuntu server for Rancher cluster nodes (admin/master and agent nodes)
+#. - MAAS server as cluster admin for Rancher master/agent nodes
+#. - Password-less ssh key provided for node setup
+#. Usage: on the MAAS server
+#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
+#. $ bash ~/models/tools/rancher/demo_deploy.sh <key> "<hosts>" <master_ip>
+#.     "<agent ips>" [<extras>]
+#. <key>: name of private key for cluster node ssh (in current folder)
+#. <hosts>: space separated list of hostnames managed by MAAS
+#. <master_ip>: IP of cluster admin node
+#. <agent_ips>: space separated list of agent node IPs
+#. <extras>: optional name of script for extra setup functions as needed
+
+key=$1
+nodes="$2"
+admin_ip=$3
+agent_ips="$4"
+extras=$5
+
+source ~/models/tools/maas/deploy.sh $1 "$2" $5
+eval `ssh-agent`
+ssh-add $key
+if [[ "x$extras" != "x" ]]; then source $extras; fi
+scp -o StrictHostKeyChecking=no $key ubuntu@$admin_ip:/home/ubuntu/$key
+echo "Setting up Rancher..."
+ssh -x ubuntu@$admin_ip <<EOF
+exec ssh-agent bash
+ssh-add $key
+git clone https://gerrit.opnfv.org/gerrit/models
+bash models/tools/rancher/rancher-cluster.sh all "$agent_ips"
+EOF
+# TODO: Figure this out... Have to break the setup into two steps as something
+# causes the ssh session to end before the prometheus setup, if both scripts 
+# (k8s-cluster and prometheus-tools) are in the same ssh session
+echo "Setting up Prometheus..."
+ssh -x ubuntu@$admin_ip <<EOF
+exec ssh-agent bash
+ssh-add $key
+bash models/tools/prometheus/prometheus-tools.sh all "$agent_ips"
+EOF
+echo "All done!"
diff --git a/tools/rancher/rancher-cluster.sh b/tools/rancher/rancher-cluster.sh
new file mode 100644
index 0000000..42b3c58
--- /dev/null
+++ b/tools/rancher/rancher-cluster.sh
@@ -0,0 +1,529 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property, Inc
+#  
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  
+# http://www.apache.org/licenses/LICENSE-2.0
+#  
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#. What this is: Functions for testing with rancher. 
+#. Prerequisites: 
+#. - Ubuntu server for master and agent nodes
+#. Usage:
+#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
+#. $ cd ~/models/tools/rancher
+#.
+#. Usage:
+#. $ bash rancher_cluster.sh all "<agents>"
+#.   Automate setup and start demo blueprints.
+#.   <agents>: space-separated list of agent node IPs
+#. $ bash rancher_cluster.sh setup "<agents>"
+#.   Installs and starts master and agent nodes.
+#. $ bash rancher_cluster.sh master
+#.   Setup the Rancher master node.
+#. $ bash rancher_cluster.sh agents "<agents>"
+#.   Installs and starts agent nodes.
+#. $ bash rancher_cluster.sh demo
+#.   Start demo blueprints.
+#. $ bash rancher_cluster.sh clean "<agents>"
+#.   Removes Rancher and installed blueprints from the master and agent nodes.
+#.
+#. To call the procedures, directly, e.g. public_endpoint nginx/lb
+#. $ source rancher-cluster.sh 
+#. See below for function-specific usage
+#.
+
+# Install master
+function setup_master() {
+  docker_installed=$(dpkg-query -W --showformat='${Status}\n' docker-ce | grep -c "install ok")
+  if [[ $docker_installed == 0 ]]; then
+    echo "${FUNCNAME[0]}: installing and starting docker"
+    # Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/
+    sudo apt-get remove -y docker docker-engine docker.io
+    sudo apt-get update
+    sudo apt-get install -y \
+      linux-image-extra-$(uname -r) \
+      linux-image-extra-virtual
+    sudo apt-get install -y \
+      apt-transport-https \
+      ca-certificates \
+      curl \
+      software-properties-common
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    sudo add-apt-repository \
+     "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+     $(lsb_release -cs) \
+     stable"
+    sudo apt-get update
+    sudo apt-get install -y docker-ce
+
+    echo "${FUNCNAME[0]}: installing jq"
+    sudo apt-get install -y jq
+  fi
+
+  echo "${FUNCNAME[0]}: installing rancher server (master)"
+  sudo docker run -d --restart=unless-stopped -p 8080:8080 --name rancher rancher/server
+
+  echo "${FUNCNAME[0]}: wait until server is up at http://$1:8080"
+  delay=0
+  id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id')
+  while [[ "$id" == "" ]]; do
+    echo "${FUNCNAME[0]}: rancher server is not yet up, checking again in 10 seconds"
+    sleep 10
+    let delay=$delay+10
+    id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id')
+  done
+  echo "${FUNCNAME[0]}: rancher server is up after $delay seconds"
+
+  rm -rf ~/rancher 
+  mkdir ~/rancher 
+}     
+
+# Install rancher CLI tools
+# Usage example: install_cli_tools 172.16.0.2
+function install_cli_tools() {
+  echo "${FUNCNAME[0]}: installing rancher CLI tools for master $1"
+  cd ~
+  echo "${FUNCNAME[0]}: install Rancher CLI"
+  rm -rf rancher-v0.6.3
+  wget -q https://releases.rancher.com/cli/v0.6.3/rancher-linux-amd64-v0.6.3.tar.gz
+  gzip -d -f rancher-linux-amd64-v0.6.3.tar.gz
+  tar -xvf rancher-linux-amd64-v0.6.3.tar
+  sudo mv rancher-v0.6.3/rancher /usr/bin/rancher
+  echo "${FUNCNAME[0]}: install Rancher Compose"
+  rm -rf rancher-compose-v0.12.5
+  wget -q https://releases.rancher.com/compose/v0.12.5/rancher-compose-linux-amd64-v0.12.5.tar.gz
+  gzip -d -f rancher-compose-linux-amd64-v0.12.5.tar.gz
+  tar -xvf rancher-compose-linux-amd64-v0.12.5.tar
+  sudo mv rancher-compose-v0.12.5/rancher-compose /usr/bin/rancher-compose
+  echo "${FUNCNAME[0]}: setup Rancher CLI environment"
+  # CLI setup http://rancher.com/docs/rancher/v1.6/en/cli/
+  # Under the UI "API" select "Add account API key" and name it. Export the keys:
+  # The following scripted approach assumes you have 1 project/environment (Default)
+  # Set the url that Rancher is on
+  export RANCHER_URL=http://$1:8080/v1
+  id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id')
+  export RANCHER_ENVIRONMENT=$id
+  curl -s -o /tmp/keys -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{"accountId":"reference[account]", "description":"string", "name":"string", "publicValue":"string", "secretValue":"password"}' http://$1:8080/v2-beta/projects/$id/apikeys
+#  curl -s -o /tmp/keys -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d {"type":"apikey","accountId":"1a1","name":"admin","description":null,"created":null,"kind":null,"removed":null,"uuid":null} http://$1:8080/v2-beta/projects/$id/apikey
+  export RANCHER_ACCESS_KEY=$(jq -r '.publicValue' /tmp/keys)
+  export RANCHER_SECRET_KEY=$(jq -r '.secretValue' /tmp/keys)
+  # create the env file ~/.rancher/cli.json
+  rancher config <<EOF
+$RANCHER_URL
+$RANCHER_ACCESS_KEY
+$RANCHER_SECRET_KEY
+EOF
+  
+  master=$(rancher config --print | jq -r '.url' | cut -d '/' -f 3) 
+  echo "${FUNCNAME[0]}: Create registration token"
+  # added sleep to allow server time to be ready to create registration tokens (otherwise error is returned)
+  sleep 5
+  curl -s -o /tmp/token -X POST -u "${RANCHER_ACCESS_KEY}:${RANCHER_SECRET_KEY}" -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{"name":"master"}' http://$master/v1/registrationtokens 
+  while [[ $(jq -r ".type" /tmp/token) != "registrationToken" ]]; do
+    sleep 5
+    curl -s -o /tmp/token -X POST -u "${RANCHER_ACCESS_KEY}:${RANCHER_SECRET_KEY}" -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{"name":"master"}' http://$master/v1/registrationtokens 
+  done
+  id=$(jq -r ".id" /tmp/token)
+  echo "${FUNCNAME[0]}: registration token id=$id"
+
+  echo "${FUNCNAME[0]}: wait until registration command is created"
+  command=$(curl -s -u "${RANCHER_ACCESS_KEY}:${RANCHER_SECRET_KEY}" -H 'Accept: application/json' http://$master/v1/registrationtokens/$id | jq -r '.command')
+  while [[ "$command" == "null" ]]; do
+    echo "${FUNCNAME[0]}: registration command is not yet created, checking again in 10 seconds"
+    sleep 10
+    command=$(curl -s -u "${RANCHER_ACCESS_KEY}:${RANCHER_SECRET_KEY}" -H 'Accept: application/json' http://$master/v1/registrationtokens/$id | jq -r '.command')
+  done
+
+  export RANCHER_REGISTER_COMMAND="$command"
+
+#  echo "${FUNCNAME[0]}: activate rancher debug"
+#  export RANCHER_CLIENT_DEBUG=true
+
+  echo "${FUNCNAME[0]}: Install docker-compose for syntax checks"
+  sudo apt install -y docker-compose
+
+  cd ~/rancher
+}
+
+# Start an agent host
+# Usage example: start_host Default 172.16.0.7
+function setup_agent() {
+  echo "${FUNCNAME[0]}: SSH to host $2 in env $1 and execute registration command"
+
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$2 "sudo apt-get install -y docker.io; sudo service docker start"
+  ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$2 $RANCHER_REGISTER_COMMAND
+
+  echo "${FUNCNAME[0]}: wait until agent $2 is active"
+  delay=0
+  id=$(rancher hosts | awk "/$2/{print \$1}")
+  while [[ "$id" == "" ]]; do
+    echo "${FUNCNAME[0]}: agent $2 is not yet created, checking again in 10 seconds"
+    sleep 10
+    let delay=$delay+10
+    id=$(rancher hosts | awk "/$2/{print \$1}")
+  done
+
+  echo "${FUNCNAME[0]}: agent $2 id=$id"
+  state=$(rancher inspect $id | jq -r '.state')
+  while [[ "$state" != "active" ]]; do
+    echo "${FUNCNAME[0]}: host $2 state is $state, checking again in 10 seconds"
+    sleep 10
+    let delay=$delay+10
+    state=$(rancher inspect $id | jq -r '.state')
+  done
+  echo "${FUNCNAME[0]}: agent $2 state is $state after $delay seconds"  
+}
+
+# Delete an agent host
+# Usage example: delete_host 172.16.0.7
+function stop_agent() {
+  echo "${FUNCNAME[0]}: deleting host $1"
+  rancher rm --stop $(rancher hosts | awk "/$1/{print \$1}") 
+}
+
+# Test service at access points
+# Usage example: check_service nginx/nginx http "Welcome to nginx!"
+function check_service() {
+  echo "${FUNCNAME[0]}: checking service state for $1 over $2 with match string $3"
+  service=$1
+  scheme=$2
+  match="$3"
+  id=$(rancher ps | grep " $service " | awk "{print \$1}")
+  n=0
+  while [[ "$(rancher inspect $id | jq -r ".publicEndpoints[$n].ipAddress")" != "null" ]]; do
+    ip=$(rancher inspect $id | jq -r ".publicEndpoints[$n].ipAddress")
+    port=$(rancher inspect $id | jq -r ".publicEndpoints[$n].port")
+    while [[ $(wget -qO- $scheme://$ip:$port | grep -c "$match") == 0 ]]; do
+      echo "$service service is NOT active at address $scheme://$ip:$port, waiting 10 seconds"
+      sleep 10
+    done
+    echo "$service service is active at address $scheme://$ip:$port"
+    let n=$n+1
+  done
+}
+
+# Wait n 10-second tries for service to be active
+# Usage example: wait_till_healthy nginx/nginx 6
+function wait_till_healthy() {
+  service=$1
+  tries=$2
+
+  let delay=$tries*10
+  echo "${FUNCNAME[0]}: waiting for service $service to be ready in $delay seconds"
+  id=$(rancher ps | grep " $service " | awk "{print \$1}")
+  health=$(rancher inspect $id | jq -r ".healthState")
+  state=$(rancher inspect $id | jq -r ".state")
+  while [[ $tries > 0 && "$health" != "healthy" ]]; do
+    health=$(rancher inspect $id | jq -r ".healthState")
+    echo $service is $health
+    sleep 10
+  done
+  echo $service state is $(rancher inspect $id | jq -r ".state")
+}
+
+# Start service based upon docker image and simple templates
+# Usage example: start_simple_service nginx nginx:latest 8081:80 3
+# Usage example: start_simple_service dokuwiki ununseptium/dokuwiki-docker 8082:80 2
+function start_simple_service() {
+  echo "${FUNCNAME[0]}: starting service $1 with image $2, ports $3, and scale $4"
+  service=$1
+  image=$2
+  # port is either a single (unexposed) port, or an source:target pair (source
+  # is the external port)
+  ports=$3
+  scale=$4
+ 
+  echo "${FUNCNAME[0]}: creating service folder ~/rancher/$service"
+  mkdir ~/rancher/$service
+  cd  ~/rancher/$service
+  echo "${FUNCNAME[0]}: creating docker-compose.yml"
+  # Define service via docker-compose.yml
+  cat <<EOF >docker-compose.yml
+version: '2'
+services:
+  $service:
+    image: $image
+    ports:
+      - "$ports"
+EOF
+
+  echo "${FUNCNAME[0]}: syntax checking docker-compose.yml"
+  docker-compose -f docker-compose.yml config
+
+  echo "${FUNCNAME[0]}: creating rancher-compose.yml"
+  cat <<EOF >rancher-compose.yml
+version: '2'
+services:
+  # Reference the service that you want to extend
+  $service:
+    scale: $scale
+EOF
+
+  echo "${FUNCNAME[0]}: starting service $service"
+  rancher up -s $service -d
+
+  wait_till_healthy "$service/$service" 6
+  cd  ~/rancher
+}
+
+# Add load balancer to a service
+# Usage example: lb_service nginx 8000 8081
+# Usage example: lb_service dokuwiki 8001 8082
+function lb_service() {
+  echo "${FUNCNAME[0]}: adding load balancer port $2 to service $1, port $3"
+  service=$1
+  lbport=$2
+  port=$3
+ 
+  cd  ~/rancher/$service
+  echo "${FUNCNAME[0]}: creating docker-compose-lb.yml"
+  # Define lb service via docker-compose.yml
+  cat <<EOF >docker-compose-lb.yml
+version: '2'
+services:
+  lb:
+    ports:
+    - $lbport
+    image: rancher/lb-service-haproxy:latest
+EOF
+
+  echo "${FUNCNAME[0]}: syntax checking docker-compose-lb.yml"
+  docker-compose -f docker-compose-lb.yml config
+
+  echo "${FUNCNAME[0]}: creating rancher-compose-lb.yml"
+  cat <<EOF >rancher-compose-lb.yml
+version: '2'
+services:
+  lb:
+    scale: 1
+    lb_config:
+      port_rules:
+      - source_port: $lbport
+        target_port: $port
+        service: $service/$service
+    health_check:
+      port: 42
+      interval: 2000
+      unhealthy_threshold: 3
+      healthy_threshold: 2
+      response_timeout: 2000
+EOF
+
+  echo "${FUNCNAME[0]}: starting service lb"
+  rancher up -s $service -d --file docker-compose-lb.yml --rancher-file rancher-compose-lb.yml
+
+  wait_till_healthy "$service/lb" 6
+  cd  ~/rancher
+}
+
+# Change scale of a service
+# Usage example: scale_service nginx 1
+function scale_service() {
+  echo "${FUNCNAME[0]}: scaling service $1 to $2 instances"
+  id=$(rancher ps | grep " $1 " | awk '{print $1}')
+  rancher scale $id=$2
+
+  scale=$(rancher inspect $id | jq -r '.currentScale')
+  health=$(rancher inspect $id | jq -r '.healthState')
+  while [[ $scale != $2 || "$health" != "healthy" ]]; do
+    echo $service is scaled at $scale and is $health
+    scale=$(rancher inspect $id | jq -r '.currentScale')
+    health=$(rancher inspect $id | jq -r '.healthState')
+    sleep 10
+  done
+  echo $service is scaled at $scale and is $health
+}
+
+# Get public endpoint for a service
+# Usage example public_endpoint nginx/lb
+function public_endpoint() {
+    id=$(rancher ps | grep " $1 " | awk "{print \$1}")
+    ip=$(rancher inspect $id | jq -r ".publicEndpoints[0].ipAddress")
+    port=$(rancher inspect $id | jq -r ".publicEndpoints[0].port")
+    echo "${FUNCNAME[0]}: $1 is accessible at http://$ip:$port"
+}
+
+# Stop a stack
+# Usage example: stop_stack nginx
+function stop_stack() {
+  echo "${FUNCNAME[0]}: stopping stack $1"
+  rancher stop $(rancher stacks | awk "/$1/{print \$1}")
+}
+
+# Start a stopped stack
+# Usage example: start_stack nginx
+function start_stack() {
+  echo "${FUNCNAME[0]}: starting stack $1"
+  rancher start $(rancher stacks | awk "/$1/{print \$1}")
+  wait_till_healthy $1 6
+}
+
+# Delete a stack
+# Usage example: delete_stack dokuwiki
+function delete_stack() {
+  id=$(rancher stacks | grep "$1" | awk "{print \$1}")
+  echo "${FUNCNAME[0]}: deleting stack $1 with id $id"
+  rancher rm --stop $id 
+}
+
+# Delete a service
+# Usage example: delete_service nginx/lb
+function delete_service() {
+  id=$(rancher ps | grep "$1" | awk "{print \$1}")
+  echo "${FUNCNAME[0]}: deleting service $1 with id $id"
+  rancher rm --stop $id 
+}
+
+# Start a complex service, i.e. with yaml file customizations
+# Usage example: start_complex_service grafana 3000:3000 1
+function start_complex_service() {
+  echo "${FUNCNAME[0]}: starting service $1 at ports $2, and scale $3"
+  service=$1
+  # port is either a single (unexposed) port, or an source:target pair (source
+  # is the external port)
+  ports=$2
+  scale=$3
+ 
+  echo "${FUNCNAME[0]}: creating service folder ~/rancher/$service"
+  mkdir ~/rancher/$service
+  cd  ~/rancher/$service
+  echo "${FUNCNAME[0]}: creating docker-compose.yml"
+  # Define service via docker-compose.yml
+  case "$service" in
+    grafana)
+      cat <<EOF >docker-compose.yml
+grafana:
+    image: grafana/grafana:latest
+    ports:
+        - $ports
+    environment:
+        GF_SECURITY_ADMIN_USER: "admin"
+        GF_SECURITY_ADMIN_PASSWORD: "password"
+        GF_SECURITY_SECRET_KEY: $(uuidgen)
+EOF
+    ;;
+         
+    *)
+  esac
+
+  echo "${FUNCNAME[0]}: starting service $service"
+  rancher up -s $service -d
+
+  wait_till_healthy "$service/$service" 6
+  cd  ~/rancher
+}
+
+# Automated demo
+# Usage example: rancher_demo start "172.16.0.7 172.16.0.8 172.16.0.9"
+# Usage example: rancher_demo clean "172.16.0.7 172.16.0.8 172.16.0.9"
+function demo() {
+  # Deploy apps
+  # Nginx web server, accessible on each machine port 8081, and via load
+  # balancer port 8001
+  start=`date +%s`
+  setup "$1"
+  start_simple_service nginx nginx:latest 8081:80 3
+  check_service nginx/nginx http "Welcome to nginx!"
+  lb_service nginx 8001 80
+  check_service nginx/lb http "Welcome to nginx!"
+  # Dokuwiki server, accessible on each machine port 8082, and via load
+  # balancer port 8002
+  start_simple_service dokuwiki ununseptium/dokuwiki-docker 8082:80 2
+  check_service dokuwiki/dokuwiki http "This topic does not exist yet"
+  lb_service dokuwiki 8002 80
+  check_service dokuwiki/lb http "This topic does not exist yet"
+  # Grafana server, accessible on one machine at port 3000
+  start_complex_service grafana 3000:3000 1
+  id=$(rancher ps | grep " grafana/grafana " | awk "{print \$1}")
+  source ~/models/tools/prometheus/prometheus-tools.sh setup "$agents"
+  grafana_ip=$(rancher inspect $id | jq -r ".publicEndpoints[0].ipAddress")
+  prometheus_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
+  connect_grafana $prometheus_ip $grafana_ip
+  public_endpoint nginx/lb
+  public_endpoint dokuwiki/lb
+  public_endpoint grafana/grafana
+
+  end=`date +%s`
+  runtime=$((end-start))
+  runtime=$((runtime/60))
+  echo "${FUNCNAME[0]}: Demo duration = $runtime minutes"
+}
+
+# Automate the installation
+function setup() {
+  # Installation: http://rancher.com/docs/rancher/v1.6/en/
+  # Install rancher server (master) at primary interface of host
+  # Account control is disabled (open access to API), and Default env created
+  ip=$(ip route get 1 | awk '{print $NF;exit}')
+  setup_master $ip
+  # Install rancher CLI tools (rancher, rancher-compose), register with master
+  # and setup CLI environment (e.g. API access/secret keys)
+  install_cli_tools $ip
+
+  # Add agent hosts per http://rancher.com/docs/rancher/v1.6/en/hosts/custom/
+  agents="$1"
+  for agent in $agents; do
+    setup_agent Default $agent
+  done
+}
+
+# Clean the installation
+function clean() {
+  delete_service nginx/lb
+  delete_stack nginx
+  delete_service dokuwiki/lb
+  delete_stack dokuwiki
+  agents="$1"
+  for agent in $agents; do
+    stop_agent $agent
+  done
+  sudo docker stop rancher
+  sudo docker rm -v rancher
+  sudo apt-get remove -y docker-ce
+}
+
+export WORK_DIR=$(pwd)
+case "$1" in
+  master)
+    ip=$(ip route get 1 | awk '{print $NF;exit}')
+    setup_master $ip
+    ;;
+  agents)
+    agents="$2"
+    for agent in $agents; do
+      setup_agent Default $agent
+    done
+    ;;
+  ceph)
+    # TODO Ceph support for rancher, e.g. re
+    # http://rancher.com/docs/rancher/latest/en/rancher-services/storage-service/
+    # https://github.com/rancher/rancher/issues/8722
+    # setup_ceph "$2" $3 $4 $5
+    ;;
+  demo)
+    demo "$2"
+    ;;
+  setup)
+    setup "$2"
+    ;;
+  all)
+    setup "$2"
+    demo "$2"
+    check_service nginx/lb
+    check_service dokuwiki/lb
+    check_service grafana/grafana
+    ;;
+  clean)
+    clean "$2"
+    ;;
+  *)
+    if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then grep '#. ' $0; fi
+esac