Specify a disk for each ceph OSD node

JIRA: MODELS-2

Change-Id: I03ef8e14fcdff44aec97424d4c16b52bf5443df9
Signed-off-by: Bryan Sullivan <bryan.sullivan@att.com>

3 files changed, 25 insertions, 20 deletions

diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh
index facc535..031db9a 100644
--- a/tools/kubernetes/ceph-helm.sh
+++ b/tools/kubernetes/ceph-helm.sh
@@ -20,13 +20,14 @@
 #. - key-based auth setup for ssh/scp between master and agent nodes
 #. - 192.168.0.0/16 should not be used on your server network interface subnets
 #. Usage:
-#  Intended to be called from k8s-cluster.sh in this folder. To run directly:
-#. $ bash ceph-helm.sh "<nodes>" <cluster-net> <public-net> [ceph_dev]
+#. Intended to be called from k8s-cluster.sh in this folder. To run directly:
+#. $ bash ceph-helm.sh "<nodes>" <cluster-net> <public-net> "<ceph_dev>"
 #.     nodes: space-separated list of ceph node IPs
 #.     cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
 #.     public-net: CIDR of public network
-#.     ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
-#.               if not provided, ceph data will be stored on osd nodes in /ceph
+#.     ceph-dev: space-separated list of disks (e.g. sda, sdb) to use on each
+#.               worker
+#.               NOTE: ***DISK MUST NOT BE USED FOR ANY OTHER PURPOSE***
 #.
 #. Status: work in progress, incomplete
 #
@@ -79,10 +80,10 @@ EOG
 }
 
 function setup_ceph() {
-  nodes=$1
+  nodes="$1"
   private_net=$2
   public_net=$3
-  dev=$4
+  dev="$4"
 
   log "Install ceph and ceph-common"
   make_ceph_setup
@@ -145,8 +146,10 @@ EOF
   log "ceph status is 'HEALTH_OK'"
   kubectl -n ceph exec -it ceph-mon-0 -- ceph -s
 
+  i=1
   for node in $nodes; do
-    log "install ceph, setup resolv.conf, zap disk for $node"
+    disk=$(echo "$dev" | cut -d ' ' -f $i)
+    log "install ceph, setup resolv.conf, zap disk $disk for $node"
     if [[ "$dist" == "ubuntu" ]]; then
       ssh -x -o StrictHostKeyChecking=no $USER@$node \
         sudo apt-get install -y ceph ceph-common
@@ -163,7 +166,7 @@ nameserver $kubedns
 search ceph.svc.cluster.local svc.cluster.local cluster.local
 options ndots:5
 EOF
-sudo ceph-disk zap /dev/$dev
+sudo ceph-disk zap /dev/$disk
 EOG
     log "Run ceph-osd at $node"
     name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname)
@@ -172,7 +175,7 @@ EOG
     # TODO: leave out sudo... resulted in "./helm-install-ceph-osd.sh: line 40: helm: command not found"
     ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
       $USER@$node sudo chmod 777 /var/lib/ceph/tmp
-    ./helm-install-ceph-osd.sh $name /dev/$dev
+    ./helm-install-ceph-osd.sh $name /dev/$disk
   done
 
   for node in $nodes; do
@@ -245,7 +248,7 @@ EOG
 }
 
 if [[ "$1" != "" ]]; then
-  setup_ceph "$1" $2 $3 $4
+  setup_ceph "$1" $2 $3 "$4"
 else
   grep '#. ' $0
 fi
diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh
index 3dc70c4..264cf05 100644
--- a/tools/kubernetes/demo_deploy.sh
+++ b/tools/kubernetes/demo_deploy.sh
@@ -28,7 +28,7 @@
 #. Usage: on the MAAS server
 #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models
 #. $ bash ~/models/tools/kubernetes/demo_deploy.sh "<hosts>" <os> <key>
-#.   <master> "<workers>" <pub-net> <priv-net> <ceph-mode> <ceph-dev> [<extras>]
+#.   <master> "<workers>" <pub-net> <priv-net> <ceph-mode> "<ceph-dev>" [<extras>]
 #. <hosts>: space separated list of hostnames managed by MAAS
 #. <os>: OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7)
 #. <key>: name of private key for cluster node ssh (in current folder)
@@ -38,7 +38,8 @@
 #. <pub-net>: CID formatted public network
 #. <priv-net>: CIDR formatted private network (may be same as pub-net)
 #. <ceph-mode>: "helm" or "baremetal"
-#. <ceph-dev>: disk (e.g. sda, sdb) or folder (e.g. "/ceph")
+#. <ceph-dev>: space-separated list of disks (e.g. sda, sdb) to use on each
+#.             worker, or folder (e.g. "/ceph")
 #. <extras>: optional name of script for extra setup functions as needed
 #.
 #. See tools/demo_deploy.sh in the OPNFV VES repo for additional environment
@@ -79,7 +80,7 @@ k8s_workers="$5"
 k8s_priv_net=$6
 k8s_pub_net=$7
 k8s_ceph_mode=$8
-k8s_ceph_dev=$9
+k8s_ceph_dev="$9"
 export k8s_nodes
 export k8s_user
 export k8s_key
@@ -123,7 +124,7 @@ run_master "bash k8s-cluster.sh demo stop nginx"
 
 if [[ "$k8s_master" != "$k8s_workers" ]]; then
   echo; echo "$0 $(date): Setting up ceph-helm"
-  run_master "bash k8s-cluster.sh ceph \"$k8s_workers\" $k8s_priv_net $k8s_pub_net $k8s_ceph_mode $k8s_ceph_dev"
+  run_master "bash k8s-cluster.sh ceph \"$k8s_workers\" $k8s_priv_net $k8s_pub_net $k8s_ceph_mode \"$k8s_ceph_dev\""
 
   echo; echo "$0 $(date): Verifying kubernetes+helm+ceph install..."
   run_master "bash k8s-cluster.sh demo start dokuwiki"
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh
index d24a03c..f51b169 100644
--- a/tools/kubernetes/k8s-cluster.sh
+++ b/tools/kubernetes/k8s-cluster.sh
@@ -27,13 +27,14 @@
 #. $ bash k8s-cluster.sh helm
 #.     Setup helm as kubernetes app management tool. Note this is a
 #.     prerequisite for selecting "helm" ceph-mode as described below.
-#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
+#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> "<ceph_dev>"
 #.     nodes: space-separated list of ceph node IPs
 #.     cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
 #.     public-net: CIDR of public network
 #.     ceph-mode: "helm" or "baremetal"
-#.     ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
-#.               if not provided, ceph data will be stored on osd nodes in /ceph
+#.     ceph-dev: space-separated list of disks (e.g. sda, sdb) to use on each
+#.               worker, or folder (e.g. "/ceph")
+#.               NOTE: ***DISK MUST NOT BE USED FOR ANY OTHER PURPOSE***
 #. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev]
 #.     Runs all the steps above, including starting dokuwiki demo app.
 #. $ bash k8s-cluster.sh demo <start|stop> <chart>
@@ -281,9 +282,9 @@ EOF
 
 function setup_ceph() {
   if [[ "$4" == "helm" ]]; then
-    source ./ceph-helm.sh "$1" $2 $3 $5
+    source ./ceph-helm.sh "$1" $2 $3 "$5"
   else
-    source ./ceph-baremetal.sh "$1" $2 $3 $5
+    source ./ceph-baremetal.sh "$1" $2 $3 "$5"
   fi
 }
 
@@ -304,7 +305,7 @@ case "$1" in
     setup_k8s_workers "$2"
     ;;
   ceph)
-    setup_ceph "$2" $3 $4 $5 $6
+    setup_ceph "$2" $3 $4 $5 "$6"
     ;;
   helm)
     bash ./helm-tools.sh setup