1 files changed, 84 insertions, 9 deletions

diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh
index a1be588..084a4f7 100644
--- a/tools/kubernetes/ceph-helm.sh
+++ b/tools/kubernetes/ceph-helm.sh
@@ -37,21 +37,75 @@ function log() {
   echo "$f:$l ($(date)) $1"
 }
 
+function make_ceph_setup() {
+  tee ~/ceph_setup.sh <<'EOG'
+#!/bin/bash
+# Basic server pre-reqs
+dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g')
+if [[ "$dist" == "ubuntu" ]]; then
+  sudo apt-get install -y ceph ceph-common
+else
+  # per http://docs.ceph.com/docs/master/install/get-packages/
+  sudo tee /etc/yum.repos.d/ceph.repo <<'EOF'
+[ceph]
+name=Ceph packages for $basearch
+baseurl=https://download.ceph.com/rpm-luminous/el7/x86_64
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-noarch]
+name=Ceph noarch packages
+baseurl=https://download.ceph.com/rpm-luminous/el7/noarch
+enabled=1
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+
+[ceph-source]
+name=Ceph source packages
+baseurl=https://download.ceph.com/rpm-luminous/el7/SRPMS
+enabled=0
+priority=2
+gpgcheck=1
+gpgkey=https://download.ceph.com/keys/release.asc
+EOF
+  # TODO: find out why package us unsigned and thus need --nogpgcheck
+  sudo rpm --import 'https://download.ceph.com/keys/release.asc'
+  sudo yum install --nogpgcheck -y ceph ceph-common
+fi
+EOG
+}
+
 function setup_ceph() {
   nodes=$1
   private_net=$2
   public_net=$3
   dev=$4
-	
-  log "Install ceph prerequisites"
-	sudo apt-get -y install ceph ceph-common
 
+  log "Install ceph and ceph-common"
+  make_ceph_setup
+  bash ~/ceph_setup.sh
+	
   # per https://github.com/att/netarbiter/tree/master/sds/ceph-docker/examples/helm
   log "Clone netarbiter"
   git clone https://github.com/att/netarbiter.git
-  cd netarbiter/sds/ceph-docker/examples/helm
+
+  if [[ "$dist" != "ubuntu" ]]; then
+    log "Update ceph-helm chart to point to centos images"
+    sed -i -- 's~daemon: docker.io/knowpd~#daemon: docker.io/knowpd~' \
+      netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+    sed -i -- 's~#daemon: docker.io/ceph~daemon: docker.io/ceph~' \
+      netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+    sed -i -- 's~ceph_init: docker.io/knowpd~#ceph_init: docker.io/knowpd~' \
+      netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+    sed -i -- 's~#ceph_init: docker.io/kollakube~ceph_init: docker.io/kollakube~' \
+      netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml
+  fi
 
   log "Prepare a ceph namespace in your K8s cluster"
+  cd netarbiter/sds/ceph-docker/examples/helm
   ./prep-ceph-ns.sh
 
   log "Run ceph-mon, ceph-mgr, ceph-mon-check, and rbd-provisioner"
@@ -93,24 +147,45 @@ EOF
 
   for node in $nodes; do
     log "install ceph, setup resolv.conf, zap disk for $node"
-    ssh -x -o StrictHostKeyChecking=no ubuntu@$node <<EOG
+    if [[ "$dist" == "ubuntu" ]]; then
+      ssh -x -o StrictHostKeyChecking=no $USER@$node \
+        sudo apt-get install -y ceph ceph-common
+    else
+      scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+        ~/ceph_setup.sh $USER@$node:/home/$USER/.
+      ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+        $USER@$node bash /home/$USER/ceph_setup.sh
+    fi
+
+    ssh -x -o StrictHostKeyChecking=no $USER@$node <<EOG
 cat <<EOF | sudo tee /etc/resolv.conf
 nameserver $kubedns
 search ceph.svc.cluster.local svc.cluster.local cluster.local
 options ndots:5
 EOF
-sudo apt install -y ceph ceph-common
 sudo ceph-disk zap /dev/$dev
 EOG
     log "Run ceph-osd at $node"
-    name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname)
+    name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname)
+    # TODO: try sudo due to error
+    # command_check_call: Running command: /usr/bin/ceph-osd --cluster ceph --mkfs -i 0 --monmap /var/lib/ceph/tmp/mnt.JKiQbp/activate.monmap --osd-data /var/lib/ceph/tmp/mnt.JKiQbp --osd-uuid 23e72c93-e5b3-48ad-b919-ef59fe92b189 --setuser ceph --setgroup disk ... -1 bluestore(/var/lib/ceph/tmp/mnt.JKiQbp) _setup_block_symlink_or_file failed to open block file: (13) Permission denied
+    # TODO: leave out sudo... resulted in "./helm-install-ceph-osd.sh: line 40: helm: command not found"
+    ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+      $USER@$node sudo chmod 777 /var/lib/ceph/tmp
     ./helm-install-ceph-osd.sh $name /dev/$dev
   done
 
   for node in $nodes; do
-    name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname)
+    name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname)
     pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}")
-    log "verify ceph-osd is Running at node $name"
+    while "$pod" == "" ; do
+      log "ceph-osd pod not yet created at node $name, waiting 10 seconds"
+      kubectl get pods --namespace ceph
+      sleep 10
+      pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}")
+    done
+      
+    log "wait till ceph-osd pod $pod is Running at node $name"
     status=$(kubectl get pods --namespace ceph $pod | awk "/$pod/ {print \$3}")
     while [[ "x$status" != "xRunning" ]]; do
       log "$pod status is $status. Waiting 10 seconds for it to be Running."