diff options
Diffstat (limited to 'tools/kubernetes/ceph-helm.sh')
-rw-r--r-- | tools/kubernetes/ceph-helm.sh | 93 |
1 files changed, 84 insertions, 9 deletions
diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh index a1be588..084a4f7 100644 --- a/tools/kubernetes/ceph-helm.sh +++ b/tools/kubernetes/ceph-helm.sh @@ -37,21 +37,75 @@ function log() { echo "$f:$l ($(date)) $1" } +function make_ceph_setup() { + tee ~/ceph_setup.sh <<'EOG' +#!/bin/bash +# Basic server pre-reqs +dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g') +if [[ "$dist" == "ubuntu" ]]; then + sudo apt-get install -y ceph ceph-common +else + # per http://docs.ceph.com/docs/master/install/get-packages/ + sudo tee /etc/yum.repos.d/ceph.repo <<'EOF' +[ceph] +name=Ceph packages for $basearch +baseurl=https://download.ceph.com/rpm-luminous/el7/x86_64 +enabled=1 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc + +[ceph-noarch] +name=Ceph noarch packages +baseurl=https://download.ceph.com/rpm-luminous/el7/noarch +enabled=1 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc + +[ceph-source] +name=Ceph source packages +baseurl=https://download.ceph.com/rpm-luminous/el7/SRPMS +enabled=0 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc +EOF + # TODO: find out why package us unsigned and thus need --nogpgcheck + sudo rpm --import 'https://download.ceph.com/keys/release.asc' + sudo yum install --nogpgcheck -y ceph ceph-common +fi +EOG +} + function setup_ceph() { nodes=$1 private_net=$2 public_net=$3 dev=$4 - - log "Install ceph prerequisites" - sudo apt-get -y install ceph ceph-common + log "Install ceph and ceph-common" + make_ceph_setup + bash ~/ceph_setup.sh + # per https://github.com/att/netarbiter/tree/master/sds/ceph-docker/examples/helm log "Clone netarbiter" git clone https://github.com/att/netarbiter.git - cd netarbiter/sds/ceph-docker/examples/helm + + if [[ "$dist" != "ubuntu" ]]; then + log "Update ceph-helm chart to point to centos images" + sed -i -- 's~daemon: docker.io/knowpd~#daemon: docker.io/knowpd~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~#daemon: docker.io/ceph~daemon: docker.io/ceph~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~ceph_init: docker.io/knowpd~#ceph_init: docker.io/knowpd~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~#ceph_init: docker.io/kollakube~ceph_init: docker.io/kollakube~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + fi log "Prepare a ceph namespace in your K8s cluster" + cd netarbiter/sds/ceph-docker/examples/helm ./prep-ceph-ns.sh log "Run ceph-mon, ceph-mgr, ceph-mon-check, and rbd-provisioner" @@ -93,24 +147,45 @@ EOF for node in $nodes; do log "install ceph, setup resolv.conf, zap disk for $node" - ssh -x -o StrictHostKeyChecking=no ubuntu@$node <<EOG + if [[ "$dist" == "ubuntu" ]]; then + ssh -x -o StrictHostKeyChecking=no $USER@$node \ + sudo apt-get install -y ceph ceph-common + else + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ~/ceph_setup.sh $USER@$node:/home/$USER/. + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $USER@$node bash /home/$USER/ceph_setup.sh + fi + + ssh -x -o StrictHostKeyChecking=no $USER@$node <<EOG cat <<EOF | sudo tee /etc/resolv.conf nameserver $kubedns search ceph.svc.cluster.local svc.cluster.local cluster.local options ndots:5 EOF -sudo apt install -y ceph ceph-common sudo ceph-disk zap /dev/$dev EOG log "Run ceph-osd at $node" - name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname) + name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname) + # TODO: try sudo due to error + # command_check_call: Running command: /usr/bin/ceph-osd --cluster ceph --mkfs -i 0 --monmap /var/lib/ceph/tmp/mnt.JKiQbp/activate.monmap --osd-data /var/lib/ceph/tmp/mnt.JKiQbp --osd-uuid 23e72c93-e5b3-48ad-b919-ef59fe92b189 --setuser ceph --setgroup disk ... -1 bluestore(/var/lib/ceph/tmp/mnt.JKiQbp) _setup_block_symlink_or_file failed to open block file: (13) Permission denied + # TODO: leave out sudo... resulted in "./helm-install-ceph-osd.sh: line 40: helm: command not found" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $USER@$node sudo chmod 777 /var/lib/ceph/tmp ./helm-install-ceph-osd.sh $name /dev/$dev done for node in $nodes; do - name=$(ssh -x -o StrictHostKeyChecking=no ubuntu@$node hostname) + name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname) pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}") - log "verify ceph-osd is Running at node $name" + while "$pod" == "" ; do + log "ceph-osd pod not yet created at node $name, waiting 10 seconds" + kubectl get pods --namespace ceph + sleep 10 + pod=$(kubectl get pods --namespace ceph | awk "/$name/{print \$1}") + done + + log "wait till ceph-osd pod $pod is Running at node $name" status=$(kubectl get pods --namespace ceph $pod | awk "/$pod/ {print \$3}") while [[ "x$status" != "xRunning" ]]; do log "$pod status is $status. Waiting 10 seconds for it to be Running." |