From c5ad6561884f1ee8e868b7922c8b27bda588d6d9 Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Fri, 8 Dec 2017 18:13:25 -0800 Subject: Specify a disk for each ceph OSD node JIRA: MODELS-2 Change-Id: I03ef8e14fcdff44aec97424d4c16b52bf5443df9 Signed-off-by: Bryan Sullivan --- tools/kubernetes/ceph-helm.sh | 23 +++++++++++++---------- tools/kubernetes/demo_deploy.sh | 9 +++++---- tools/kubernetes/k8s-cluster.sh | 13 +++++++------ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh index facc535..031db9a 100644 --- a/tools/kubernetes/ceph-helm.sh +++ b/tools/kubernetes/ceph-helm.sh @@ -20,13 +20,14 @@ #. - key-based auth setup for ssh/scp between master and agent nodes #. - 192.168.0.0/16 should not be used on your server network interface subnets #. Usage: -# Intended to be called from k8s-cluster.sh in this folder. To run directly: -#. $ bash ceph-helm.sh "" [ceph_dev] +#. Intended to be called from k8s-cluster.sh in this folder. To run directly: +#. $ bash ceph-helm.sh "" "" #. nodes: space-separated list of ceph node IPs #. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24 #. public-net: CIDR of public network -#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE*** -#. if not provided, ceph data will be stored on osd nodes in /ceph +#. ceph-dev: space-separated list of disks (e.g. sda, sdb) to use on each +#. worker +#. NOTE: ***DISK MUST NOT BE USED FOR ANY OTHER PURPOSE*** #. #. Status: work in progress, incomplete # @@ -79,10 +80,10 @@ EOG } function setup_ceph() { - nodes=$1 + nodes="$1" private_net=$2 public_net=$3 - dev=$4 + dev="$4" log "Install ceph and ceph-common" make_ceph_setup @@ -145,8 +146,10 @@ EOF log "ceph status is 'HEALTH_OK'" kubectl -n ceph exec -it ceph-mon-0 -- ceph -s + i=1 for node in $nodes; do - log "install ceph, setup resolv.conf, zap disk for $node" + disk=$(echo "$dev" | cut -d ' ' -f $i) + log "install ceph, setup resolv.conf, zap disk $disk for $node" if [[ "$dist" == "ubuntu" ]]; then ssh -x -o StrictHostKeyChecking=no $USER@$node \ sudo apt-get install -y ceph ceph-common @@ -163,7 +166,7 @@ nameserver $kubedns search ceph.svc.cluster.local svc.cluster.local cluster.local options ndots:5 EOF -sudo ceph-disk zap /dev/$dev +sudo ceph-disk zap /dev/$disk EOG log "Run ceph-osd at $node" name=$(ssh -x -o StrictHostKeyChecking=no $USER@$node hostname) @@ -172,7 +175,7 @@ EOG # TODO: leave out sudo... resulted in "./helm-install-ceph-osd.sh: line 40: helm: command not found" ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ $USER@$node sudo chmod 777 /var/lib/ceph/tmp - ./helm-install-ceph-osd.sh $name /dev/$dev + ./helm-install-ceph-osd.sh $name /dev/$disk done for node in $nodes; do @@ -245,7 +248,7 @@ EOG } if [[ "$1" != "" ]]; then - setup_ceph "$1" $2 $3 $4 + setup_ceph "$1" $2 $3 "$4" else grep '#. ' $0 fi diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index 3dc70c4..264cf05 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -28,7 +28,7 @@ #. Usage: on the MAAS server #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models #. $ bash ~/models/tools/kubernetes/demo_deploy.sh "" -#. "" [] +#. "" "" [] #. : space separated list of hostnames managed by MAAS #. : OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7) #. : name of private key for cluster node ssh (in current folder) @@ -38,7 +38,8 @@ #. : CID formatted public network #. : CIDR formatted private network (may be same as pub-net) #. : "helm" or "baremetal" -#. : disk (e.g. sda, sdb) or folder (e.g. "/ceph") +#. : space-separated list of disks (e.g. sda, sdb) to use on each +#. worker, or folder (e.g. "/ceph") #. : optional name of script for extra setup functions as needed #. #. See tools/demo_deploy.sh in the OPNFV VES repo for additional environment @@ -79,7 +80,7 @@ k8s_workers="$5" k8s_priv_net=$6 k8s_pub_net=$7 k8s_ceph_mode=$8 -k8s_ceph_dev=$9 +k8s_ceph_dev="$9" export k8s_nodes export k8s_user export k8s_key @@ -123,7 +124,7 @@ run_master "bash k8s-cluster.sh demo stop nginx" if [[ "$k8s_master" != "$k8s_workers" ]]; then echo; echo "$0 $(date): Setting up ceph-helm" - run_master "bash k8s-cluster.sh ceph \"$k8s_workers\" $k8s_priv_net $k8s_pub_net $k8s_ceph_mode $k8s_ceph_dev" + run_master "bash k8s-cluster.sh ceph \"$k8s_workers\" $k8s_priv_net $k8s_pub_net $k8s_ceph_mode \"$k8s_ceph_dev\"" echo; echo "$0 $(date): Verifying kubernetes+helm+ceph install..." run_master "bash k8s-cluster.sh demo start dokuwiki" diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index d24a03c..f51b169 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -27,13 +27,14 @@ #. $ bash k8s-cluster.sh helm #. Setup helm as kubernetes app management tool. Note this is a #. prerequisite for selecting "helm" ceph-mode as described below. -#. $ bash k8s-cluster.sh ceph "" [ceph_dev] +#. $ bash k8s-cluster.sh ceph "" "" #. nodes: space-separated list of ceph node IPs #. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24 #. public-net: CIDR of public network #. ceph-mode: "helm" or "baremetal" -#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE*** -#. if not provided, ceph data will be stored on osd nodes in /ceph +#. ceph-dev: space-separated list of disks (e.g. sda, sdb) to use on each +#. worker, or folder (e.g. "/ceph") +#. NOTE: ***DISK MUST NOT BE USED FOR ANY OTHER PURPOSE*** #. $ bash k8s-cluster.sh all "" [ceph_dev] #. Runs all the steps above, including starting dokuwiki demo app. #. $ bash k8s-cluster.sh demo @@ -281,9 +282,9 @@ EOF function setup_ceph() { if [[ "$4" == "helm" ]]; then - source ./ceph-helm.sh "$1" $2 $3 $5 + source ./ceph-helm.sh "$1" $2 $3 "$5" else - source ./ceph-baremetal.sh "$1" $2 $3 $5 + source ./ceph-baremetal.sh "$1" $2 $3 "$5" fi } @@ -304,7 +305,7 @@ case "$1" in setup_k8s_workers "$2" ;; ceph) - setup_ceph "$2" $3 $4 $5 $6 + setup_ceph "$2" $3 $4 $5 "$6" ;; helm) bash ./helm-tools.sh setup -- cgit 1.2.3-korg