diff options
Diffstat (limited to 'tools/kubernetes/k8s-cluster.sh')
-rw-r--r-- | tools/kubernetes/k8s-cluster.sh | 178 |
1 files changed, 13 insertions, 165 deletions
diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index 3d896bf..015e217 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -24,17 +24,18 @@ #. $ bash k8s-cluster.sh master #. $ bash k8s-cluster.sh agents "<nodes>" #. nodes: space-separated list of ceph node IPs -#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> [ceph_dev] +#. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev] #. nodes: space-separated list of ceph node IPs #. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24 #. public-net: CIDR of public network +#. ceph-mode: "helm" or "baremetal" #. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE*** #. if not provided, ceph data will be stored on osd nodes in /ceph #. $ bash k8s-cluster.sh helm #. Setup helm as app kubernetes orchestration tool #. $ bash k8s-cluster.sh demo #. Install helm charts for mediawiki and dokuwiki -#. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> [ceph_dev] +#. $ bash k8s-cluster.sh all "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev] #. Runs all the steps above #. #. Status: work in progress, incomplete @@ -122,167 +123,6 @@ function setup_k8s_agents() { echo "${FUNCNAME[0]}: Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'." } -function setup_ceph() { - node_ips=$1 - cluster_net=$2 - public_net=$3 - ceph_dev=$4 - echo "${FUNCNAME[0]}: Deploying ceph-mon on localhost $HOSTNAME" - echo "${FUNCNAME[0]}: Deploying ceph-osd on nodes $node_ips" - echo "${FUNCNAME[0]}: Setting cluster-network=$cluster_net and public-network=$public_net" - mon_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') - all_nodes="$mon_ip $node_ips" - # Also caches the server fingerprints so ceph-deploy does not prompt the user - # Note this loop may be partially redundant with the ceph-deploy steps below - for node_ip in $all_nodes; do - echo "${FUNCNAME[0]}: Install ntp and ceph on $node_ip" - ssh -x -o StrictHostKeyChecking=no ubuntu@$node_ip <<EOF -sudo timedatectl set-ntp no -wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - -echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list -sudo apt update -sudo apt-get install -y ntp ceph ceph-deploy -EOF - done - - # per http://docs.ceph.com/docs/master/start/quick-ceph-deploy/ - # also https://upcommons.upc.edu/bitstream/handle/2117/101816/Degree_Thesis_Nabil_El_Alami.pdf#vote +1 - echo "${FUNCNAME[0]}: Create ceph config folder ~/ceph-cluster" - mkdir ~/ceph-cluster - cd ~/ceph-cluster - - echo "${FUNCNAME[0]}: Create new cluster with $HOSTNAME as initial ceph-mon node" - ceph-deploy new --cluster-network $cluster_net --public-network $public_net --no-ssh-copykey $HOSTNAME - # Update conf per recommendations of http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/ - cat <<EOF >>ceph.conf -osd max object name len = 256 -osd max object namespace len = 64 -EOF - cat ceph.conf - - echo "${FUNCNAME[0]}: Deploy ceph packages on other nodes" - ceph-deploy install $mon_ip $node_ips - - echo "${FUNCNAME[0]}: Deploy the initial monitor and gather the keys" - ceph-deploy mon create-initial - - if [[ "x$ceph_dev" == "x" ]]; then - n=1 - for node_ip in $node_ips; do - echo "${FUNCNAME[0]}: Prepare ceph OSD on node $node_ip" - echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts - # Using ceph-osd$n here avoids need for manual acceptance of the new server hash - ssh -x -o StrictHostKeyChecking=no ubuntu@ceph-osd$n <<EOF -echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts -sudo mkdir /ceph && sudo chown -R ceph:ceph /ceph -EOF - ceph-deploy osd prepare ceph-osd$n:/ceph - ceph-deploy osd activate ceph-osd$n:/ceph - ((n++)) - done - else - echo "${FUNCNAME[0]}: Deploy OSDs" - for node_ip in $node_ips; do - echo "${FUNCNAME[0]}: Create ceph osd on $node_ip using $ceph_dev" - ceph-deploy osd create $node_ip:$ceph_dev - done - fi - - echo "${FUNCNAME[0]}: Copy the config file and admin key to the admin node and OSD nodes" - ceph-deploy admin $mon_ip $node_ips - - echo "${FUNCNAME[0]}: Check the cluster health" - sudo ceph health - sudo ceph -s - - # per https://crondev.com/kubernetes-persistent-storage-ceph/ and https://github.com/kubernetes/kubernetes/issues/38923 - # rbd is not included in default kube-controller-manager... use attcomdev version - sudo sed -i -- 's~gcr.io/google_containers/kube-controller-manager-amd64:.*~quay.io/attcomdev/kube-controller-manager:v1.7.3~' /etc/kubernetes/manifests/kube-controller-manager.yaml - if [[ $(sudo grep -c attcomdev/kube-controller-manager /etc/kubernetes/manifests/kube-controller-manager.yaml) == 0 ]]; then - echo "${FUNCNAME[0]}: Problem patching /etc/kubernetes/manifests/kube-controller-manager.yaml... script update needed" - exit 1 - fi - mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}') - while [[ "$mgr" != "Running" ]]; do - echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr. Waiting 60 seconds for it to be 'Running'" - sleep 60 - mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}') - done - echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr" - - echo "${FUNCNAME[0]}: Create Ceph admin secret" - admin_key=$(sudo ceph auth get-key client.admin) - kubectl create secret generic ceph-secret-admin --from-literal=key="$admin_key" --namespace=kube-system --type=kubernetes.io/rbd - - echo "${FUNCNAME[0]}: Create rdb storageClass 'slow'" - cat <<EOF >/tmp/ceph-sc.yaml -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: slow -provisioner: kubernetes.io/rbd -parameters: - monitors: $mon_ip:6789 - adminId: admin - adminSecretName: ceph-secret-admin - adminSecretNamespace: "kube-system" - pool: kube - userId: kube - userSecretName: ceph-secret-user -EOF - # TODO: find out where in the above ~/.kube folders became owned by root - sudo chown -R ubuntu:ubuntu ~/.kube/* - kubectl create -f /tmp/ceph-sc.yaml - - echo "${FUNCNAME[0]}: Create storage pool 'kube'" - # https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md method - sudo ceph osd pool create kube 32 32 - - echo "${FUNCNAME[0]}: Authorize client 'kube' access to pool 'kube'" - sudo ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube' - - echo "${FUNCNAME[0]}: Create ceph-secret-user secret in namespace 'default'" - kube_key=$(sudo ceph auth get-key client.kube) - kubectl create secret generic ceph-secret-user --from-literal=key="$kube_key" --namespace=default --type=kubernetes.io/rbd - # A similar secret must be created in other namespaces that intend to access the ceph pool - - # Per https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md - - echo "${FUNCNAME[0]}: Create andtest a persistentVolumeClaim" - cat <<EOF >/tmp/ceph-pvc.yaml -{ - "kind": "PersistentVolumeClaim", - "apiVersion": "v1", - "metadata": { - "name": "claim1", - "annotations": { - "volume.beta.kubernetes.io/storage-class": "slow" - } - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "3Gi" - } - } - } -} -EOF - kubectl create -f /tmp/ceph-pvc.yaml - while [[ "x$(kubectl get pvc -o jsonpath='{.status.phase}' claim1)" != "xBound" ]]; do - echo "${FUNCNAME[0]}: Waiting for pvc claim1 to be 'Bound'" - kubectl describe pvc - sleep 10 - done - echo "${FUNCNAME[0]}: pvc claim1 successfully bound to $(kubectl get pvc -o jsonpath='{.spec.volumeName}' claim1)" - kubectl get pvc - kubectl delete pvc claim1 - kubectl describe pods -} - function wait_for_service() { echo "${FUNCNAME[0]}: Waiting for service $1 to be available" pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") @@ -406,6 +246,14 @@ function setup_helm() { # e.g. helm install stable/dokuwiki } +function setup_ceph() { + if [[ "$4" .eq "helm" ]]; then + source ./ceph-helm.sh "$1" $2 $3 $4 + else + source ./ceph-baremetal.sh "$1" $2 $3 $4 + fi +} + export WORK_DIR=$(pwd) case "$1" in master) @@ -415,7 +263,7 @@ case "$1" in setup_k8s_agents "$2" ;; ceph) - setup_ceph "$2" $3 $4 $5 + setup_ceph "$2" $3 $4 $5 $6 ;; helm) setup_helm @@ -426,7 +274,7 @@ case "$1" in all) setup_k8s_master setup_k8s_agents "$2" - setup_ceph "$2" $3 $4 $5 + setup_ceph "$2" $3 $4 $5 $6 setup_helm demo_chart dokuwiki ;; |