1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
#!/bin/bash
# Copyright 2017 AT&T Intellectual Property, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#. What this is: script to setup a Ceph-based SDS (Software Defined Storage)
#. service for a kubernetes cluster, directly on the master and worker nodes.
#. Prerequisites:
#. - Ubuntu xenial server for master and agent nodes
#. - key-based auth setup for ssh/scp between master and agent nodes
#. - 192.168.0.0/16 should not be used on your server network interface subnets
#. Usage:
# Intended to be called from k8s-cluster.sh in this folder. To run directly:
#. $ bash ceph-baremetal.sh "<nodes>" <cluster-net> <public-net> [ceph_dev]
#. nodes: space-separated list of ceph node IPs
#. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24
#. public-net: CIDR of public network
#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE***
#. if not provided, ceph data will be stored on osd nodes in /ceph
#.
#. Status: work in progress, incomplete
#
function setup_ceph() {
node_ips=$1
cluster_net=$2
public_net=$3
ceph_dev=$4
echo "${FUNCNAME[0]}: Deploying ceph-mon on localhost $HOSTNAME"
echo "${FUNCNAME[0]}: Deploying ceph-osd on nodes $node_ips"
echo "${FUNCNAME[0]}: Setting cluster-network=$cluster_net and public-network=$public_net"
mon_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}')
all_nodes="$mon_ip $node_ips"
# Also caches the server fingerprints so ceph-deploy does not prompt the user
# Note this loop may be partially redundant with the ceph-deploy steps below
for node_ip in $all_nodes; do
echo "${FUNCNAME[0]}: Install ntp and ceph on $node_ip"
ssh -x -o StrictHostKeyChecking=no ubuntu@$node_ip <<EOF
sudo timedatectl set-ntp no
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
echo deb https://download.ceph.com/debian/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
sudo apt update
sudo apt-get install -y ntp ceph ceph-deploy
EOF
done
# per http://docs.ceph.com/docs/master/start/quick-ceph-deploy/
# also https://upcommons.upc.edu/bitstream/handle/2117/101816/Degree_Thesis_Nabil_El_Alami.pdf#vote +1
echo "${FUNCNAME[0]}: Create ceph config folder ~/ceph-cluster"
mkdir ~/ceph-cluster
cd ~/ceph-cluster
echo "${FUNCNAME[0]}: Create new cluster with $HOSTNAME as initial ceph-mon node"
ceph-deploy new --cluster-network $cluster_net --public-network $public_net --no-ssh-copykey $HOSTNAME
# Update conf per recommendations of http://docs.ceph.com/docs/jewel/rados/configuration/filesystem-recommendations/
cat <<EOF >>ceph.conf
osd max object name len = 256
osd max object namespace len = 64
EOF
cat ceph.conf
echo "${FUNCNAME[0]}: Deploy ceph packages on other nodes"
ceph-deploy install $mon_ip $node_ips
echo "${FUNCNAME[0]}: Deploy the initial monitor and gather the keys"
ceph-deploy mon create-initial
if [[ "x$ceph_dev" == "x" ]]; then
n=1
for node_ip in $node_ips; do
echo "${FUNCNAME[0]}: Prepare ceph OSD on node $node_ip"
echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
# Using ceph-osd$n here avoids need for manual acceptance of the new server hash
ssh -x -o StrictHostKeyChecking=no ubuntu@ceph-osd$n <<EOF
echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts
sudo mkdir /ceph && sudo chown -R ceph:ceph /ceph
EOF
ceph-deploy osd prepare ceph-osd$n:/ceph
ceph-deploy osd activate ceph-osd$n:/ceph
((n++))
done
else
echo "${FUNCNAME[0]}: Deploy OSDs"
for node_ip in $node_ips; do
echo "${FUNCNAME[0]}: Create ceph osd on $node_ip using $ceph_dev"
ceph-deploy osd create $node_ip:$ceph_dev
done
fi
echo "${FUNCNAME[0]}: Copy the config file and admin key to the admin node and OSD nodes"
ceph-deploy admin $mon_ip $node_ips
echo "${FUNCNAME[0]}: Check the cluster health"
sudo ceph health
sudo ceph -s
# per https://crondev.com/kubernetes-persistent-storage-ceph/ and https://github.com/kubernetes/kubernetes/issues/38923
# rbd is not included in default kube-controller-manager... use attcomdev version
sudo sed -i -- 's~gcr.io/google_containers/kube-controller-manager-amd64:.*~quay.io/attcomdev/kube-controller-manager:v1.7.3~' /etc/kubernetes/manifests/kube-controller-manager.yaml
if [[ $(sudo grep -c attcomdev/kube-controller-manager /etc/kubernetes/manifests/kube-controller-manager.yaml) == 0 ]]; then
echo "${FUNCNAME[0]}: Problem patching /etc/kubernetes/manifests/kube-controller-manager.yaml... script update needed"
exit 1
fi
mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
while [[ "$mgr" != "Running" ]]; do
echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr. Waiting 60 seconds for it to be 'Running'"
sleep 60
mgr=$(kubectl get pods --all-namespaces | grep kube-controller-manager | awk '{print $4}')
done
echo "${FUNCNAME[0]}: kube-controller-manager status is $mgr"
echo "${FUNCNAME[0]}: Create Ceph admin secret"
admin_key=$(sudo ceph auth get-key client.admin)
kubectl create secret generic ceph-secret-admin --from-literal=key="$admin_key" --namespace=kube-system --type=kubernetes.io/rbd
echo "${FUNCNAME[0]}: Create rdb storageClass 'general'"
cat <<EOF >/tmp/ceph-sc.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: general
provisioner: kubernetes.io/rbd
parameters:
monitors: $mon_ip:6789
adminId: admin
adminSecretName: ceph-secret-admin
adminSecretNamespace: "kube-system"
pool: kube
userId: kube
userSecretName: ceph-secret-user
EOF
# TODO: find out where in the above ~/.kube folders became owned by root
sudo chown -R ubuntu:ubuntu ~/.kube/*
kubectl create -f /tmp/ceph-sc.yaml
echo "${FUNCNAME[0]}: Create storage pool 'kube'"
# https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md method
sudo ceph osd pool create kube 32 32
echo "${FUNCNAME[0]}: Authorize client 'kube' access to pool 'kube'"
sudo ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube'
echo "${FUNCNAME[0]}: Create ceph-secret-user secret in namespace 'default'"
kube_key=$(sudo ceph auth get-key client.kube)
kubectl create secret generic ceph-secret-user --from-literal=key="$kube_key" --namespace=default --type=kubernetes.io/rbd
# A similar secret must be created in other namespaces that intend to access the ceph pool
# Per https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md
echo "${FUNCNAME[0]}: Create andtest a persistentVolumeClaim"
cat <<EOF >/tmp/ceph-pvc.yaml
{
"kind": "PersistentVolumeClaim",
"apiVersion": "v1",
"metadata": {
"name": "claim1",
"annotations": {
"volume.beta.kubernetes.io/storage-class": "general"
}
},
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "3Gi"
}
}
}
}
EOF
kubectl create -f /tmp/ceph-pvc.yaml
while [[ "x$(kubectl get pvc -o jsonpath='{.status.phase}' claim1)" != "xBound" ]]; do
echo "${FUNCNAME[0]}: Waiting for pvc claim1 to be 'Bound'"
kubectl describe pvc
sleep 10
done
echo "${FUNCNAME[0]}: pvc claim1 successfully bound to $(kubectl get pvc -o jsonpath='{.spec.volumeName}' claim1)"
kubectl get pvc
kubectl delete pvc claim1
kubectl describe pods
}
if [[ "$1" != "" ]]; then
setup_ceph "$1" $2 $3 $4
else
grep '#. ' $0
fi
|
