diff options
-rw-r--r-- | tools/cloudify/k8s-cloudify.sh | 8 | ||||
-rw-r--r-- | tools/kubernetes/demo_deploy.sh | 69 | ||||
-rw-r--r-- | tools/kubernetes/k8s-cluster.sh | 149 |
3 files changed, 156 insertions, 70 deletions
diff --git a/tools/cloudify/k8s-cloudify.sh b/tools/cloudify/k8s-cloudify.sh index 36abb63..8f52f88 100644 --- a/tools/cloudify/k8s-cloudify.sh +++ b/tools/cloudify/k8s-cloudify.sh @@ -9,7 +9,6 @@ # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # @@ -19,6 +18,7 @@ #. Usage: #. From a server with access to the kubernetes master node: #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ cd models/tools/cloudify #. $ scp -r ~/models/tools/cloudify ubuntu@<k8s-master>:/home/ubuntu/. #. <k8s-master>: IP or hostname of kubernetes master server #. $ ssh -x ubuntu@<k8s-master> cloudify/k8s-cloudify.sh prereqs @@ -51,8 +51,7 @@ function fail() { function log() { f=$(caller 0 | awk '{print $2}') l=$(caller 0 | awk '{print $1}') - echo "" - echo "$f:$l ($(date)) $1" + echo; echo "$f:$l ($(date)) $1" } function prereqs() { @@ -178,7 +177,7 @@ function service_port() { sleep 10 ((tries--)) done - if [[ "$port" == "" ]]; then + if [[ "$port" == "null" ]]; then jq -r '.items' /tmp/json fail "node_port not found for service" fi @@ -283,6 +282,7 @@ function stop() { -d "{\"deployment_id\":\"$bp\", \"workflow_id\":\"uninstall\"}" \ -o /tmp/json http://$manager_ip/api/v3.1/executions id=$(jq -r ".id" /tmp/json) + log "uninstall execution id = $id" status="" tries=1 while [[ "$status" != "terminated" && $tries -lt 10 ]]; do diff --git a/tools/kubernetes/demo_deploy.sh b/tools/kubernetes/demo_deploy.sh index 7922434..c585a3e 100644 --- a/tools/kubernetes/demo_deploy.sh +++ b/tools/kubernetes/demo_deploy.sh @@ -34,7 +34,7 @@ #. <key>: name of private key for cluster node ssh (in current folder) #. <hosts>: space separated list of hostnames managed by MAAS #. <master>: IP of cluster master node -#. <workers>: space separated list of agent node IPs +#. <workers>: space separated list of worker node IPs #. <pub-net>: CID formatted public network #. <priv-net>: CIDR formatted private network (may be same as pub-net) #. <ceph-mode>: "helm" or "baremetal" @@ -51,43 +51,56 @@ ceph_mode=$7 ceph_dev=$8 extras=$9 +function run_master() { +ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$master <<EOF +exec ssh-agent bash +ssh-add $key +$1 +EOF +} + source ~/models/tools/maas/deploy.sh $1 "$2" $9 eval `ssh-agent` ssh-add $key -if [[ "x$extras" != "x" ]]; then source $extras; fi scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $key ubuntu@$master:/home/ubuntu/$key -echo "$0 $(date): Setting up kubernetes..." + +echo; echo "$0 $(date): Setting up kubernetes master..." scp -r -o StrictHostKeyChecking=no ~/models/tools/kubernetes/* \ ubuntu@$master:/home/ubuntu/. -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master <<EOF -exec ssh-agent bash -ssh-add $key -bash k8s-cluster.sh all "$workers" $priv_net $pub_net $ceph_mode $ceph_dev -EOF -# TODO: Figure this out... Have to break the setup into two steps as something -# causes the ssh session to end before the prometheus setup, if both scripts -# are in the same ssh session -echo "Setting up Prometheus..." -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master mkdir -p \ - /home/ubuntu/models/tools/prometheus +run_master "bash k8s-cluster.sh master" + +echo; echo "$0 $(date): Setting up kubernetes workers..." +run_master "bash k8s-cluster.sh workers \"$workers\"" + +echo; echo "$0 $(date): Setting up helm..." +run_master "bash k8s-cluster.sh helm" + +echo; echo "$0 $(date): Verifying kubernetes+helm install..." +run_master "bash k8s-cluster.sh demo start nginx" +run_master "bash k8s-cluster.sh demo stop nginx" + +echo; echo "$0 $(date): Setting up ceph-helm" +run_master "bash k8s-cluster.sh ceph \"$workers\" $priv_net $pub_net $ceph_mode $ceph_dev" + +echo; echo "$0 $(date): Verifying kubernetes+helm+ceph install..." +run_master "bash k8s-cluster.sh demo start dokuwiki" + +echo; echo "Setting up Prometheus..." scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \ - ubuntu@$master:/home/ubuntu/models/tools/prometheus -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master <<EOF -exec ssh-agent bash -ssh-add $key -cd models/tools/prometheus -bash prometheus-tools.sh all "$workers" -EOF -echo "$0 $(date): Setting up cloudify..." + ubuntu@$master:/home/ubuntu/. +run_master "bash prometheus-tools.sh all \"$workers\"" + +echo; echo "$0 $(date): Setting up cloudify..." scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \ ubuntu@$master:/home/ubuntu/. -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \ - bash cloudify/k8s-cloudify.sh prereqs -ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master \ - bash cloudify/k8s-cloudify.sh setup -source ~/models/tools/cloudify/k8s-cloudify.sh demo start $master +run_master "bash cloudify/k8s-cloudify.sh prereqs" +run_master "bash cloudify/k8s-cloudify.sh setup" + +echo; echo "$0 $(date): Verifying kubernetes+helm+ceph+cloudify install..." +bash ~/models/tools/cloudify/k8s-cloudify.sh demo start $master -echo "$0 $(date): All done!" +echo; echo "$0 $(date): All done!" export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/" diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index 54c5973..817caaf 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -15,14 +15,14 @@ # #. What this is: script to setup a kubernetes cluster with calico as sni #. Prerequisites: -#. - Ubuntu xenial server for master and agent nodes -#. - key-based auth setup for ssh/scp between master and agent nodes +#. - Ubuntu xenial server for master and worker nodes +#. - key-based auth setup for ssh/scp between master and worker nodes #. - 192.168.0.0/16 should not be used on your server network interface subnets #. Usage: #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models #. $ cd ~/models/tools/kubernetes #. $ bash k8s-cluster.sh master -#. $ bash k8s-cluster.sh agents "<nodes>" +#. $ bash k8s-cluster.sh workers "<nodes>" #. nodes: space-separated list of ceph node IPs #. $ bash k8s-cluster.sh ceph "<nodes>" <cluster-net> <public-net> <ceph-mode> [ceph_dev] #. nodes: space-separated list of ceph node IPs @@ -44,10 +44,15 @@ #. Status: work in progress, incomplete # +function fail() { + log $1 + exit 1 +} + function log() { f=$(caller 0 | awk '{print $2}') l=$(caller 0 | awk '{print $1}') - echo "$f:$l ($(date)) $1" + echo; echo "$f:$l ($(date)) $1" } function setup_prereqs() { @@ -55,16 +60,21 @@ function setup_prereqs() { cat <<'EOG' >/tmp/prereqs.sh #!/bin/bash # Basic server pre-reqs -sudo apt-get -y remove kubectl kubelet kubeadm +echo; echo "prereqs.sh: ($(date)) Basic prerequisites" sudo apt-get update sudo apt-get upgrade -y -# Set hostname if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then + echo; echo "prereqs.sh: ($(date)) Add $HOSTNAME to /etc/hosts" echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" | sudo tee -a /etc/hosts fi -# Install docker 1.12 (default for xenial is 1.12.6) +echo; echo "prereqs.sh: ($(date)) Install latest docker" sudo apt-get install -y docker.io -sudo service docker start +# Alternate for 1.12.6 +#sudo apt-get install -y libltdl7 +#wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb +#sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb +sudo service docker restart +echo; echo "prereqs.sh: ($(date)) Get k8s packages" export KUBE_VERSION=1.7.5 # per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ # Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/ @@ -74,11 +84,31 @@ cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list deb http://apt.kubernetes.io/ kubernetes-xenial main EOF sudo apt-get update -# Next command is to workaround bug resulting in "PersistentVolumeClaim is not bound" for pod startup (remain in Pending) -# TODO: reverify if this is still an issue in the final working script +echo; echo "prereqs.sh: ($(date)) Install kubectl, kubelet, kubeadm" sudo apt-get -y install --allow-downgrades kubectl=${KUBE_VERSION}-00 kubelet=${KUBE_VERSION}-00 kubeadm=${KUBE_VERSION}-00 -# Needed for API output parsing +echo; echo "prereqs.sh: ($(date)) Install jq for API output parsing" sudo apt-get -y install jq +echo; echo "prereqs.sh: ($(date)) Set firewall rules" +# Per https://kubernetes.io/docs/setup/independent/install-kubeadm/ +if [[ "$(sudo ufw status)" == "Status: active" ]]; then + if [[ "$1" == "master" ]]; then + sudo ufw allow 6443/tcp + sudo ufw allow 2379:2380/tcp + sudo ufw allow 10250/tcp + sudo ufw allow 10251/tcp + sudo ufw allow 10252/tcp + sudo ufw allow 10255/tcp + else + sudo ufw allow 10250/tcp + sudo ufw allow 10255/tcp + sudo ufw allow 30000:32767/tcp + fi +fi +# TODO: fix need for this workaround: disable firewall since the commands +# above do not appear to open the needed ports, even if ufw is inactive +# (symptom: nodeport requests fail unless sent from within the cluster or +# to the node IP where the pod is assigned) issue discovered ~11/16/17 +sudo ufw disable EOG } @@ -103,13 +133,15 @@ function setup_k8s_master() { sudo chown $(id -u):$(id -g) $HOME/.kube/config # Deploy pod network log "Deploy calico as CNI" - sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + # Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above + # sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + sudo kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml } -function setup_k8s_agents() { - agents="$1" +function setup_k8s_workers() { + workers="$1" export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) - log "Installing agents at $1 with joincmd: $k8s_joincmd" + log "Installing workers at $1 with joincmd: $k8s_joincmd" setup_prereqs @@ -121,13 +153,15 @@ function setup_k8s_agents() { done log "kube-dns status is $kubedns" - for agent in $agents; do - log "Install agent at $agent" - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$agent:/tmp/prereqs.sh - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent bash /tmp/prereqs.sh agent + for worker in $workers; do + log "Install worker at $worker" + if ! scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$worker:/tmp/prereqs.sh ; then + fail "Failed copying setup files to $worker" + fi + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker bash /tmp/prereqs.sh worker # Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty" per https://github.com/kubernetes/kubeadm/issues/1 - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo kubeadm reset - ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo $k8s_joincmd + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo kubeadm reset + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker sudo $k8s_joincmd done log "Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'." @@ -139,29 +173,56 @@ function wait_for_service() { log "Service $1 is at pod $pod" ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) while [[ "$ready" != "true" ]]; do - log "$1 container is not yet ready... waiting 10 seconds" + log "pod $1 is not yet ready... waiting 10 seconds" sleep 10 # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) done - log "pod $pod container status is $ready" + log "pod $pod is ready" host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod) port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1) - log "pod $pod container is at host $host_ip and port $port" - while ! curl http://$host_ip:$port ; do - log "$1 service is not yet responding... waiting 10 seconds" - sleep 10 + log "$pod pod is running on assigned node $host_ip" + log "$1 service is assigned node_port $port" + log "verify $1 service is accessible via all workers at node_port $port" + nodes=$(kubectl get nodes | awk '/Ready/ {print $1}') + for node in $nodes; do + ip=$(kubectl describe nodes $node | awk '/InternalIP/ { print $2}') + while ! curl http://$ip:$port ; do + log "$1 service is not yet responding at worker $node IP $ip... waiting 10 seconds" + sleep 10 + done + log "$1 service is accessible at worker $node at http://$ip:$port" done - log "$1 is available at http://$host_ip:$port" } -function demo_chart() { - cd ~ - rm -rf charts - git clone https://github.com/kubernetes/charts.git - cd charts/stable +function stop_chart() { + log "stop chart $1" + service=$(kubectl get services --namespace default | awk "/$1/ {print \$1}") + kubectl delete services --namespace default $service + secret=$(kubectl get secrets --namespace default | awk "/$1/ {print \$1}") + kubectl delete secrets --namespace default $secret + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + kubectl delete pods --namespace default $pod + release=$(echo $service | cut -d '-' -f 1) + helm del --purge $release + job=$(kubectl get jobs --namespace default | awk "/$1/ {print \$1}") + kubectl delete jobs --namespace default $job +} + +function start_chart() { + rm -rf /tmp/git/charts + git clone https://github.com/kubernetes/charts.git /tmp/git/charts + cd /tmp/git/charts/stable case "$1" in + nginx) + rm -rf /tmp/git/helm + git clone https://github.com/kubernetes/helm.git /tmp/git/helm + cd /tmp/git/helm/docs/examples + sed -i -- 's/type: ClusterIP/type: NodePort/' ./nginx/values.yaml + helm install --name nx -f ./nginx/values.yaml ./nginx + wait_for_service nx-nginx + ;; mediawiki) # NOT YET WORKING # mariadb: Readiness probe failed: mysqladmin: connect to server at 'localhost' failed @@ -265,13 +326,19 @@ function setup_ceph() { fi } +workers="$2" +privnet=$3 +pubnet=$4 +ceph_mode=$5 +ceph_dev=$6 + export WORK_DIR=$(pwd) case "$1" in master) setup_k8s_master ;; - agents) - setup_k8s_agents "$2" + workers) + setup_k8s_workers "$2" ;; ceph) setup_ceph "$2" $3 $4 $5 $6 @@ -280,14 +347,20 @@ case "$1" in setup_helm ;; demo) - demo_chart $2 + if [[ "$2" == "start" ]]; then + start_chart $3 + else + stop_chart $3 + fi ;; all) setup_k8s_master - setup_k8s_agents "$2" + setup_k8s_workers "$2" setup_helm + start_chart nginx + stop_chart nginx setup_ceph "$2" $3 $4 $5 $6 - demo_chart dokuwiki + start_chart dokuwiki ;; clean) # TODO |