From 0ed35881347dc09ede49d1520f4870a326bf640e Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Mon, 4 Dec 2017 16:19:31 -0800 Subject: Add centos support JIRA: MODELS-2 Reverified ubuntu still works Change-Id: I388238b70306cd9c6989d11c09dddcf809d081a1 Signed-off-by: Bryan Sullivan --- tools/kubernetes/ceph-helm.sh | 93 +++++++++++++++-- tools/kubernetes/demo_deploy.sh | 58 ++++++----- tools/kubernetes/k8s-cluster.sh | 226 ++++++++++++++++++++++++++++------------ tools/maas/deploy.sh | 21 ++-- 4 files changed, 288 insertions(+), 110 deletions(-) diff --git a/tools/kubernetes/ceph-helm.sh b/tools/kubernetes/ceph-helm.sh index a1be588..084a4f7 100644 --- a/tools/kubernetes/ceph-helm.sh +++ b/tools/kubernetes/ceph-helm.sh @@ -37,21 +37,75 @@ function log() { echo "$f:$l ($(date)) $1" } +function make_ceph_setup() { + tee ~/ceph_setup.sh <<'EOG' +#!/bin/bash +# Basic server pre-reqs +dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g') +if [[ "$dist" == "ubuntu" ]]; then + sudo apt-get install -y ceph ceph-common +else + # per http://docs.ceph.com/docs/master/install/get-packages/ + sudo tee /etc/yum.repos.d/ceph.repo <<'EOF' +[ceph] +name=Ceph packages for $basearch +baseurl=https://download.ceph.com/rpm-luminous/el7/x86_64 +enabled=1 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc + +[ceph-noarch] +name=Ceph noarch packages +baseurl=https://download.ceph.com/rpm-luminous/el7/noarch +enabled=1 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc + +[ceph-source] +name=Ceph source packages +baseurl=https://download.ceph.com/rpm-luminous/el7/SRPMS +enabled=0 +priority=2 +gpgcheck=1 +gpgkey=https://download.ceph.com/keys/release.asc +EOF + # TODO: find out why package us unsigned and thus need --nogpgcheck + sudo rpm --import 'https://download.ceph.com/keys/release.asc' + sudo yum install --nogpgcheck -y ceph ceph-common +fi +EOG +} + function setup_ceph() { nodes=$1 private_net=$2 public_net=$3 dev=$4 - - log "Install ceph prerequisites" - sudo apt-get -y install ceph ceph-common + log "Install ceph and ceph-common" + make_ceph_setup + bash ~/ceph_setup.sh + # per https://github.com/att/netarbiter/tree/master/sds/ceph-docker/examples/helm log "Clone netarbiter" git clone https://github.com/att/netarbiter.git - cd netarbiter/sds/ceph-docker/examples/helm + + if [[ "$dist" != "ubuntu" ]]; then + log "Update ceph-helm chart to point to centos images" + sed -i -- 's~daemon: docker.io/knowpd~#daemon: docker.io/knowpd~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~#daemon: docker.io/ceph~daemon: docker.io/ceph~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~ceph_init: docker.io/knowpd~#ceph_init: docker.io/knowpd~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + sed -i -- 's~#ceph_init: docker.io/kollakube~ceph_init: docker.io/kollakube~' \ + netarbiter/sds/ceph-docker/examples/helm/ceph/values.yaml + fi log "Prepare a ceph namespace in your K8s cluster" + cd netarbiter/sds/ceph-docker/examples/helm ./prep-ceph-ns.sh log "Run ceph-mon, ceph-mgr, ceph-mon-check, and rbd-provisioner" @@ -93,24 +147,45 @@ EOF for node in $nodes; do log "install ceph, setup resolv.conf, zap disk for $node" - ssh -x -o StrictHostKeyChecking=no ubuntu@$node < "" -#. "" [] -#. : name of private key for cluster node ssh (in current folder) +#. $ bash ~/models/tools/kubernetes/demo_deploy.sh "" +#. "" [] #. : space separated list of hostnames managed by MAAS +#. : OS to deploy, one of "ubuntu" (Xenial) or "centos" (Centos 7) +#. : name of private key for cluster node ssh (in current folder) #. : IP of cluster master node #. : space separated list of worker node IPs #. : CID formatted public network @@ -58,7 +58,7 @@ function step_end() { function run_master() { start=$((`date +%s`/60)) ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ubuntu@$k8s_master <~/k8s_env.sh -k8s_key=$1 -k8s_nodes="$2" -k8s_master=$3 -k8s_workers="$4" -k8s_priv_net=$5 -k8s_pub_net=$6 -k8s_ceph_mode=$7 -k8s_ceph_dev=$8 -export k8s_key +k8s_nodes="$1" +k8s_user=$2 +k8s_key=$3 +k8s_master=$4 +k8s_workers="$5" +k8s_priv_net=$6 +k8s_pub_net=$7 +k8s_ceph_mode=$8 +k8s_ceph_dev=$9 export k8s_nodes +export k8s_user +export k8s_key export k8s_master export k8s_workers export k8s_priv_net @@ -89,17 +92,17 @@ EOF source ~/k8s_env.sh env | grep k8s_ -source ~/models/tools/maas/deploy.sh $k8s_key "$k8s_nodes" $extras +source ~/models/tools/maas/deploy.sh $k8s_user $k8s_key "$k8s_nodes" $extras eval `ssh-agent` ssh-add $k8s_key scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_key \ - ubuntu@$k8s_master:/home/ubuntu/$k8s_key + $k8s_user@$k8s_master:/home/$k8s_user/$k8s_key scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ~/k8s_env.sh \ - ubuntu@$k8s_master:/home/ubuntu/. + $k8s_user@$k8s_master:/home/$k8s_user/. echo; echo "$0 $(date): Setting up kubernetes master..." scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - ~/models/tools/kubernetes/* ubuntu@$k8s_master:/home/ubuntu/. + ~/models/tools/kubernetes/* $k8s_user@$k8s_master:/home/$k8s_user/. run_master "bash k8s-cluster.sh master" echo; echo "$0 $(date): Setting up kubernetes workers..." @@ -120,12 +123,12 @@ run_master "bash k8s-cluster.sh demo start dokuwiki" echo; echo "Setting up Prometheus..." scp -r -o StrictHostKeyChecking=no ~/models/tools/prometheus/* \ - ubuntu@$k8s_master:/home/ubuntu/. + $k8s_user@$k8s_master:/home/$k8s_user/. run_master "bash prometheus-tools.sh all \"$k8s_workers\"" echo; echo "$0 $(date): Setting up cloudify..." scp -r -o StrictHostKeyChecking=no ~/models/tools/cloudify \ - ubuntu@$k8s_master:/home/ubuntu/. + $k8s_user@$k8s_master:/home/$k8s_user/. run_master "bash cloudify/k8s-cloudify.sh prereqs" run_master "bash cloudify/k8s-cloudify.sh setup" @@ -135,6 +138,7 @@ run "bash $HOME/models/tools/cloudify/k8s-cloudify.sh demo start" echo; echo "$0 $(date): Setting up VES" # not re-cloned if existing - allows patch testing locally if [[ ! -d ~/ves ]]; then + echo; echo "$0 $(date): Cloning VES" git clone https://gerrit.opnfv.org/gerrit/ves ~/ves fi ves_influxdb_host=$k8s_master:8086 @@ -143,18 +147,18 @@ ves_grafana_host=$k8s_master:3000 export ves_grafana_host ves_grafana_auth=admin:admin export ves_grafana_auth -ves_kafka_hostname=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master hostname) +ves_kafka_hostname=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master hostname) export ves_kafka_hostname ves_loglevel=$ves_loglevel export ves_loglevel # Can't pass quoted strings in commands start=$((`date +%s`/60)) -bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_master "$k8s_workers" -step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_master \"$k8s_workers\"" +bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_user $k8s_master "$k8s_workers" +step_end "bash $HOME/ves/tools/demo_deploy.sh $k8s_key $k8s_user $k8s_master \"$k8s_workers\"" echo; echo "$0 $(date): All done!" -export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) -export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") +export NODE_PORT=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services dw-dokuwiki) +export NODE_IP=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $k8s_user@$k8s_master kubectl get nodes --namespace default -o jsonpath="{.items[0].status.addresses[0].address}") echo "Helm chart demo app dokuwiki is available at http://$NODE_IP:$NODE_PORT/" # TODO update Cloudify demo app to have public exposed service address port=$( bash ~/models/tools/cloudify/k8s-cloudify.sh port nginx $k8s_master) diff --git a/tools/kubernetes/k8s-cluster.sh b/tools/kubernetes/k8s-cluster.sh index 1700a6a..cf1e840 100644 --- a/tools/kubernetes/k8s-cluster.sh +++ b/tools/kubernetes/k8s-cluster.sh @@ -15,7 +15,7 @@ # #. What this is: script to setup a kubernetes cluster with calico as sni #. Prerequisites: -#. - Ubuntu xenial server for master and worker nodes +#. - Ubuntu Xenial or Centos 7 server for master and worker nodes #. - key-based auth setup for ssh/scp between master and worker nodes #. - 192.168.0.0/16 should not be used on your server network interface subnets #. Usage: @@ -54,64 +54,103 @@ function log() { f=$(caller 0 | awk '{print $2}') l=$(caller 0 | awk '{print $1}') echo; echo "$f:$l ($(date)) $1" + kubectl get pods --all-namespaces } function setup_prereqs() { log "Create prerequisite setup script" - cat <<'EOG' >/tmp/prereqs.sh + cat <<'EOG' >~/prereqs.sh #!/bin/bash # Basic server pre-reqs -echo; echo "prereqs.sh: ($(date)) Basic prerequisites" -sudo apt-get update -sudo apt-get upgrade -y +dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g') if [[ $(grep -c $HOSTNAME /etc/hosts) -eq 0 ]]; then echo; echo "prereqs.sh: ($(date)) Add $HOSTNAME to /etc/hosts" - echo "$(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" \ + # have to add "/sbin" to path of IP command for centos + echo "$(/sbin/ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME" \ | sudo tee -a /etc/hosts fi -echo; echo "prereqs.sh: ($(date)) Install latest docker" -sudo apt-get install -y docker.io -# Alternate for 1.12.6 -#sudo apt-get install -y libltdl7 -#wget https://packages.docker.com/1.12/apt/repo/pool/main/d/docker-engine/docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb -#sudo dpkg -i docker-engine_1.12.6~cs8-0~ubuntu-xenial_amd64.deb -sudo service docker restart -echo; echo "prereqs.sh: ($(date)) Get k8s packages" -export KUBE_VERSION=1.7.5 -# per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ -# Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/ -sudo apt-get update && sudo apt-get install -y apt-transport-https -curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -cat <>/tmp/kubeadm.out cat /tmp/kubeadm.out export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) log "Cluster join command for manual use if needed: $k8s_joincmd" - - # Start cluster - log "Start the cluster" mkdir -p $HOME/.kube sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config # Deploy pod network log "Deploy calico as CNI" # Updated to deploy Calico 2.6 per the create-cluster-kubeadm guide above - # sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml - sudo kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + # kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + kubectl apply -f https://docs.projectcalico.org/v2.6/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml + + # TODO: document process dependency + # Failure to wait for all calico pods to be running can cause the first worker + # to be incompletely setup. Symptom is that node_ports cannot be routed + # via that node (no response - incoming SYN packets are dropped). + log "Wait for calico pods to be Running" + # calico-etcd, calico-kube-controllers, calico-node + pods=$(kubectl get pods --namespace kube-system | grep -c calico) + while [[ $pods -lt 3 ]]; do + log "all calico pods are not yet created. Waiting 10 seconds" + sleep 10 + pods=$(kubectl get pods --namespace kube-system | grep -c calico) + done + + pods=$(kubectl get pods --all-namespaces | awk '/calico/ {print $2}') + for pod in $pods; do + status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}") + while [[ "$status" != "Running" ]]; do + log "$pod status is $status. Waiting 10 seconds" + sleep 10 + status=$(kubectl get pods --all-namespaces | awk "/$pod/ {print \$4}") + done + log "$pod status is $status" + done log "Wait for kubedns to be Running" - kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}') + kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}') while [[ "$kubedns" != "Running" ]]; do - log "kube-dns status is $kubedns. Waiting 60 seconds for it to be 'Running'" + log "kube-dns status is $kubedns. Waiting 60 seconds" sleep 60 - kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}') + kubedns=$(kubectl get pods --all-namespaces | awk '/kube-dns/ {print $4}') done log "kube-dns status is $kubedns" } @@ -156,35 +220,61 @@ function setup_k8s_workers() { export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) log "Installing workers at $1 with joincmd: $k8s_joincmd" +# TODO: kubeadm reset below is workaround for +# Ubuntu: "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty" +# per https://github.com/kubernetes/kubeadm/issues/1 +# Centos: "Failed to start ContainerManager failed to initialize top +# level QOS containers: root container /kubepods doesn't exist" + tee start_worker.sh < /dev/null 2>&1 & -bash /tmp/prereqs.sh worker -# Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet -# is not empty" per https://github.com/kubernetes/kubeadm/issues/1 -sudo kubeadm reset -sudo $k8s_joincmd -EOF + $USER@$worker bash start_worker.sh done for worker in $workers; do - host=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$worker hostname) - log "checking node $host" + host=$(ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no $USER@$worker hostname) + log "checking that node $host is 'Ready'" status=$(kubectl get nodes | awk "/$host/ {print \$2}") while [[ "$status" != "Ready" ]]; do - log "node $host is \"$status\", waiting 10 seconds for it to be 'Ready'." + log "node $host is \"$status\", waiting 10 seconds" status=$(kubectl get nodes | awk "/$host/ {print \$2}") + ((tries++)) + if [[ tries -gt 18 ]]; then + log "node $host is \"$status\" after 3 minutes; resetting kubeadm" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + $USER@$worker bash start_worker.sh + tries=1 + fi sleep 10 done log "node $host is 'Ready'." done + + log "***** kube proxy pods *****" + pods=$(kubectl get pods --all-namespaces | awk '/kube-proxy/ {print $2}') + for pod in $pods; do + echo; echo "**** $pod ****" + kubectl describe pods --namespace kube-system $pod + echo; echo "**** $pod logs ****" + kubectl logs --namespace kube-system $pod + done + log "Cluster is ready (all nodes in 'kubectl get nodes' show as 'Ready')." } @@ -196,6 +286,8 @@ function setup_ceph() { fi } +dist=$(grep --m 1 ID /etc/os-release | awk -F '=' '{print $2}' | sed 's/"//g') + workers="$2" privnet=$3 pubnet=$4 diff --git a/tools/maas/deploy.sh b/tools/maas/deploy.sh index 55984da..484a389 100644 --- a/tools/maas/deploy.sh +++ b/tools/maas/deploy.sh @@ -18,13 +18,15 @@ #. #. Prerequisites: #. - MAAS server configured to admin a set of servers +#. - User is logged into the MAAS server e.g. via maas login opnfv #. - Password-less ssh key provided for node setup #. Usage: on the MAAS server #. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models -#. $ source ~/models/tools/maas/demo_deploy.sh "" [] +#. $ source ~/models/tools/maas/demo_deploy.sh "" [] +#. : "xenial" (Ubtuntu Xenial) or "centos" (Centos 7) #. : name of private key for cluster node ssh (in current folder) #. : space separated list of hostnames managed by MAAS -#. : optional name of script for extra setup functions as needed +#. : optional name and parameters of script for extra setup functions function log() { f=$(caller 0 | awk '{print $2}') @@ -57,7 +59,11 @@ function deploy_nodes() { log "Deploying node $node" id=$(maas opnfv machines read hostname=$node | jq -r '.[0].system_id') maas opnfv machines allocate system_id=$id - maas opnfv machine deploy $id + if [[ "$os" == "ubuntu" ]]; then + maas opnfv machine deploy $id + else + maas opnfv machine deploy $id distro_series=$os hwe_kernel=generic + fi done } @@ -68,9 +74,10 @@ function wait_nodes_status() { done } -key=$1 -nodes="$2" -extras=$3 +os=$1 +key=$2 +nodes="$3" +extras="$4" release_nodes "$nodes" wait_nodes_status "$nodes" Ready @@ -78,4 +85,4 @@ deploy_nodes "$nodes" wait_nodes_status "$nodes" Deployed eval `ssh-agent` ssh-add $key -if [[ "x$extras" != "x" ]]; then source $extras; fi +if [[ "x$extras" != "x" ]]; then source $extras $5 $6 $7 $8 $9; fi -- cgit 1.2.3-korg