diff options
Diffstat (limited to 'src/vagrant/kubeadm')
23 files changed, 1166 insertions, 0 deletions
diff --git a/src/vagrant/kubeadm/Vagrantfile b/src/vagrant/kubeadm/Vagrantfile new file mode 100644 index 0000000..dc5efb1 --- /dev/null +++ b/src/vagrant/kubeadm/Vagrantfile @@ -0,0 +1,34 @@ +$num_workers=2 + +Vagrant.require_version ">= 1.8.6" +Vagrant.configure("2") do |config| + + config.vm.box = "ceph/ubuntu-xenial" + config.vm.provider :libvirt do |libvirt| + libvirt.memory = 4096 + libvirt.cpus = 4 + end + + config.vm.define "registry" do |config| + config.vm.hostname = "registry" + #config.vm.provision "shell", path: "registry_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.5" + end + + config.vm.define "master" do |config| + config.vm.hostname = "master" + config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.provision "shell", path: "master_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.10" + end + + (1 .. $num_workers).each do |i| + config.vm.define vm_name = "worker%d" % [i] do |config| + config.vm.hostname = vm_name + config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.provision "shell", path: "worker_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.#{i+20}" + end + end + +end diff --git a/src/vagrant/kubeadm/deploy.sh b/src/vagrant/kubeadm/deploy.sh new file mode 100755 index 0000000..eb61ad8 --- /dev/null +++ b/src/vagrant/kubeadm/deploy.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex +DIR="$(dirname `readlink -f $0`)" + +cd $DIR +../cleanup.sh +vagrant up +vagrant ssh master -c "/vagrant/kata/nginx-app.sh" +vagrant ssh master -c "/vagrant/virtlet/virtlet.sh" diff --git a/src/vagrant/kubeadm/host_setup.sh b/src/vagrant/kubeadm/host_setup.sh new file mode 100644 index 0000000..1cb46f6 --- /dev/null +++ b/src/vagrant/kubeadm/host_setup.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.5 registry +192.168.1.10 master +192.168.1.21 worker1 +192.168.1.22 worker2 +192.168.1.23 worker3 +EOF + +curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list +deb http://apt.kubernetes.io/ kubernetes-xenial main +EOF +sudo apt-get update +sudo apt-get install -y --allow-unauthenticated kubelet=1.12.2-00 kubeadm=1.12.2-00 kubectl=1.12.2-00 kubernetes-cni=0.6.0-00 +echo 'Environment="KUBELET_EXTRA_ARGS=--feature-gates=DevicePlugins=true"' | sudo tee /etc/default/kubelet +echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward +sudo modprobe ip_vs +sudo modprobe ip_vs_rr +sudo modprobe ip_vs_wrr +sudo modprobe ip_vs_sh +sudo modprobe br_netfilter +sudo modprobe nf_conntrack_ipv4 + +sudo swapoff -a +sudo systemctl daemon-reload +sudo systemctl stop kubelet +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm/istio/bookinfo.sh b/src/vagrant/kubeadm/istio/bookinfo.sh new file mode 100755 index 0000000..c4eef11 --- /dev/null +++ b/src/vagrant/kubeadm/istio/bookinfo.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# +# Copyright (c) 2018 Huawei Technologies Canada Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +cd /vagrant/istio-source/ +export PATH=$PWD/bin:$PATH + +# Run the test application: bookinfo +kubectl apply -f <(istioctl kube-inject -f samples/bookinfo/platform/kube/bookinfo.yaml) + +# Define the ingress gateway for the application +kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml + +# Wait for bookinfo deployed +kubectl get services +kubectl get pods + +r="0" +while [ $r -ne "6" ] +do + sleep 30 + kubectl get pods + r=$(kubectl get pods | grep Running | wc -l) +done + +# Validate the bookinfo app +export GATEWAY_URL=$(kubectl get po -l istio=ingressgateway -n istio-system -o 'jsonpath={.items[0].status.hostIP}'):$(kubectl get svc istio-ingressgateway -n istio-system -o 'jsonpath={.spec.ports[0].nodePort}') +curl -o /dev/null -s -w "%{http_code}\n" http://${GATEWAY_URL}/productpage + diff --git a/src/vagrant/kubeadm/istio/clean_bookinfo.sh b/src/vagrant/kubeadm/istio/clean_bookinfo.sh new file mode 100755 index 0000000..7c539c0 --- /dev/null +++ b/src/vagrant/kubeadm/istio/clean_bookinfo.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright (c) 2018 Huawei Technologies Canada Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +cd /vagrant/istio-source/ +export PATH=$PWD/bin:$PATH + +# Clean up bookinfo +echo "" | samples/bookinfo/platform/kube/cleanup.sh + +kubectl get virtualservices +kubectl get destinationrules +kubectl get gateway +kubectl get pods diff --git a/src/vagrant/kubeadm/istio/deploy.sh b/src/vagrant/kubeadm/istio/deploy.sh new file mode 100755 index 0000000..e896580 --- /dev/null +++ b/src/vagrant/kubeadm/istio/deploy.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (c) 2018 Huawei Technologies Canada Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +# Get latest istio version, refer: https://git.io/getLatestIstio +if [ "x${ISTIO_VERSION}" = "x" ] ; then + ISTIO_VERSION=$(curl -L -s https://api.github.com/repos/istio/istio/releases/latest | \ + grep tag_name | sed "s/ *\"tag_name\": *\"\(.*\)\",*/\1/") +fi + +ISTIO_DIR_NAME="istio-$ISTIO_VERSION" + +cd /vagrant +curl -L https://git.io/getLatestIstio | sh - +mv $ISTIO_DIR_NAME istio-source +cd /vagrant/istio-source/ + +# Persistently append istioctl bin path to PATH env +echo 'export PATH="$PATH:/vagrant/istio-source/bin"' >> ~/.bashrc +echo "source <(kubectl completion bash)" >> ~/.bashrc +source ~/.bashrc + +# Install Istio’s Custom Resource Definitions first +kubectl apply -f install/kubernetes/helm/istio/templates/crds.yaml + +# Wait 30s for Kubernetes to register the Istio CRDs +sleep 30 + +kubectl apply -f install/kubernetes/istio-demo.yaml + +# Validate the installation +kubectl get svc -n istio-system +kubectl get pods -n istio-system +kubectl get namespace -L istio-injection + +r="1" +while [ $r -ne "0" ] +do + sleep 30 + kubectl get pods -n istio-system + r=$(kubectl get pods -n istio-system | egrep -v 'NAME|Running|Completed' | wc -l) +done + diff --git a/src/vagrant/kubeadm/istio/istio.sh b/src/vagrant/kubeadm/istio/istio.sh new file mode 100755 index 0000000..9c2caf6 --- /dev/null +++ b/src/vagrant/kubeadm/istio/istio.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/vagrant/istio/deploy.sh +/vagrant/istio/bookinfo.sh +/vagrant/istio/clean_bookinfo.sh + diff --git a/src/vagrant/kubeadm/kata/containerd.service b/src/vagrant/kubeadm/kata/containerd.service new file mode 100644 index 0000000..1ae7fe8 --- /dev/null +++ b/src/vagrant/kubeadm/kata/containerd.service @@ -0,0 +1,22 @@ +[Unit] +Description=containerd container runtime +Documentation=https://containerd.io +After=network.target + +[Service] +ExecStartPre=-/sbin/modprobe overlay +ExecStart=/usr/local/bin/containerd + +Delegate=yes +KillMode=process +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +LimitNOFILE=infinity +# Comment TasksMax if your systemd version does not supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity + +[Install] +WantedBy=multi-user.target diff --git a/src/vagrant/kubeadm/kata/kata_setup.sh b/src/vagrant/kubeadm/kata/kata_setup.sh new file mode 100644 index 0000000..1fd77b5 --- /dev/null +++ b/src/vagrant/kubeadm/kata/kata_setup.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +ARCH=$(arch) +BRANCH="${BRANCH:-stable-1.7}" +sudo sh -c "echo 'deb http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/ /' > /etc/apt/sources.list.d/kata-containers.list" +curl -sL http://download.opensuse.org/repositories/home:/katacontainers:/releases:/${ARCH}:/${BRANCH}/xUbuntu_$(lsb_release -rs)/Release.key | sudo apt-key add - +sudo -E apt-get update +sudo -E apt-get -y install kata-runtime kata-proxy kata-shim + +wget https://github.com/opencontainers/runc/releases/download/v1.0.0-rc6/runc.amd64 +sudo cp runc.amd64 /usr/sbin/runc +sudo chmod 755 /usr/sbin/runc +wget http://github.com/containerd/containerd/releases/download/v1.2.2/containerd-1.2.2.linux-amd64.tar.gz >& /dev/null +sudo tar -C /usr/local -xzf containerd-1.2.2.linux-amd64.tar.gz +wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/crictl-v1.13.0-linux-amd64.tar.gz >& /dev/null +sudo tar -C /usr/local/bin -xzf crictl-v1.13.0-linux-amd64.tar.gz +echo "runtime-endpoint: unix:///run/containerd/containerd.sock" | sudo tee /etc/crictl.yaml +wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.13.0/critest-v1.13.0-linux-amd64.tar.gz >& /dev/null +sudo tar C /usr/local/bin -xzf critest-v1.13.0-linux-amd64.tar.gz +sudo cp /vagrant/kata/containerd.service /etc/systemd/system/ +sudo systemctl start containerd +sudo mkdir -p /opt/cni/bin +sudo mkdir -p /etc/cni/net.d +sudo mkdir -p /etc/containerd +containerd config default | sudo tee /etc/containerd/config.toml +sudo sed -i "s,\[plugins.cri.registry.mirrors\],\[plugins.cri.registry.mirrors\]\n \[plugins.cri.registry.mirrors.\"registry:5000\"\]\n endpoint = \[\"http://registry:5000\"\]," /etc/containerd/config.toml +sudo sed -i "/.*untrusted_workload_runtime.*/,+5s/runtime_type.*/runtime_type=\"io.containerd.runtime.v1.linux\"/" /etc/containerd/config.toml +sudo sed -i "/.*untrusted_workload_runtime.*/,+5s/runtime_engine.*/runtime_engine=\"kata-runtime\"/" /etc/containerd/config.toml +sudo systemctl restart containerd + +cat << EOF | sudo tee /etc/systemd/system/kubelet.service.d/0-containerd.conf +[Service] +Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock" +EOF + +sudo systemctl daemon-reload +sudo systemctl restart kubelet diff --git a/src/vagrant/kubeadm/kata/nginx-app.sh b/src/vagrant/kubeadm/kata/nginx-app.sh new file mode 100755 index 0000000..fb9540e --- /dev/null +++ b/src/vagrant/kubeadm/kata/nginx-app.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kubectl delete services --all +kubectl delete rc --all +kubectl delete pods --all +kubectl create -f /vagrant/kata/nginx-app.yaml +kubectl get nodes +kubectl get services +kubectl get pods +kubectl get rc +r=0 +while [ "$r" -eq "0" ] +do + sleep 30 + r=$(kubectl get pods | grep Running | wc -l) +done +svcip=$(kubectl get services nginx -o json | grep clusterIP | cut -f4 -d'"') +wget http://$svcip diff --git a/src/vagrant/kubeadm/kata/nginx-app.yaml b/src/vagrant/kubeadm/kata/nginx-app.yaml new file mode 100644 index 0000000..9de4ef4 --- /dev/null +++ b/src/vagrant/kubeadm/kata/nginx-app.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + type: NodePort + ports: + - port: 80 + protocol: TCP + name: http + selector: + app: nginx +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: nginx +spec: + replicas: 2 + template: + metadata: + labels: + app: nginx + annotations: + io.kubernetes.cri.untrusted-workload: "true" + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/src/vagrant/kubeadm/master_setup.sh b/src/vagrant/kubeadm/master_setup.sh new file mode 100644 index 0000000..cec8877 --- /dev/null +++ b/src/vagrant/kubeadm/master_setup.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -ex + +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce=18.03.1~ce-0~ubuntu +cat << EOF | sudo tee /etc/docker/daemon.json +{ + "insecure-registries": ["registry:5000"] +} +EOF +sudo service docker restart + +sudo kubeadm init --apiserver-advertise-address=192.168.1.10 --service-cidr=10.96.0.0/16 --pod-network-cidr=10.32.0.0/12 --token 8c5adc.1cec8dbf339093f0 +mkdir ~/.kube +sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config + +kubectl apply -f http://git.io/weave-kube-1.6 +kubectl apply -f /vagrant/multus/cni_multus.yml diff --git a/src/vagrant/kubeadm/multus/Dockerfile b/src/vagrant/kubeadm/multus/Dockerfile new file mode 100644 index 0000000..7923d0d --- /dev/null +++ b/src/vagrant/kubeadm/multus/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:16.04 +ENV PATH="/usr/local/go/bin:$PATH" +WORKDIR /go/src/ +RUN apt-get update && apt-get install -y wget git gcc +RUN wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz +RUN git clone https://github.com/Intel-Corp/multus-cni +RUN cd multus-cni; bash ./build + +FROM busybox +COPY --from=0 /go/src/multus-cni/bin/multus /root diff --git a/src/vagrant/kubeadm/multus/busybox.yaml b/src/vagrant/kubeadm/multus/busybox.yaml new file mode 100644 index 0000000..7fd1b8d --- /dev/null +++ b/src/vagrant/kubeadm/multus/busybox.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: busybox +spec: + replicas: 2 + template: + metadata: + labels: + app: busybox + spec: + containers: + - name: busybox + image: busybox:1.27.2 + command: + - sleep + - "36000" diff --git a/src/vagrant/kubeadm/multus/cni_multus.yml b/src/vagrant/kubeadm/multus/cni_multus.yml new file mode 100644 index 0000000..123392b --- /dev/null +++ b/src/vagrant/kubeadm/multus/cni_multus.yml @@ -0,0 +1,88 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cnimultus + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: cnimultus-cfg + namespace: kube-system + labels: + tier: node + app: cnimultus +data: + cni-conf.json: | + { + "name": "multus", + "type": "multus", + "delegates": [ + { + "name": "weave", + "type": "weave-net", + "hairpinMode": true, + "masterplugin": true + }, + { + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } + } + ] + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: cnimultus-ds + namespace: kube-system + labels: + tier: node + app: cnimultus +spec: + template: + metadata: + labels: + tier: node + app: cnimultus + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: cnimultus + containers: + - name: run-cni + image: registry:5000/multus-cni:latest + command: ['sh', '-c', 'cp /multus/cni-conf.json /etc/cni/net.d/05-multus.conf; cp /root/multus /opt/cni/bin; while true; do sleep 10000; done' ] + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: etc-cni + mountPath: /etc/cni/net.d + - name: cnimultus-cfg + mountPath: /multus/ + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + - name: etc-cni + hostPath: + path: /etc/cni/net.d + - name: cnimultus-cfg + configMap: + name: cnimultus-cfg diff --git a/src/vagrant/kubeadm/multus/multus.sh b/src/vagrant/kubeadm/multus/multus.sh new file mode 100755 index 0000000..9461a6f --- /dev/null +++ b/src/vagrant/kubeadm/multus/multus.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +while true +do + kubectl get pods -n kube-system | grep kube-cnimultus-ds | grep -v Run | wc -l | grep "^0$" && break + sleep 60 +done + +kubectl delete rc --all +kubectl apply -f /vagrant/multus/busybox.yaml +r="0" +while [ $r -ne "2" ] +do + r=$(kubectl get pods | grep Running | wc -l) + sleep 60 +done + +kubectl get pods --all-namespaces +kubectl get pods | grep Run | sed "s/ .*//" | xargs -I {} kubectl exec -i {} ip a | grep mtu | wc -l | grep "^6$" || exit 1 diff --git a/src/vagrant/kubeadm/registry_setup.sh b/src/vagrant/kubeadm/registry_setup.sh new file mode 100644 index 0000000..5466f1c --- /dev/null +++ b/src/vagrant/kubeadm/registry_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.5 registry +EOF + +sudo apt-get update +sudo apt-get install -y docker.io +cat << EOF | sudo tee /etc/docker/daemon.json +{ + "insecure-registries": ["registry:5000"] +} +EOF +sudo service docker restart + +sudo docker pull registry:2 +sudo docker run -d -p 5000:5000 --restart=always --name registry registry:2 +sudo docker build . -f /vagrant/multus/Dockerfile -t multus-cni +sudo docker tag multus-cni localhost:5000/multus-cni +sudo docker push localhost:5000/multus-cni diff --git a/src/vagrant/kubeadm/virtlet/cirros-vm.yaml b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml new file mode 100644 index 0000000..334142b --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cirros-vm + annotations: + # This tells CRI Proxy that this pod belongs to Virtlet runtime + kubernetes.io/target-runtime: virtlet.cloud + # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason + VirtletDiskDriver: virtio + # inject ssh keys via cloud-init + VirtletSSHKeys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost + # set root volume size + VirtletRootVolumeSize: 1Gi +spec: + # This nodeAffinity specification tells Kubernetes to run this + # pod only on the nodes that have extraRuntime=virtlet label. + # This label is used by Virtlet DaemonSet to select nodes + # that must have Virtlet runtime + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + containers: + - name: cirros-vm + # This specifies the image to use. + # virtlet.cloud/ prefix is used by CRI proxy, the remaining part + # of the image name is prepended with https:// and used to download the image + image: virtlet.cloud/cirros + imagePullPolicy: IfNotPresent + # tty and stdin required for `kubectl attach -t` to work + tty: true + stdin: true + resources: + limits: + # This memory limit is applied to the libvirt domain definition + memory: 160Mi diff --git a/src/vagrant/kubeadm/virtlet/images.yaml b/src/vagrant/kubeadm/virtlet/images.yaml new file mode 100644 index 0000000..1541ca7 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/images.yaml @@ -0,0 +1,3 @@ +translations: + - name: cirros + url: https://github.com/mirantis/virtlet/releases/download/v0.9.3/cirros.img diff --git a/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml new file mode 100644 index 0000000..1bb4882 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml @@ -0,0 +1,521 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system +spec: + selector: + matchLabels: + runtime: virtlet + template: + metadata: + creationTimestamp: null + labels: + runtime: virtlet + name: virtlet + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + containers: + - command: + - /libvirt.sh + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: libvirt + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /run + name: run + - mountPath: /var/lib/virtlet + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/run/libvirt + name: libvirt-sockets + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /dev + name: dev + - image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: virtlet + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/run/virtlet.sock </dev/null + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /run + name: run + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /dev + name: dev + - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/run/libvirt + name: libvirt-sockets + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: k8s-pods-dir + - mountPath: /var/log/vms + name: vms-log + - mountPath: /etc/virtlet/images + name: image-name-translations + - mountPath: /var/log/pods + name: pods-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /var/run/netns + mountPropagation: Bidirectional + name: netns-dir + - command: + - /vms.sh + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: vms + resources: {} + volumeMounts: + - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir + - mountPath: /dev + name: dev + - mountPath: /lib/modules + name: modules + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + initContainers: + - command: + - /prepare-node.sh + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: VIRTLET_DISABLE_KVM + valueFrom: + configMapKeyRef: + key: disable_kvm + name: virtlet-config + optional: true + - name: VIRTLET_SRIOV_SUPPORT + valueFrom: + configMapKeyRef: + key: sriov_support + name: virtlet-config + optional: true + - name: VIRTLET_DOWNLOAD_PROTOCOL + valueFrom: + configMapKeyRef: + key: download_protocol + name: virtlet-config + optional: true + - name: VIRTLET_LOGLEVEL + valueFrom: + configMapKeyRef: + key: loglevel + name: virtlet-config + optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + key: calico-subnet + name: virtlet-config + optional: true + - name: IMAGE_REGEXP_TRANSLATION + valueFrom: + configMapKeyRef: + key: image_regexp_translation + name: virtlet-config + optional: true + - name: VIRTLET_RAW_DEVICES + valueFrom: + configMapKeyRef: + key: raw_devices + name: virtlet-config + optional: true + - name: VIRTLET_DISABLE_LOGGING + valueFrom: + configMapKeyRef: + key: disable_logging + name: virtlet-config + optional: true + - name: VIRTLET_CPU_MODEL + valueFrom: + configMapKeyRef: + key: cpu-model + name: virtlet-config + optional: true + - name: KUBELET_ROOT_DIR + valueFrom: + configMapKeyRef: + key: kubelet_root_dir + name: virtlet-config + optional: true + - name: VIRTLET_IMAGE_TRANSLATIONS_DIR + value: /etc/virtlet/images + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: prepare-node + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /kubelet-volume-plugins + name: k8s-flexvolume-plugins-dir + - mountPath: /run + name: run + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostlog + name: log + - mountPath: /host-var-lib + name: var-lib + - mountPath: /dev + name: dev + - mountPath: /var/lib/virtlet + name: virtlet + serviceAccountName: virtlet + volumes: + - hostPath: + path: /dev + name: dev + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /lib/modules + name: modules + - hostPath: + path: /boot + name: boot + - hostPath: + path: /run + name: run + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /var/lib/virtlet + name: virtlet + - hostPath: + path: /var/lib/libvirt + name: libvirt + - hostPath: + path: /var/log + name: log + - hostPath: + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + - hostPath: + path: /var/lib/kubelet/pods + name: k8s-pods-dir + - hostPath: + path: /var/lib + name: var-lib + - hostPath: + path: /var/log/virtlet/vms + name: vms-log + - hostPath: + path: /var/log/libvirt + name: libvirt-log + - hostPath: + path: /var/run/libvirt + name: libvirt-sockets + - hostPath: + path: /var/log/pods + name: pods-log + - hostPath: + path: /var/run/netns + name: netns-dir + - configMap: + name: virtlet-image-translations + name: image-name-translations + updateStrategy: {} + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: virtlet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + - nodes + verbs: + - create + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: configmap-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet-userdata-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: kubelet-node-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: configmap-reader +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: vm-userdata-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-userdata-reader +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet-crd +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create +- apiGroups: + - virtlet.k8s + resources: + - virtletimagemappings + - virtletconfigmappings + verbs: + - list + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: virtlet-crd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-crd +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletimagemappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletImageMapping + plural: virtletimagemappings + shortNames: + - vim + singular: virtletimagemapping + scope: Namespaced + version: v1 + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletconfigmappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletConfigMapping + plural: virtletconfigmappings + shortNames: + - vcm + singular: virtletconfigmapping + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + properties: + calicoSubnetSize: + maximum: 32 + minimum: 0 + type: integer + cniConfigDir: + type: string + cniPluginDir: + type: string + cpuModel: + type: string + criSocketPath: + type: string + databasePath: + type: string + disableKVM: + type: boolean + disableLogging: + type: boolean + downloadProtocol: + pattern: ^https?$ + type: string + enableRegexpImageTranslation: + type: boolean + enableSriov: + type: boolean + fdServerSocketPath: + type: string + imageDir: + type: string + imageTranslationConfigsDir: + type: string + kubeletRootDir: + type: string + libvirtURI: + type: string + logLevel: + maximum: 2147483647 + minimum: 0 + type: integer + rawDevices: + type: string + skipImageTranslation: + type: boolean + streamPort: + maximum: 65535 + minimum: 1 + type: integer + nodeName: + type: string + nodeSelector: + type: object + priority: + type: integer + version: v1 + diff --git a/src/vagrant/kubeadm/virtlet/virtlet.sh b/src/vagrant/kubeadm/virtlet/virtlet.sh new file mode 100755 index 0000000..4ed527e --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +kubectl label node worker1 extraRuntime=virtlet +kubectl label node worker2 extraRuntime=virtlet +kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y +kubectl create configmap -n kube-system virtlet-image-translations --from-file /vagrant/virtlet/images.yaml +kubectl create -f /vagrant/virtlet/virtlet-ds.yaml + +kubectl delete pod --all +kubectl create -f /vagrant/virtlet/cirros-vm.yaml +r="0" +while [ $r -ne "1" ] +do + r=$(kubectl get pods cirros-vm | grep Running | wc -l) + sleep 60 +done +sleep 360 +kubectl get pods cirros-vm -o custom-columns=:.status.podIP | xargs ping -c 4 +echo 'login by user:cirros & password:gocubsgo' diff --git a/src/vagrant/kubeadm/virtlet/virtlet_setup.sh b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh new file mode 100644 index 0000000..b2dfaa0 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +wget https://github.com/Mirantis/criproxy/releases/download/v0.14.0/criproxy_0.14.0_amd64.deb +echo "criproxy criproxy/primary_cri select containerd" | sudo debconf-set-selections +sudo dpkg -i criproxy_0.14.0_amd64.deb +sudo sed -i "s/EnvironmentFile/#EnvironmentFile/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +sudo systemctl daemon-reload +sudo systemctl restart kubelet diff --git a/src/vagrant/kubeadm/worker_setup.sh b/src/vagrant/kubeadm/worker_setup.sh new file mode 100644 index 0000000..6b08712 --- /dev/null +++ b/src/vagrant/kubeadm/worker_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -ex + +bash /vagrant/kata/kata_setup.sh +bash /vagrant/virtlet/virtlet_setup.sh +sleep 120 +sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 |