diff options
Diffstat (limited to 'src/vagrant')
13 files changed, 572 insertions, 0 deletions
diff --git a/src/vagrant/kubeadm_virtlet/Vagrantfile b/src/vagrant/kubeadm_virtlet/Vagrantfile new file mode 100644 index 0000000..aa3792f --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/Vagrantfile @@ -0,0 +1,33 @@ +$num_workers=3 + +Vagrant.require_version ">= 1.8.6" +Vagrant.configure("2") do |config| + + config.vm.box = "bento/ubuntu-16.04" + config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", 4096] + vb.customize ["modifyvm", :id, "--cpus", 4] + vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] + vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"] + end + + config.vm.provision "shell", path: "host_setup.sh", privileged: false + + config.vm.define "master" do |config| + config.vm.hostname = "master" + config.vm.provision "shell", path: "master_setup.sh", privileged: false + config.vm.network :private_network, ip: "10.96.0.10" + config.vm.network :private_network, ip: "10.244.0.10" + end + + (1 .. $num_workers).each do |i| + config.vm.define vm_name = "worker%d" % [i] do |config| + config.vm.hostname = vm_name + config.vm.provision "shell", path: "worker_setup.sh", privileged: false + config.vm.network :private_network, ip: "10.96.0.#{i+20}" + config.vm.network :private_network, ip: "10.244.0.#{i+20}" + end + end + +end diff --git a/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml new file mode 100644 index 0000000..8beb03f --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cirros-vm + annotations: + # This tells CRI Proxy that this pod belongs to Virtlet runtime + kubernetes.io/target-runtime: virtlet + # An optional annotation specifying the count of virtual CPUs. + # Note that annotation values must always be strings, + # thus numeric values need to be quoted. + # Defaults to "1". + VirtletVCPUCount: "1" + # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason + VirtletDiskDriver: virtio + # inject ssh keys via cloud-init + VirtletSSHKeys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost + # cloud-init user data + VirtletCloudInitUserDataScript: | + #!/bin/sh + echo "Hi there" +spec: + # This nodeAffinity specification tells Kubernetes to run this + # pod only on the nodes that have extraRuntime=virtlet label. + # This label is used by Virtlet DaemonSet to select nodes + # that must have Virtlet runtime + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + containers: + - name: cirros-vm + # This specifies the image to use. + # virtlet/ prefix is used by CRI proxy, the remaining part + # of the image name is prepended with https:// and used to download the image + image: virtlet/cirros + # Virtlet currently ignores image tags, but their meaning may change + # in future, so it’s better not to set them for VM pods. If there’s no tag + # provided in the image specification kubelet defaults to + # imagePullPolicy: Always, which means that the image is always + # redownloaded when the pod is created. In order to make pod creation + # faster and more reliable, we set imagePullPolicy to IfNotPresent here + # so a previously downloaded image is reused if there is one + # in Virtlet’s image store + imagePullPolicy: IfNotPresent + # tty and stdin required for `kubectl attach -t` to work + tty: true + stdin: true + resources: + limits: + # This memory limit is applied to the libvirt domain definition + memory: 160Mi diff --git a/src/vagrant/kubeadm_virtlet/examples/images.yaml b/src/vagrant/kubeadm_virtlet/examples/images.yaml new file mode 100644 index 0000000..3a84585 --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/images.yaml @@ -0,0 +1,3 @@ +translations: + - name: cirros + url: http://github.com/mirantis/virtlet/releases/download/v0.8.2/cirros.img diff --git a/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh b/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh new file mode 100755 index 0000000..bfd0613 --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +kubectl create -f /vagrant/examples/nginx-app.yaml +kubectl get nodes +kubectl get services +kubectl get pods +kubectl get rc +sleep 120 +svcip=$(kubectl get services nginx -o json | grep clusterIP | cut -f4 -d'"') +wget http://$svcip diff --git a/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml b/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml new file mode 100644 index 0000000..f80881a --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + type: NodePort + ports: + - port: 80 + protocol: TCP + name: http + selector: + app: nginx +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: nginx +spec: + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml new file mode 100644 index 0000000..ed037d9 --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml @@ -0,0 +1,360 @@ +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: virtlet + namespace: kube-system +spec: + template: + metadata: + name: virtlet + labels: + runtime: virtlet + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + # hostPID is true to (1) enable VMs to survive virtlet container restart + # (to be checked) and (2) to enable the use of nsenter in init container + hostPID: true + # bootstrap procedure needs to create a configmap in kube-system namespace + serviceAccountName: virtlet + + # only run Virtlet pods on the nodes with extraRuntime=virtlet label + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + + initContainers: + # The init container first copies virtlet's flexvolume driver + # to the default kubelet plugin dir to have it in the proper place by the + # time kubelet is restarted by CRI proxy bootstrap procedure. + # After that it checks if there's already saved kubelet config + # and considers that CRI proxy bootstrap is already done if it exists. + # If it doesn't, it drops criproxy binary into /opt/criproxy/bin + # if it's not already there and then starts criproxy installation. + # The possibility to put criproxy binary in advance into + # /opt/criproxy/bin may be helpful for the purpose of + # debugging criproxy + # At the end it ensures that /var/lib/libvirt/images exists on node. + - name: prepare-node + image: openretriever/virtlet + imagePullPolicy: IfNotPresent + command: + - /prepare-node.sh + volumeMounts: + - name: k8s-flexvolume-plugins-dir + mountPath: /kubelet-volume-plugins + - name: criproxybin + mountPath: /opt/criproxy/bin + - name: run + mountPath: /run + - name: dockersock + mountPath: /var/run/docker.sock + - name: criproxyconf + mountPath: /etc/criproxy + - name: log + mountPath: /hostlog + # for ensuring that /var/lib/libvirt/images exists on node + - name: var-lib + mountPath: /host-var-lib + securityContext: + privileged: true + + containers: + - name: libvirt + image: openretriever/virtlet + # In case we inject local virtlet image we want to use it not officially available one + imagePullPolicy: IfNotPresent + command: + - /libvirt.sh + volumeMounts: + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /run + name: run + - mountPath: /var/lib/virtlet + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/run/libvirt + name: libvirt-sockets + # the log dir is needed here because otherwise libvirt will produce errors + # like this: + # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory + - name: vms-log + mountPath: /var/log/vms + - name: dev + mountPath: /dev + securityContext: + privileged: true + env: + - name: VIRTLET_DISABLE_KVM + valueFrom: + configMapKeyRef: + name: virtlet-config + key: disable_kvm + optional: true + - name: virtlet + image: openretriever/virtlet + # In case we inject local virtlet image we want to use it not officially available one + imagePullPolicy: IfNotPresent + volumeMounts: + - mountPath: /run + name: run + # /boot and /lib/modules are required by supermin + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /var/lib/virtlet + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /etc/cni + name: cniconf + - mountPath: /opt/cni/bin + name: cnibin + - mountPath: /var/run/libvirt + name: libvirt-sockets + - mountPath: /var/lib/cni + name: cnidata + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + # below `:shared` is unofficial way to pass this option docker + # which then will allow virtlet to see what kubelet mounts in + # underlaying directories, after virtlet container is created + - mountPath: /var/lib/kubelet/pods:shared + name: k8s-pods-dir + - name: vms-log + mountPath: /var/log/vms + - mountPath: /etc/virtlet/images + name: image-name-translations + - name: pods-log + mountPath: /kubernetes-log + securityContext: + privileged: true + env: + - name: VIRTLET_DISABLE_KVM + valueFrom: + configMapKeyRef: + name: virtlet-config + key: disable_kvm + optional: true + - name: VIRTLET_DOWNLOAD_PROTOCOL + valueFrom: + configMapKeyRef: + name: virtlet-config + key: download_protocol + optional: true + - name: VIRTLET_LOGLEVEL + valueFrom: + configMapKeyRef: + name: virtlet-config + key: loglevel + optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + name: virtlet-config + key: calico-subnet + optional: true + - name: IMAGE_REGEXP_TRANSLATION + valueFrom: + configMapKeyRef: + name: virtlet-config + key: image_regexp_translation + optional: true + - name: IMAGE_TRANSLATIONS_DIR + value: /etc/virtlet/images + - name: KUBERNETES_POD_LOGS + value: "/kubernetes-log" + # TODO: should we rename it? + - name: VIRTLET_VM_LOG_LOCATION + value: "1" + - name: vms + image: openretriever/virtlet + imagePullPolicy: IfNotPresent + command: + - /vms.sh + volumeMounts: + - mountPath: /var/lib/virtlet + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - name: vms-log + mountPath: /var/log/vms + - name: dev + mountPath: /dev + volumes: + # /dev is needed for host raw device access + - hostPath: + path: /dev + name: dev + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /lib/modules + name: modules + - hostPath: + path: /boot + name: boot + - hostPath: + path: /run + name: run + # TODO: don't hardcode docker socket location here + # This will require CRI proxy installation to run + # in host mount namespace. + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /var/lib/virtlet + name: virtlet + - hostPath: + path: /var/lib/libvirt + name: libvirt + - hostPath: + path: /etc/cni + name: cniconf + - hostPath: + path: /opt/cni/bin + name: cnibin + - hostPath: + path: /var/lib/cni + name: cnidata + - hostPath: + path: /opt/criproxy/bin + name: criproxybin + - hostPath: + path: /etc/criproxy + name: criproxyconf + - hostPath: + path: /var/log + name: log + - hostPath: + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + - hostPath: + path: /var/lib/kubelet/pods + name: k8s-pods-dir + - hostPath: + path: /var/lib + name: var-lib + - hostPath: + path: /var/log/virtlet/vms + name: vms-log + - hostPath: + path: /var/run/libvirt + name: libvirt-sockets + - hostPath: + path: /var/log/pods + name: pods-log + - configMap: + name: virtlet-image-translations + name: image-name-translations +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: virtlet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: virtlet + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: configmap-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kubelet-node-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: configmap-reader +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: virtlet-crd +rules: + - apiGroups: + - "apiextensions.k8s.io" + resources: + - customresourcedefinitions + verbs: + - create + - apiGroups: + - "virtlet.k8s" + resources: + - virtletimagemappings + verbs: + - list + - get +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: virtlet-crd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-crd +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: virtlet + namespace: kube-system diff --git a/src/vagrant/kubeadm_virtlet/examples/virtlet.sh b/src/vagrant/kubeadm_virtlet/examples/virtlet.sh new file mode 100755 index 0000000..68d738d --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/examples/virtlet.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +kubectl label node worker1 extraRuntime=virtlet +kubectl label node worker2 extraRuntime=virtlet +kubectl label node worker3 extraRuntime=virtlet +kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y +kubectl create configmap -n kube-system virtlet-image-translations --from-file images.yaml +kubectl create -f virtlet-ds.yaml +kubectl create -f cirros-vm.yaml diff --git a/src/vagrant/kubeadm_virtlet/host_setup.sh b/src/vagrant/kubeadm_virtlet/host_setup.sh new file mode 100644 index 0000000..003bf2b --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/host_setup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +1.8.1.0.1 localhost +10.96.0.10 master +10.96.0.21 worker1 +10.96.0.22 worker2 +10.96.0.23 worker3 +EOF + +curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list +deb http://apt.kubernetes.io/ kubernetes-xenial main +EOF +sudo apt-get update +sudo apt-get install -y docker.io +sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 +sudo rm -rf /var/lib/kubelet diff --git a/src/vagrant/kubeadm_virtlet/master_setup.sh b/src/vagrant/kubeadm_virtlet/master_setup.sh new file mode 100644 index 0000000..5ae2487 --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/master_setup.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +sudo kubeadm init --apiserver-advertise-address=10.96.0.10 --service-cidr=10.96.0.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0 +sudo cp /etc/kubernetes/admin.conf $HOME/ +sudo chown $(id -u):$(id -g) $HOME/admin.conf +export KUBECONFIG=$HOME/admin.conf +echo "export KUBECONFIG=$HOME/admin.conf" >> $HOME/.bash_profile +kubectl apply -f http://git.io/weave-kube-1.6 diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service new file mode 100644 index 0000000..bb2f1de --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service @@ -0,0 +1,11 @@ +[Unit] +Description=CRI Proxy + +[Service] +ExecStart=/usr/local/bin/criproxy -v 3 -alsologtostderr -connect /var/run/dockershim.sock,virtlet:/run/virtlet.sock -listen /run/criproxy.sock +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +WantedBy=kubelet.service diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service new file mode 100644 index 0000000..c629a4b --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service @@ -0,0 +1,11 @@ +[Unit] +Description=dockershim for criproxy + +[Service] +ExecStart=/usr/local/bin/dockershim ...... +Restart=always +StartLimitInterval=0 +RestartSec=10 + +[Install] +RequiredBy=criproxy.service diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf new file mode 100644 index 0000000..412a48d --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf @@ -0,0 +1,2 @@ +[Service] +Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=/run/criproxy.sock --image-service-endpoint=/run/criproxy.sock --enable-controller-attach-detach=false" diff --git a/src/vagrant/kubeadm_virtlet/worker_setup.sh b/src/vagrant/kubeadm_virtlet/worker_setup.sh new file mode 100644 index 0000000..bc28dab --- /dev/null +++ b/src/vagrant/kubeadm_virtlet/worker_setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ex +sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 10.96.0.10:6443 || true + +sudo docker run --rm openretriever/virtlet tar -c /criproxy | sudo tar -C /usr/local/bin -xv +sudo ln -s /usr/local/bin/criproxy /usr/local/bin/dockershim + +sudo mkdir /etc/criproxy +sudo touch /etc/criproxy/node.conf +sudo cp -r /vagrant/virtlet/etc/systemd/system/* /etc/systemd/system/ +sudo systemctl stop kubelet +sudo systemctl daemon-reload +sudo systemctl enable criproxy dockershim +sudo systemctl start criproxy dockershim +sudo systemctl daemon-reload +sudo systemctl start kubelet |