diff options
21 files changed, 471 insertions, 289 deletions
diff --git a/src/cni/multus/install_cni.sh b/src/cni/multus/install_cni.sh deleted file mode 100644 index 0d1ee6b..0000000 --- a/src/cni/multus/install_cni.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -ex - -export PATH=/usr/local/go/bin:$PATH -apt-get update && apt-get install -y wget -rm -rf multus-cni -wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz -git clone https://github.com/Intel-Corp/multus-cni -cd multus-cni; bash ./build -cp bin/multus /opt/cni/bin -cp /etc/kube-cnimultus/cni-conf.json /etc/cni/net.d/05-multus.conf diff --git a/src/vagrant/kubeadm_multus/Vagrantfile b/src/vagrant/kubeadm/Vagrantfile index 9320074..e861859 100644 --- a/src/vagrant/kubeadm_multus/Vagrantfile +++ b/src/vagrant/kubeadm/Vagrantfile @@ -9,11 +9,15 @@ Vagrant.configure("2") do |config| libvirt.cpus = 4 end - config.vm.synced_folder "../..", "/src" - config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.define "registry" do |config| + config.vm.hostname = "registry" + config.vm.provision "shell", path: "registry_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.5" + end config.vm.define "master" do |config| config.vm.hostname = "master" + config.vm.provision "shell", path: "host_setup.sh", privileged: false config.vm.provision "shell", path: "master_setup.sh", privileged: false config.vm.network :private_network, ip: "192.168.1.10" end @@ -21,6 +25,7 @@ Vagrant.configure("2") do |config| (1 .. $num_workers).each do |i| config.vm.define vm_name = "worker%d" % [i] do |config| config.vm.hostname = vm_name + config.vm.provision "shell", path: "host_setup.sh", privileged: false config.vm.provision "shell", path: "worker_setup.sh", privileged: false config.vm.network :private_network, ip: "192.168.1.#{i+20}" end diff --git a/src/vagrant/kubeadm_multus/deploy.sh b/src/vagrant/kubeadm/deploy.sh index 9c9e51e..c727483 100755 --- a/src/vagrant/kubeadm_multus/deploy.sh +++ b/src/vagrant/kubeadm/deploy.sh @@ -6,4 +6,4 @@ DIR="$(dirname `readlink -f $0`)" cd $DIR ../cleanup.sh vagrant up -vagrant ssh master -c "/vagrant/examples/multus.sh" +vagrant ssh master -c "/vagrant/multus/multus.sh" diff --git a/src/vagrant/kubeadm/host_setup.sh b/src/vagrant/kubeadm/host_setup.sh new file mode 100644 index 0000000..d792119 --- /dev/null +++ b/src/vagrant/kubeadm/host_setup.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.05 registry +192.168.1.10 master +192.168.1.21 worker1 +192.168.1.22 worker2 +192.168.1.23 worker3 +EOF + +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce=18.03.1~ce-0~ubuntu +cat << EOF | sudo tee /etc/docker/daemon.json +{ + "insecure-registries": ["registry:5000"] +} +EOF +sudo service docker restart + +curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list +deb http://apt.kubernetes.io/ kubernetes-xenial main +EOF +sudo apt-get update +sudo apt-get install -y --allow-unauthenticated kubelet=1.12.2-00 kubeadm=1.12.2-00 kubectl=1.12.2-00 kubernetes-cni=0.6.0-00 +echo 'Environment="KUBELET_EXTRA_ARGS=--feature-gates=DevicePlugins=true"' | sudo tee /etc/default/kubelet + +sudo modprobe ip_vs +sudo modprobe ip_vs_rr +sudo modprobe ip_vs_wrr +sudo modprobe ip_vs_sh +sudo swapoff -a +sudo systemctl daemon-reload +sudo systemctl stop kubelet +sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm/master_setup.sh b/src/vagrant/kubeadm/master_setup.sh new file mode 100644 index 0000000..a898d8d --- /dev/null +++ b/src/vagrant/kubeadm/master_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +sudo kubeadm init --apiserver-advertise-address=192.168.1.10 --service-cidr=10.96.0.0/16 --pod-network-cidr=10.32.0.0/12 --token 8c5adc.1cec8dbf339093f0 +mkdir ~/.kube +sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config + +kubectl apply -f http://git.io/weave-kube-1.6 +kubectl apply -f /vagrant/multus/cni_multus.yml diff --git a/src/vagrant/kubeadm/multus/Dockerfile b/src/vagrant/kubeadm/multus/Dockerfile new file mode 100644 index 0000000..7923d0d --- /dev/null +++ b/src/vagrant/kubeadm/multus/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:16.04 +ENV PATH="/usr/local/go/bin:$PATH" +WORKDIR /go/src/ +RUN apt-get update && apt-get install -y wget git gcc +RUN wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz +RUN git clone https://github.com/Intel-Corp/multus-cni +RUN cd multus-cni; bash ./build + +FROM busybox +COPY --from=0 /go/src/multus-cni/bin/multus /root diff --git a/src/vagrant/kubeadm_multus/examples/busybox.yaml b/src/vagrant/kubeadm/multus/busybox.yaml index 7fd1b8d..7fd1b8d 100644 --- a/src/vagrant/kubeadm_multus/examples/busybox.yaml +++ b/src/vagrant/kubeadm/multus/busybox.yaml diff --git a/src/cni/multus/kube_cni_multus.yml b/src/vagrant/kubeadm/multus/cni_multus.yml index cd91737..50372ca 100644 --- a/src/cni/multus/kube_cni_multus.yml +++ b/src/vagrant/kubeadm/multus/cni_multus.yml @@ -8,7 +8,7 @@ metadata: kind: ConfigMap apiVersion: v1 metadata: - name: kube-cnimultus-cfg + name: cnimultus-cfg namespace: kube-system labels: tier: node @@ -45,7 +45,7 @@ data: apiVersion: extensions/v1beta1 kind: DaemonSet metadata: - name: kube-cnimultus-ds + name: cnimultus-ds namespace: kube-system labels: tier: node @@ -65,31 +65,24 @@ spec: operator: Exists effect: NoSchedule serviceAccountName: cnimultus - initContainers: - - name: install-cni - image: ubuntu:16.04 - command: - - bash - - "-c" - - "apt-get update && apt-get install -y git && git clone http://github.com/opnfv/container4nfv && cd container4nfv && git fetch https://gerrit.opnfv.org/gerrit/container4nfv refs/changes/81/47681/5 && git checkout FETCH_HEAD && bash ./src/cni/multus/install_cni.sh" + containers: + - name: run-cni + image: registry:5000/multus-cni + command: ['sh', '-c', 'cp /multus/cni-conf.json /etc/cni/net.d/05-multus.conf; cp /root/multus /opt/cni/bin; while true; do sleep 10000; done' ] volumeMounts: - name: cni-bin mountPath: /opt/cni/bin - - name: cni-cfg + - name: etc-cni mountPath: /etc/cni/net.d - name: cnimultus-cfg - mountPath: /etc/kube-cnimultus - containers: - - name: run-cni - image: busybox:1.27.2 - command: ['sh', '-c', 'while true; do sleep 10000; done' ] + mountPath: /multus/ volumes: - name: cni-bin hostPath: path: /opt/cni/bin - - name: cni-cfg + - name: etc-cni hostPath: path: /etc/cni/net.d - name: cnimultus-cfg configMap: - name: kube-cnimultus-cfg + name: cnimultus-cfg diff --git a/src/vagrant/kubeadm_multus/examples/multus.sh b/src/vagrant/kubeadm/multus/multus.sh index d7b39a0..9461a6f 100755 --- a/src/vagrant/kubeadm_multus/examples/multus.sh +++ b/src/vagrant/kubeadm/multus/multus.sh @@ -24,7 +24,7 @@ do done kubectl delete rc --all -kubectl apply -f /vagrant/examples/busybox.yaml +kubectl apply -f /vagrant/multus/busybox.yaml r="0" while [ $r -ne "2" ] do diff --git a/src/vagrant/kubeadm/registry_setup.sh b/src/vagrant/kubeadm/registry_setup.sh new file mode 100644 index 0000000..188502d --- /dev/null +++ b/src/vagrant/kubeadm/registry_setup.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.5 registry +EOF + +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce=18.06.0~ce~3-0~ubuntu +cat << EOF | sudo tee /etc/docker/daemon.json +{ + "insecure-registries": ["registry:5000"] +} +EOF +sudo service docker restart + +sudo docker pull registry:2 +sudo docker run -d -p 5000:5000 --restart=always --name registry registry:2 +sudo docker build . -f /vagrant/multus/Dockerfile -t multus-cni +sudo docker tag multus-cni localhost:5000/multus-cni +sudo docker push localhost:5000/multus-cni diff --git a/src/vagrant/kubeadm_multus/worker_setup.sh b/src/vagrant/kubeadm/worker_setup.sh index 74e4178..858cafd 100644 --- a/src/vagrant/kubeadm_multus/worker_setup.sh +++ b/src/vagrant/kubeadm/worker_setup.sh @@ -1,4 +1,6 @@ #!/bin/bash set -ex -sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true + +sleep 120 +sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 diff --git a/src/vagrant/kubeadm_multus/host_setup.sh b/src/vagrant/kubeadm_multus/host_setup.sh deleted file mode 100644 index 524a967..0000000 --- a/src/vagrant/kubeadm_multus/host_setup.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -ex - -cat << EOF | sudo tee /etc/hosts -127.0.0.1 localhost -192.168.1.10 master -192.168.1.21 worker1 -192.168.1.22 worker2 -192.168.1.23 worker3 -EOF - -sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -sudo apt-key adv -k 58118E89F3A912897C070ADBF76221572C52609D -cat << EOF | sudo tee /etc/apt/sources.list.d/docker.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF - -curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list -deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF -sudo apt-get update -sudo apt-get install -y --allow-unauthenticated --allow-downgrades docker-engine=1.12.6-0~ubuntu-xenial kubelet=1.9.1-00 kubeadm=1.9.1-00 kubectl=1.9.1-00 kubernetes-cni=0.6.0-00 - -sudo swapoff -a -sudo systemctl daemon-reload -sudo systemctl stop kubelet -sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_multus/master_setup.sh b/src/vagrant/kubeadm_multus/master_setup.sh deleted file mode 100644 index dfc3d05..0000000 --- a/src/vagrant/kubeadm_multus/master_setup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -ex - -sudo kubeadm init --apiserver-advertise-address=192.168.1.10 --service-cidr=10.96.0.0/16 --pod-network-cidr=10.32.0.0/12 --token 8c5adc.1cec8dbf339093f0 - -mkdir ~/.kube -sudo cp /etc/kubernetes/admin.conf $HOME/.kube/config -sudo chown $(id -u):$(id -g) $HOME/.kube/config - -kubectl apply -f http://git.io/weave-kube-1.6 -kubectl apply -f /src/cni/multus/kube_cni_multus.yml diff --git a/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml index 8beb03f..334142b 100644 --- a/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml +++ b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml @@ -4,21 +4,14 @@ metadata: name: cirros-vm annotations: # This tells CRI Proxy that this pod belongs to Virtlet runtime - kubernetes.io/target-runtime: virtlet - # An optional annotation specifying the count of virtual CPUs. - # Note that annotation values must always be strings, - # thus numeric values need to be quoted. - # Defaults to "1". - VirtletVCPUCount: "1" + kubernetes.io/target-runtime: virtlet.cloud # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason VirtletDiskDriver: virtio # inject ssh keys via cloud-init VirtletSSHKeys: | ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost - # cloud-init user data - VirtletCloudInitUserDataScript: | - #!/bin/sh - echo "Hi there" + # set root volume size + VirtletRootVolumeSize: 1Gi spec: # This nodeAffinity specification tells Kubernetes to run this # pod only on the nodes that have extraRuntime=virtlet label. @@ -36,17 +29,9 @@ spec: containers: - name: cirros-vm # This specifies the image to use. - # virtlet/ prefix is used by CRI proxy, the remaining part + # virtlet.cloud/ prefix is used by CRI proxy, the remaining part # of the image name is prepended with https:// and used to download the image - image: virtlet/cirros - # Virtlet currently ignores image tags, but their meaning may change - # in future, so it’s better not to set them for VM pods. If there’s no tag - # provided in the image specification kubelet defaults to - # imagePullPolicy: Always, which means that the image is always - # redownloaded when the pod is created. In order to make pod creation - # faster and more reliable, we set imagePullPolicy to IfNotPresent here - # so a previously downloaded image is reused if there is one - # in Virtlet’s image store + image: virtlet.cloud/cirros imagePullPolicy: IfNotPresent # tty and stdin required for `kubectl attach -t` to work tty: true diff --git a/src/vagrant/kubeadm_virtlet/examples/images.yaml b/src/vagrant/kubeadm_virtlet/examples/images.yaml index 3a84585..1541ca7 100644 --- a/src/vagrant/kubeadm_virtlet/examples/images.yaml +++ b/src/vagrant/kubeadm_virtlet/examples/images.yaml @@ -1,3 +1,3 @@ translations: - name: cirros - url: http://github.com/mirantis/virtlet/releases/download/v0.8.2/cirros.img + url: https://github.com/mirantis/virtlet/releases/download/v0.9.3/cirros.img diff --git a/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml index ed037d9..1bb4882 100644 --- a/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml +++ b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml @@ -1,25 +1,21 @@ --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: + creationTimestamp: null name: virtlet namespace: kube-system spec: + selector: + matchLabels: + runtime: virtlet template: metadata: - name: virtlet + creationTimestamp: null labels: runtime: virtlet + name: virtlet spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - # hostPID is true to (1) enable VMs to survive virtlet container restart - # (to be checked) and (2) to enable the use of nsenter in init container - hostPID: true - # bootstrap procedure needs to create a configmap in kube-system namespace - serviceAccountName: virtlet - - # only run Virtlet pods on the nodes with extraRuntime=virtlet label affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -29,50 +25,21 @@ spec: operator: In values: - virtlet - - initContainers: - # The init container first copies virtlet's flexvolume driver - # to the default kubelet plugin dir to have it in the proper place by the - # time kubelet is restarted by CRI proxy bootstrap procedure. - # After that it checks if there's already saved kubelet config - # and considers that CRI proxy bootstrap is already done if it exists. - # If it doesn't, it drops criproxy binary into /opt/criproxy/bin - # if it's not already there and then starts criproxy installation. - # The possibility to put criproxy binary in advance into - # /opt/criproxy/bin may be helpful for the purpose of - # debugging criproxy - # At the end it ensures that /var/lib/libvirt/images exists on node. - - name: prepare-node - image: openretriever/virtlet + containers: + - command: + - /libvirt.sh + image: mirantis/virtlet:v1.4.1 imagePullPolicy: IfNotPresent - command: - - /prepare-node.sh - volumeMounts: - - name: k8s-flexvolume-plugins-dir - mountPath: /kubelet-volume-plugins - - name: criproxybin - mountPath: /opt/criproxy/bin - - name: run - mountPath: /run - - name: dockersock - mountPath: /var/run/docker.sock - - name: criproxyconf - mountPath: /etc/criproxy - - name: log - mountPath: /hostlog - # for ensuring that /var/lib/libvirt/images exists on node - - name: var-lib - mountPath: /host-var-lib + name: libvirt + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null + resources: {} securityContext: privileged: true - - containers: - - name: libvirt - image: openretriever/virtlet - # In case we inject local virtlet image we want to use it not officially available one - imagePullPolicy: IfNotPresent - command: - - /libvirt.sh volumeMounts: - mountPath: /sys/fs/cgroup name: cgroup @@ -90,117 +57,176 @@ spec: name: libvirt - mountPath: /var/run/libvirt name: libvirt-sockets - # the log dir is needed here because otherwise libvirt will produce errors - # like this: - # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory - - name: vms-log - mountPath: /var/log/vms - - name: dev - mountPath: /dev + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /dev + name: dev + - image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: virtlet + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/run/virtlet.sock </dev/null + resources: {} securityContext: privileged: true - env: - - name: VIRTLET_DISABLE_KVM - valueFrom: - configMapKeyRef: - name: virtlet-config - key: disable_kvm - optional: true - - name: virtlet - image: openretriever/virtlet - # In case we inject local virtlet image we want to use it not officially available one - imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /run name: run - # /boot and /lib/modules are required by supermin - mountPath: /lib/modules name: modules readOnly: true - mountPath: /boot name: boot readOnly: true + - mountPath: /dev + name: dev - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional name: virtlet - mountPath: /var/lib/libvirt name: libvirt - - mountPath: /etc/cni - name: cniconf - - mountPath: /opt/cni/bin - name: cnibin - mountPath: /var/run/libvirt name: libvirt-sockets - - mountPath: /var/lib/cni - name: cnidata - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec name: k8s-flexvolume-plugins-dir - # below `:shared` is unofficial way to pass this option docker - # which then will allow virtlet to see what kubelet mounts in - # underlaying directories, after virtlet container is created - - mountPath: /var/lib/kubelet/pods:shared + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional name: k8s-pods-dir - - name: vms-log - mountPath: /var/log/vms + - mountPath: /var/log/vms + name: vms-log - mountPath: /etc/virtlet/images name: image-name-translations - - name: pods-log - mountPath: /kubernetes-log - securityContext: - privileged: true + - mountPath: /var/log/pods + name: pods-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /var/run/netns + mountPropagation: Bidirectional + name: netns-dir + - command: + - /vms.sh + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: vms + resources: {} + volumeMounts: + - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir + - mountPath: /dev + name: dev + - mountPath: /lib/modules + name: modules + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + initContainers: + - command: + - /prepare-node.sh env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName - name: VIRTLET_DISABLE_KVM valueFrom: configMapKeyRef: - name: virtlet-config key: disable_kvm + name: virtlet-config optional: true - - name: VIRTLET_DOWNLOAD_PROTOCOL + - name: VIRTLET_SRIOV_SUPPORT valueFrom: configMapKeyRef: + key: sriov_support name: virtlet-config + optional: true + - name: VIRTLET_DOWNLOAD_PROTOCOL + valueFrom: + configMapKeyRef: key: download_protocol + name: virtlet-config optional: true - name: VIRTLET_LOGLEVEL valueFrom: configMapKeyRef: - name: virtlet-config key: loglevel + name: virtlet-config optional: true - name: VIRTLET_CALICO_SUBNET valueFrom: configMapKeyRef: - name: virtlet-config key: calico-subnet + name: virtlet-config optional: true - name: IMAGE_REGEXP_TRANSLATION valueFrom: configMapKeyRef: - name: virtlet-config key: image_regexp_translation + name: virtlet-config + optional: true + - name: VIRTLET_RAW_DEVICES + valueFrom: + configMapKeyRef: + key: raw_devices + name: virtlet-config + optional: true + - name: VIRTLET_DISABLE_LOGGING + valueFrom: + configMapKeyRef: + key: disable_logging + name: virtlet-config + optional: true + - name: VIRTLET_CPU_MODEL + valueFrom: + configMapKeyRef: + key: cpu-model + name: virtlet-config + optional: true + - name: KUBELET_ROOT_DIR + valueFrom: + configMapKeyRef: + key: kubelet_root_dir + name: virtlet-config optional: true - - name: IMAGE_TRANSLATIONS_DIR + - name: VIRTLET_IMAGE_TRANSLATIONS_DIR value: /etc/virtlet/images - - name: KUBERNETES_POD_LOGS - value: "/kubernetes-log" - # TODO: should we rename it? - - name: VIRTLET_VM_LOG_LOCATION - value: "1" - - name: vms - image: openretriever/virtlet + image: mirantis/virtlet:v1.4.1 imagePullPolicy: IfNotPresent - command: - - /vms.sh + name: prepare-node + resources: {} + securityContext: + privileged: true volumeMounts: + - mountPath: /kubelet-volume-plugins + name: k8s-flexvolume-plugins-dir + - mountPath: /run + name: run + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostlog + name: log + - mountPath: /host-var-lib + name: var-lib + - mountPath: /dev + name: dev - mountPath: /var/lib/virtlet name: virtlet - - mountPath: /var/lib/libvirt - name: libvirt - - name: vms-log - mountPath: /var/log/vms - - name: dev - mountPath: /dev + serviceAccountName: virtlet volumes: - # /dev is needed for host raw device access - hostPath: path: /dev name: dev @@ -216,9 +242,6 @@ spec: - hostPath: path: /run name: run - # TODO: don't hardcode docker socket location here - # This will require CRI proxy installation to run - # in host mount namespace. - hostPath: path: /var/run/docker.sock name: dockersock @@ -229,21 +252,6 @@ spec: path: /var/lib/libvirt name: libvirt - hostPath: - path: /etc/cni - name: cniconf - - hostPath: - path: /opt/cni/bin - name: cnibin - - hostPath: - path: /var/lib/cni - name: cnidata - - hostPath: - path: /opt/criproxy/bin - name: criproxybin - - hostPath: - path: /etc/criproxy - name: criproxyconf - - hostPath: path: /var/log name: log - hostPath: @@ -259,18 +267,27 @@ spec: path: /var/log/virtlet/vms name: vms-log - hostPath: + path: /var/log/libvirt + name: libvirt-log + - hostPath: path: /var/run/libvirt name: libvirt-sockets - hostPath: path: /var/log/pods name: pods-log + - hostPath: + path: /var/run/netns + name: netns-dir - configMap: name: virtlet-image-translations name: image-name-translations + updateStrategy: {} + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: + creationTimestamp: null name: virtlet roleRef: apiGroup: rbac.authorization.k8s.io @@ -280,23 +297,29 @@ subjects: - kind: ServiceAccount name: virtlet namespace: kube-system + --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole metadata: + creationTimestamp: null name: virtlet namespace: kube-system rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create +- apiGroups: + - "" + resources: + - configmaps + - nodes + verbs: + - create + - get + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: + creationTimestamp: null name: configmap-reader rules: - apiGroups: @@ -307,10 +330,27 @@ rules: - get - list - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet-userdata-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: + creationTimestamp: null name: kubelet-node-binding roleRef: apiGroup: rbac.authorization.k8s.io @@ -320,29 +360,49 @@ subjects: - apiGroup: rbac.authorization.k8s.io kind: Group name: system:nodes + --- -kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding metadata: + creationTimestamp: null + name: vm-userdata-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-userdata-reader +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null name: virtlet-crd rules: - - apiGroups: - - "apiextensions.k8s.io" - resources: - - customresourcedefinitions - verbs: - - create - - apiGroups: - - "virtlet.k8s" - resources: - - virtletimagemappings - verbs: - - list - - get +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create +- apiGroups: + - virtlet.k8s + resources: + - virtletimagemappings + - virtletconfigmappings + verbs: + - list + - get + --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: + creationTimestamp: null name: virtlet-crd roleRef: apiGroup: rbac.authorization.k8s.io @@ -352,9 +412,110 @@ subjects: - kind: ServiceAccount name: virtlet namespace: kube-system + --- apiVersion: v1 kind: ServiceAccount metadata: + creationTimestamp: null name: virtlet namespace: kube-system + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletimagemappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletImageMapping + plural: virtletimagemappings + shortNames: + - vim + singular: virtletimagemapping + scope: Namespaced + version: v1 + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletconfigmappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletConfigMapping + plural: virtletconfigmappings + shortNames: + - vcm + singular: virtletconfigmapping + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + properties: + calicoSubnetSize: + maximum: 32 + minimum: 0 + type: integer + cniConfigDir: + type: string + cniPluginDir: + type: string + cpuModel: + type: string + criSocketPath: + type: string + databasePath: + type: string + disableKVM: + type: boolean + disableLogging: + type: boolean + downloadProtocol: + pattern: ^https?$ + type: string + enableRegexpImageTranslation: + type: boolean + enableSriov: + type: boolean + fdServerSocketPath: + type: string + imageDir: + type: string + imageTranslationConfigsDir: + type: string + kubeletRootDir: + type: string + libvirtURI: + type: string + logLevel: + maximum: 2147483647 + minimum: 0 + type: integer + rawDevices: + type: string + skipImageTranslation: + type: boolean + streamPort: + maximum: 65535 + minimum: 1 + type: integer + nodeName: + type: string + nodeSelector: + type: object + priority: + type: integer + version: v1 + diff --git a/src/vagrant/kubeadm_virtlet/host_setup.sh b/src/vagrant/kubeadm_virtlet/host_setup.sh index b2ee85c..f211f19 100644 --- a/src/vagrant/kubeadm_virtlet/host_setup.sh +++ b/src/vagrant/kubeadm_virtlet/host_setup.sh @@ -10,20 +10,33 @@ cat << EOF | sudo tee /etc/hosts 192.168.1.23 worker3 EOF -sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -sudo apt-key adv -k 58118E89F3A912897C070ADBF76221572C52609D -cat << EOF | sudo tee /etc/apt/sources.list.d/docker.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF +sudo apt-get update +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce=18.03.1~ce-0~ubuntu curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list deb http://apt.kubernetes.io/ kubernetes-xenial main EOF sudo apt-get update -sudo apt-get install -y --allow-unauthenticated --allow-downgrades docker-engine=1.12.6-0~ubuntu-xenial kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00 +sudo apt-get install -y --allow-unauthenticated kubelet=1.12.2-00 kubeadm=1.12.2-00 kubectl=1.12.2-00 kubernetes-cni=0.6.0-00 -sudo rm -rf /var/lib/kubelet -sudo systemctl stop kubelet +sudo modprobe ip_vs +sudo modprobe ip_vs_rr +sudo modprobe ip_vs_wrr +sudo modprobe ip_vs_sh +sudo swapoff -a sudo systemctl daemon-reload +sudo systemctl stop kubelet sudo systemctl start kubelet diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service deleted file mode 100644 index bb2f1de..0000000 --- a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/criproxy.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=CRI Proxy - -[Service] -ExecStart=/usr/local/bin/criproxy -v 3 -alsologtostderr -connect /var/run/dockershim.sock,virtlet:/run/virtlet.sock -listen /run/criproxy.sock -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -WantedBy=kubelet.service diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service deleted file mode 100644 index c629a4b..0000000 --- a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/dockershim.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=dockershim for criproxy - -[Service] -ExecStart=/usr/local/bin/dockershim ...... -Restart=always -StartLimitInterval=0 -RestartSec=10 - -[Install] -RequiredBy=criproxy.service diff --git a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf b/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf deleted file mode 100644 index 412a48d..0000000 --- a/src/vagrant/kubeadm_virtlet/virtlet/etc/systemd/system/kubelet.service.d/20-criproxy.conf +++ /dev/null @@ -1,2 +0,0 @@ -[Service] -Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --container-runtime-endpoint=/run/criproxy.sock --image-service-endpoint=/run/criproxy.sock --enable-controller-attach-detach=false" diff --git a/src/vagrant/kubeadm_virtlet/worker_setup.sh b/src/vagrant/kubeadm_virtlet/worker_setup.sh index 4472874..bc37fb3 100644 --- a/src/vagrant/kubeadm_virtlet/worker_setup.sh +++ b/src/vagrant/kubeadm_virtlet/worker_setup.sh @@ -1,18 +1,12 @@ #!/bin/bash set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true +sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 -sudo docker pull openretriever/virtlet -sudo docker run --rm openretriever/virtlet tar -c /criproxy | sudo tar -C /usr/local/bin -xv -sudo ln -s /usr/local/bin/criproxy /usr/local/bin/dockershim - -sudo mkdir /etc/criproxy -sudo touch /etc/criproxy/node.conf -sudo cp -r /vagrant/virtlet/etc/systemd/system/* /etc/systemd/system/ -sudo systemctl stop kubelet -sudo systemctl daemon-reload -sudo systemctl enable criproxy dockershim -sudo systemctl start criproxy dockershim +wget https://github.com/Mirantis/criproxy/releases/download/v0.12.0/criproxy_0.12.0_amd64.deb +sudo dpkg -i criproxy_0.12.0_amd64.deb +sudo sed -i "s/EnvironmentFile/#EnvironmentFile/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf sudo systemctl daemon-reload -sudo systemctl start kubelet +sudo systemctl restart dockershim +sudo systemctl restart criproxy +sudo systemctl restart kubelet |