Merge "enable virtlet & kata"

8 files changed, 601 insertions, 3 deletions

diff --git a/src/vagrant/kubeadm/Vagrantfile b/src/vagrant/kubeadm/Vagrantfile
index e861859..dc5efb1 100644
--- a/src/vagrant/kubeadm/Vagrantfile
+++ b/src/vagrant/kubeadm/Vagrantfile
@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
 
   config.vm.define "registry" do |config|
     config.vm.hostname = "registry"
-    config.vm.provision "shell", path: "registry_setup.sh", privileged: false
+    #config.vm.provision "shell", path: "registry_setup.sh", privileged: false
     config.vm.network :private_network, ip: "192.168.1.5"
   end
 
diff --git a/src/vagrant/kubeadm/deploy.sh b/src/vagrant/kubeadm/deploy.sh
index 0154031..82b2483 100755
--- a/src/vagrant/kubeadm/deploy.sh
+++ b/src/vagrant/kubeadm/deploy.sh
@@ -6,6 +6,6 @@ DIR="$(dirname `readlink -f $0`)"
 cd $DIR
 ../cleanup.sh
 vagrant up
-vagrant ssh master -c "/vagrant/istio/istio.sh"
-vagrant ssh master -c "/vagrant/multus/multus.sh"
 vagrant ssh master -c "/vagrant/kata/nginx-app.sh"
+vagrant ssh master -c "/vagrant/virtlet/virtlet.sh"
+vagrant ssh master -c "/vagrant/istio/istio.sh"
diff --git a/src/vagrant/kubeadm/virtlet/cirros-vm.yaml b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml
new file mode 100644
index 0000000..334142b
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml
@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cirros-vm
+  annotations:
+    # This tells CRI Proxy that this pod belongs to Virtlet runtime
+    kubernetes.io/target-runtime: virtlet.cloud
+    # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
+    VirtletDiskDriver: virtio
+    # inject ssh keys via cloud-init
+    VirtletSSHKeys: |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+    # set root volume size
+    VirtletRootVolumeSize: 1Gi
+spec:
+  # This nodeAffinity specification tells Kubernetes to run this
+  # pod only on the nodes that have extraRuntime=virtlet label.
+  # This label is used by Virtlet DaemonSet to select nodes
+  # that must have Virtlet runtime
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: extraRuntime
+            operator: In
+            values:
+            - virtlet
+  containers:
+  - name: cirros-vm
+    # This specifies the image to use.
+    # virtlet.cloud/ prefix is used by CRI proxy, the remaining part
+    # of the image name is prepended with https:// and used to download the image
+    image: virtlet.cloud/cirros
+    imagePullPolicy: IfNotPresent
+    # tty and stdin required for `kubectl attach -t` to work
+    tty: true
+    stdin: true
+    resources:
+      limits:
+        # This memory limit is applied to the libvirt domain definition
+        memory: 160Mi
diff --git a/src/vagrant/kubeadm/virtlet/images.yaml b/src/vagrant/kubeadm/virtlet/images.yaml
new file mode 100644
index 0000000..1541ca7
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/images.yaml
@@ -0,0 +1,3 @@
+translations:
+  - name: cirros
+    url: https://github.com/mirantis/virtlet/releases/download/v0.9.3/cirros.img
diff --git a/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml
new file mode 100644
index 0000000..1bb4882
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml
@@ -0,0 +1,521 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  name: virtlet
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      runtime: virtlet
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        runtime: virtlet
+      name: virtlet
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: extraRuntime
+                operator: In
+                values:
+                - virtlet
+      containers:
+      - command:
+        - /libvirt.sh
+        image: mirantis/virtlet:v1.4.1
+        imagePullPolicy: IfNotPresent
+        name: libvirt
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null
+        resources: {}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+        - mountPath: /boot
+          name: boot
+          readOnly: true
+        - mountPath: /run
+          name: run
+        - mountPath: /var/lib/virtlet
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - mountPath: /var/run/libvirt
+          name: libvirt-sockets
+        - mountPath: /var/log/vms
+          name: vms-log
+        - mountPath: /var/log/libvirt
+          name: libvirt-log
+        - mountPath: /dev
+          name: dev
+      - image: mirantis/virtlet:v1.4.1
+        imagePullPolicy: IfNotPresent
+        name: virtlet
+        readinessProbe:
+          exec:
+            command:
+            - /bin/sh
+            - -c
+            - socat - UNIX:/run/virtlet.sock </dev/null
+        resources: {}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /run
+          name: run
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+        - mountPath: /boot
+          name: boot
+          readOnly: true
+        - mountPath: /dev
+          name: dev
+        - mountPath: /var/lib/virtlet
+          mountPropagation: Bidirectional
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - mountPath: /var/run/libvirt
+          name: libvirt-sockets
+        - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+          name: k8s-flexvolume-plugins-dir
+        - mountPath: /var/lib/kubelet/pods
+          mountPropagation: Bidirectional
+          name: k8s-pods-dir
+        - mountPath: /var/log/vms
+          name: vms-log
+        - mountPath: /etc/virtlet/images
+          name: image-name-translations
+        - mountPath: /var/log/pods
+          name: pods-log
+        - mountPath: /var/log/libvirt
+          name: libvirt-log
+        - mountPath: /var/run/netns
+          mountPropagation: Bidirectional
+          name: netns-dir
+      - command:
+        - /vms.sh
+        image: mirantis/virtlet:v1.4.1
+        imagePullPolicy: IfNotPresent
+        name: vms
+        resources: {}
+        volumeMounts:
+        - mountPath: /var/lib/virtlet
+          mountPropagation: HostToContainer
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - mountPath: /var/log/vms
+          name: vms-log
+        - mountPath: /var/lib/kubelet/pods
+          mountPropagation: HostToContainer
+          name: k8s-pods-dir
+        - mountPath: /dev
+          name: dev
+        - mountPath: /lib/modules
+          name: modules
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
+      hostPID: true
+      initContainers:
+      - command:
+        - /prepare-node.sh
+        env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: VIRTLET_DISABLE_KVM
+          valueFrom:
+            configMapKeyRef:
+              key: disable_kvm
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_SRIOV_SUPPORT
+          valueFrom:
+            configMapKeyRef:
+              key: sriov_support
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_DOWNLOAD_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              key: download_protocol
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_LOGLEVEL
+          valueFrom:
+            configMapKeyRef:
+              key: loglevel
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_CALICO_SUBNET
+          valueFrom:
+            configMapKeyRef:
+              key: calico-subnet
+              name: virtlet-config
+              optional: true
+        - name: IMAGE_REGEXP_TRANSLATION
+          valueFrom:
+            configMapKeyRef:
+              key: image_regexp_translation
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_RAW_DEVICES
+          valueFrom:
+            configMapKeyRef:
+              key: raw_devices
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_DISABLE_LOGGING
+          valueFrom:
+            configMapKeyRef:
+              key: disable_logging
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_CPU_MODEL
+          valueFrom:
+            configMapKeyRef:
+              key: cpu-model
+              name: virtlet-config
+              optional: true
+        - name: KUBELET_ROOT_DIR
+          valueFrom:
+            configMapKeyRef:
+              key: kubelet_root_dir
+              name: virtlet-config
+              optional: true
+        - name: VIRTLET_IMAGE_TRANSLATIONS_DIR
+          value: /etc/virtlet/images
+        image: mirantis/virtlet:v1.4.1
+        imagePullPolicy: IfNotPresent
+        name: prepare-node
+        resources: {}
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /kubelet-volume-plugins
+          name: k8s-flexvolume-plugins-dir
+        - mountPath: /run
+          name: run
+        - mountPath: /var/run/docker.sock
+          name: dockersock
+        - mountPath: /hostlog
+          name: log
+        - mountPath: /host-var-lib
+          name: var-lib
+        - mountPath: /dev
+          name: dev
+        - mountPath: /var/lib/virtlet
+          name: virtlet
+      serviceAccountName: virtlet
+      volumes:
+      - hostPath:
+          path: /dev
+        name: dev
+      - hostPath:
+          path: /sys/fs/cgroup
+        name: cgroup
+      - hostPath:
+          path: /lib/modules
+        name: modules
+      - hostPath:
+          path: /boot
+        name: boot
+      - hostPath:
+          path: /run
+        name: run
+      - hostPath:
+          path: /var/run/docker.sock
+        name: dockersock
+      - hostPath:
+          path: /var/lib/virtlet
+        name: virtlet
+      - hostPath:
+          path: /var/lib/libvirt
+        name: libvirt
+      - hostPath:
+          path: /var/log
+        name: log
+      - hostPath:
+          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+        name: k8s-flexvolume-plugins-dir
+      - hostPath:
+          path: /var/lib/kubelet/pods
+        name: k8s-pods-dir
+      - hostPath:
+          path: /var/lib
+        name: var-lib
+      - hostPath:
+          path: /var/log/virtlet/vms
+        name: vms-log
+      - hostPath:
+          path: /var/log/libvirt
+        name: libvirt-log
+      - hostPath:
+          path: /var/run/libvirt
+        name: libvirt-sockets
+      - hostPath:
+          path: /var/log/pods
+        name: pods-log
+      - hostPath:
+          path: /var/run/netns
+        name: netns-dir
+      - configMap:
+          name: virtlet-image-translations
+        name: image-name-translations
+  updateStrategy: {}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: virtlet
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: virtlet
+subjects:
+- kind: ServiceAccount
+  name: virtlet
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: virtlet
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - nodes
+  verbs:
+  - create
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: configmap-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: virtlet-userdata-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: kubelet-node-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: configmap-reader
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: vm-userdata-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: virtlet-userdata-reader
+subjects:
+- kind: ServiceAccount
+  name: virtlet
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: virtlet-crd
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+- apiGroups:
+  - virtlet.k8s
+  resources:
+  - virtletimagemappings
+  - virtletconfigmappings
+  verbs:
+  - list
+  - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  creationTimestamp: null
+  name: virtlet-crd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: virtlet-crd
+subjects:
+- kind: ServiceAccount
+  name: virtlet
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  creationTimestamp: null
+  name: virtlet
+  namespace: kube-system
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  labels:
+    virtlet.cloud: ""
+  name: virtletimagemappings.virtlet.k8s
+spec:
+  group: virtlet.k8s
+  names:
+    kind: VirtletImageMapping
+    plural: virtletimagemappings
+    shortNames:
+    - vim
+    singular: virtletimagemapping
+  scope: Namespaced
+  version: v1
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  labels:
+    virtlet.cloud: ""
+  name: virtletconfigmappings.virtlet.k8s
+spec:
+  group: virtlet.k8s
+  names:
+    kind: VirtletConfigMapping
+    plural: virtletconfigmappings
+    shortNames:
+    - vcm
+    singular: virtletconfigmapping
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          properties:
+            config:
+              properties:
+                calicoSubnetSize:
+                  maximum: 32
+                  minimum: 0
+                  type: integer
+                cniConfigDir:
+                  type: string
+                cniPluginDir:
+                  type: string
+                cpuModel:
+                  type: string
+                criSocketPath:
+                  type: string
+                databasePath:
+                  type: string
+                disableKVM:
+                  type: boolean
+                disableLogging:
+                  type: boolean
+                downloadProtocol:
+                  pattern: ^https?$
+                  type: string
+                enableRegexpImageTranslation:
+                  type: boolean
+                enableSriov:
+                  type: boolean
+                fdServerSocketPath:
+                  type: string
+                imageDir:
+                  type: string
+                imageTranslationConfigsDir:
+                  type: string
+                kubeletRootDir:
+                  type: string
+                libvirtURI:
+                  type: string
+                logLevel:
+                  maximum: 2147483647
+                  minimum: 0
+                  type: integer
+                rawDevices:
+                  type: string
+                skipImageTranslation:
+                  type: boolean
+                streamPort:
+                  maximum: 65535
+                  minimum: 1
+                  type: integer
+            nodeName:
+              type: string
+            nodeSelector:
+              type: object
+            priority:
+              type: integer
+  version: v1
+
diff --git a/src/vagrant/kubeadm/virtlet/virtlet.sh b/src/vagrant/kubeadm/virtlet/virtlet.sh
new file mode 100755
index 0000000..505d7f0
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+kubectl label node worker1 extraRuntime=virtlet
+kubectl label node worker2 extraRuntime=virtlet
+kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y
+kubectl create configmap -n kube-system virtlet-image-translations --from-file /vagrant/virtlet/images.yaml
+kubectl create -f /vagrant/virtlet/virtlet-ds.yaml
+
+kubectl delete pod --all
+kubectl create -f /vagrant/virtlet/cirros-vm.yaml
+r="0"
+while [ $r -ne "1" ]
+do
+   r=$(kubectl get pods | grep Running | wc -l)
+   sleep 60
+done
+sleep 60
+kubectl get pods -o json | grep podIP | cut -f4 -d'"' | xargs ping -c 4
+echo 'login by user:cirros & password:gocubsgo'
diff --git a/src/vagrant/kubeadm/virtlet/virtlet_setup.sh b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh
new file mode 100644
index 0000000..b2dfaa0
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+wget https://github.com/Mirantis/criproxy/releases/download/v0.14.0/criproxy_0.14.0_amd64.deb
+echo "criproxy criproxy/primary_cri select containerd" | sudo debconf-set-selections
+sudo dpkg -i criproxy_0.14.0_amd64.deb
+sudo sed -i "s/EnvironmentFile/#EnvironmentFile/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+sudo systemctl daemon-reload
+sudo systemctl restart kubelet
diff --git a/src/vagrant/kubeadm/worker_setup.sh b/src/vagrant/kubeadm/worker_setup.sh
index 5f8d846..6b08712 100644
--- a/src/vagrant/kubeadm/worker_setup.sh
+++ b/src/vagrant/kubeadm/worker_setup.sh
@@ -3,5 +3,6 @@
 set -ex
 
 bash /vagrant/kata/kata_setup.sh
+bash /vagrant/virtlet/virtlet_setup.sh
 sleep 120
 sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443