support virtlet in container4nfv

Change-Id: I913149ecf374a0a8f0d0bbb65aff52854401bbd6
Signed-off-by: Guo Ruijing <ruijing.guo@intel.com>

6 files changed, 470 insertions, 0 deletions

diff --git a/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml
new file mode 100644
index 0000000..8beb03f
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/cirros-vm.yaml
@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cirros-vm
+  annotations:
+    # This tells CRI Proxy that this pod belongs to Virtlet runtime
+    kubernetes.io/target-runtime: virtlet
+    # An optional annotation specifying the count of virtual CPUs.
+    # Note that annotation values must always be strings,
+    # thus numeric values need to be quoted.
+    # Defaults to "1".
+    VirtletVCPUCount: "1"
+    # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
+    VirtletDiskDriver: virtio
+    # inject ssh keys via cloud-init
+    VirtletSSHKeys: |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+    # cloud-init user data
+    VirtletCloudInitUserDataScript: |
+      #!/bin/sh
+      echo "Hi there"
+spec:
+  # This nodeAffinity specification tells Kubernetes to run this
+  # pod only on the nodes that have extraRuntime=virtlet label.
+  # This label is used by Virtlet DaemonSet to select nodes
+  # that must have Virtlet runtime
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: extraRuntime
+            operator: In
+            values:
+            - virtlet
+  containers:
+  - name: cirros-vm
+    # This specifies the image to use.
+    # virtlet/ prefix is used by CRI proxy, the remaining part
+    # of the image name is prepended with https:// and used to download the image
+    image: virtlet/cirros
+    # Virtlet currently ignores image tags, but their meaning may change
+    # in future, so it’s better not to set them for VM pods. If there’s no tag
+    # provided in the image specification kubelet defaults to
+    # imagePullPolicy: Always, which means that the image is always
+    # redownloaded when the pod is created. In order to make pod creation
+    # faster and more reliable, we set imagePullPolicy to IfNotPresent here
+    # so a previously downloaded image is reused if there is one
+    # in Virtlet’s image store
+    imagePullPolicy: IfNotPresent
+    # tty and stdin required for `kubectl attach -t` to work
+    tty: true
+    stdin: true
+    resources:
+      limits:
+        # This memory limit is applied to the libvirt domain definition
+        memory: 160Mi
diff --git a/src/vagrant/kubeadm_virtlet/examples/images.yaml b/src/vagrant/kubeadm_virtlet/examples/images.yaml
new file mode 100644
index 0000000..3a84585
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/images.yaml
@@ -0,0 +1,3 @@
+translations:
+  - name: cirros
+    url: http://github.com/mirantis/virtlet/releases/download/v0.8.2/cirros.img
diff --git a/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh b/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh
new file mode 100755
index 0000000..bfd0613
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/nginx-app.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+kubectl create -f /vagrant/examples/nginx-app.yaml
+kubectl get nodes
+kubectl get services
+kubectl get pods
+kubectl get rc
+sleep 120
+svcip=$(kubectl get services nginx  -o json | grep clusterIP | cut -f4 -d'"')
+wget http://$svcip
diff --git a/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml b/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml
new file mode 100644
index 0000000..f80881a
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/nginx-app.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    protocol: TCP
+    name: http
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: nginx
+        ports:
+        - containerPort: 80
diff --git a/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml
new file mode 100644
index 0000000..ed037d9
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/virtlet-ds.yaml
@@ -0,0 +1,360 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: virtlet
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      name: virtlet
+      labels:
+        runtime: virtlet
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      # hostPID is true to (1) enable VMs to survive virtlet container restart
+      # (to be checked) and (2) to enable the use of nsenter in init container
+      hostPID: true
+      # bootstrap procedure needs to create a configmap in kube-system namespace
+      serviceAccountName: virtlet
+
+      # only run Virtlet pods on the nodes with extraRuntime=virtlet label
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: extraRuntime
+                operator: In
+                values:
+                - virtlet
+
+      initContainers:
+      # The init container first copies virtlet's flexvolume driver
+      # to the default kubelet plugin dir to have it in the proper place by the
+      # time kubelet is restarted by CRI proxy bootstrap procedure.
+      # After that it checks if there's already saved kubelet config
+      # and considers that CRI proxy bootstrap is already done if it exists.
+      # If it doesn't, it drops criproxy binary into /opt/criproxy/bin
+      # if it's not already there and then starts criproxy installation.
+      # The possibility to put criproxy binary in advance into
+      # /opt/criproxy/bin may be helpful for the purpose of
+      # debugging criproxy
+      # At the end it ensures that /var/lib/libvirt/images exists on node.
+      - name: prepare-node
+        image: openretriever/virtlet
+        imagePullPolicy: IfNotPresent
+        command:
+        - /prepare-node.sh
+        volumeMounts:
+        - name: k8s-flexvolume-plugins-dir
+          mountPath: /kubelet-volume-plugins
+        - name: criproxybin
+          mountPath: /opt/criproxy/bin
+        - name: run
+          mountPath: /run
+        - name: dockersock
+          mountPath: /var/run/docker.sock
+        - name: criproxyconf
+          mountPath: /etc/criproxy
+        - name: log
+          mountPath: /hostlog
+        # for ensuring that /var/lib/libvirt/images exists on node
+        - name: var-lib
+          mountPath: /host-var-lib
+        securityContext:
+          privileged: true
+
+      containers:
+      - name: libvirt
+        image: openretriever/virtlet
+        # In case we inject local virtlet image we want to use it not officially available one
+        imagePullPolicy: IfNotPresent
+        command:
+        - /libvirt.sh
+        volumeMounts:
+        - mountPath: /sys/fs/cgroup
+          name: cgroup
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+        - mountPath: /boot
+          name: boot
+          readOnly: true
+        - mountPath: /run
+          name: run
+        - mountPath: /var/lib/virtlet
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - mountPath: /var/run/libvirt
+          name: libvirt-sockets
+        # the log dir is needed here because otherwise libvirt will produce errors
+        # like this:
+        # Unable to pre-create chardev file '/var/log/vms/afd75bbb-8e97-11e7-9561-02420ac00002/cirros-vm_0.log': No such file or directory
+        - name: vms-log
+          mountPath: /var/log/vms
+        - name: dev
+          mountPath: /dev
+        securityContext:
+          privileged: true
+        env:
+        - name: VIRTLET_DISABLE_KVM
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: disable_kvm
+              optional: true
+      - name: virtlet
+        image: openretriever/virtlet
+        # In case we inject local virtlet image we want to use it not officially available one
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - mountPath: /run
+          name: run
+        # /boot and /lib/modules are required by supermin
+        - mountPath: /lib/modules
+          name: modules
+          readOnly: true
+        - mountPath: /boot
+          name: boot
+          readOnly: true
+        - mountPath: /var/lib/virtlet
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - mountPath: /etc/cni
+          name: cniconf
+        - mountPath: /opt/cni/bin
+          name: cnibin
+        - mountPath: /var/run/libvirt
+          name: libvirt-sockets
+        - mountPath: /var/lib/cni
+          name: cnidata
+        - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+          name: k8s-flexvolume-plugins-dir
+          # below `:shared` is unofficial way to pass this option docker
+          # which then will allow virtlet to see what kubelet mounts in
+          # underlaying directories, after virtlet container is created
+        - mountPath: /var/lib/kubelet/pods:shared
+          name: k8s-pods-dir
+        - name: vms-log
+          mountPath: /var/log/vms
+        - mountPath: /etc/virtlet/images
+          name: image-name-translations
+        - name: pods-log
+          mountPath: /kubernetes-log
+        securityContext:
+          privileged: true
+        env:
+        - name: VIRTLET_DISABLE_KVM
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: disable_kvm
+              optional: true
+        - name: VIRTLET_DOWNLOAD_PROTOCOL
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: download_protocol
+              optional: true
+        - name: VIRTLET_LOGLEVEL
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: loglevel
+              optional: true
+        - name: VIRTLET_CALICO_SUBNET
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: calico-subnet
+              optional: true
+        - name: IMAGE_REGEXP_TRANSLATION
+          valueFrom:
+            configMapKeyRef:
+              name: virtlet-config
+              key: image_regexp_translation
+              optional: true
+        - name: IMAGE_TRANSLATIONS_DIR
+          value: /etc/virtlet/images
+        - name: KUBERNETES_POD_LOGS
+          value: "/kubernetes-log"
+        # TODO: should we rename it?
+        - name: VIRTLET_VM_LOG_LOCATION
+          value: "1"
+      - name: vms
+        image: openretriever/virtlet
+        imagePullPolicy: IfNotPresent
+        command:
+        - /vms.sh
+        volumeMounts:
+        - mountPath: /var/lib/virtlet
+          name: virtlet
+        - mountPath: /var/lib/libvirt
+          name: libvirt
+        - name: vms-log
+          mountPath: /var/log/vms
+        - name: dev
+          mountPath: /dev
+      volumes:
+      # /dev is needed for host raw device access
+      - hostPath:
+          path: /dev
+        name: dev
+      - hostPath:
+          path: /sys/fs/cgroup
+        name: cgroup
+      - hostPath:
+          path: /lib/modules
+        name: modules
+      - hostPath:
+          path: /boot
+        name: boot
+      - hostPath:
+          path: /run
+        name: run
+      # TODO: don't hardcode docker socket location here
+      # This will require CRI proxy installation to run
+      # in host mount namespace.
+      - hostPath:
+          path: /var/run/docker.sock
+        name: dockersock
+      - hostPath:
+          path: /var/lib/virtlet
+        name: virtlet
+      - hostPath:
+          path: /var/lib/libvirt
+        name: libvirt
+      - hostPath:
+          path: /etc/cni
+        name: cniconf
+      - hostPath:
+          path: /opt/cni/bin
+        name: cnibin
+      - hostPath:
+          path: /var/lib/cni
+        name: cnidata
+      - hostPath:
+          path: /opt/criproxy/bin
+        name: criproxybin
+      - hostPath:
+          path: /etc/criproxy
+        name: criproxyconf
+      - hostPath:
+          path: /var/log
+        name: log
+      - hostPath:
+          path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+        name: k8s-flexvolume-plugins-dir
+      - hostPath:
+          path: /var/lib/kubelet/pods
+        name: k8s-pods-dir
+      - hostPath:
+          path: /var/lib
+        name: var-lib
+      - hostPath:
+          path: /var/log/virtlet/vms
+        name: vms-log
+      - hostPath:
+          path: /var/run/libvirt
+        name: libvirt-sockets
+      - hostPath:
+          path: /var/log/pods
+        name: pods-log
+      - configMap:
+          name: virtlet-image-translations
+        name: image-name-translations
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: virtlet
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: virtlet
+subjects:
+- kind: ServiceAccount
+  name: virtlet
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: virtlet
+  namespace: kube-system
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - create
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: configmap-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: kubelet-node-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: configmap-reader
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: virtlet-crd
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - customresourcedefinitions
+    verbs:
+      - create
+  - apiGroups:
+      - "virtlet.k8s"
+    resources:
+      - virtletimagemappings
+    verbs:
+      - list
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: virtlet-crd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: virtlet-crd
+subjects:
+- kind: ServiceAccount
+  name: virtlet
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: virtlet
+  namespace: kube-system
diff --git a/src/vagrant/kubeadm_virtlet/examples/virtlet.sh b/src/vagrant/kubeadm_virtlet/examples/virtlet.sh
new file mode 100755
index 0000000..68d738d
--- /dev/null
+++ b/src/vagrant/kubeadm_virtlet/examples/virtlet.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+kubectl label node worker1 extraRuntime=virtlet
+kubectl label node worker2 extraRuntime=virtlet
+kubectl label node worker3 extraRuntime=virtlet
+kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y
+kubectl create configmap -n kube-system virtlet-image-translations --from-file images.yaml
+kubectl create -f virtlet-ds.yaml
+kubectl create -f cirros-vm.yaml