summaryrefslogtreecommitdiffstats
path: root/src/vagrant/kubeadm
diff options
context:
space:
mode:
authorGuo Ruijing <ruijing.guo@intel.com>2019-02-16 00:16:49 -0500
committerGuo Ruijing <ruijing.guo@intel.com>2019-02-16 00:22:10 -0500
commit8d3c1306b354d63ec04f6372580afbeca147fe9a (patch)
tree205fd34c5ee5843b43b04e6d9599a9e0b821ed87 /src/vagrant/kubeadm
parente5fa8a647c41677cd5ccf63aa199b81bb738249c (diff)
enable virtlet & kata
Change-Id: I88f7b579a825ba7ad25da5074773e5668f0ef000 Signed-off-by: Guo Ruijing <ruijing.guo@intel.com>
Diffstat (limited to 'src/vagrant/kubeadm')
-rw-r--r--src/vagrant/kubeadm/Vagrantfile2
-rwxr-xr-xsrc/vagrant/kubeadm/deploy.sh4
-rw-r--r--src/vagrant/kubeadm/virtlet/cirros-vm.yaml42
-rw-r--r--src/vagrant/kubeadm/virtlet/images.yaml3
-rw-r--r--src/vagrant/kubeadm/virtlet/virtlet-ds.yaml521
-rwxr-xr-xsrc/vagrant/kubeadm/virtlet/virtlet.sh21
-rw-r--r--src/vagrant/kubeadm/virtlet/virtlet_setup.sh10
-rw-r--r--src/vagrant/kubeadm/worker_setup.sh1
8 files changed, 601 insertions, 3 deletions
diff --git a/src/vagrant/kubeadm/Vagrantfile b/src/vagrant/kubeadm/Vagrantfile
index e861859..dc5efb1 100644
--- a/src/vagrant/kubeadm/Vagrantfile
+++ b/src/vagrant/kubeadm/Vagrantfile
@@ -11,7 +11,7 @@ Vagrant.configure("2") do |config|
config.vm.define "registry" do |config|
config.vm.hostname = "registry"
- config.vm.provision "shell", path: "registry_setup.sh", privileged: false
+ #config.vm.provision "shell", path: "registry_setup.sh", privileged: false
config.vm.network :private_network, ip: "192.168.1.5"
end
diff --git a/src/vagrant/kubeadm/deploy.sh b/src/vagrant/kubeadm/deploy.sh
index 0154031..82b2483 100755
--- a/src/vagrant/kubeadm/deploy.sh
+++ b/src/vagrant/kubeadm/deploy.sh
@@ -6,6 +6,6 @@ DIR="$(dirname `readlink -f $0`)"
cd $DIR
../cleanup.sh
vagrant up
-vagrant ssh master -c "/vagrant/istio/istio.sh"
-vagrant ssh master -c "/vagrant/multus/multus.sh"
vagrant ssh master -c "/vagrant/kata/nginx-app.sh"
+vagrant ssh master -c "/vagrant/virtlet/virtlet.sh"
+vagrant ssh master -c "/vagrant/istio/istio.sh"
diff --git a/src/vagrant/kubeadm/virtlet/cirros-vm.yaml b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml
new file mode 100644
index 0000000..334142b
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml
@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Pod
+metadata:
+ name: cirros-vm
+ annotations:
+ # This tells CRI Proxy that this pod belongs to Virtlet runtime
+ kubernetes.io/target-runtime: virtlet.cloud
+ # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason
+ VirtletDiskDriver: virtio
+ # inject ssh keys via cloud-init
+ VirtletSSHKeys: |
+ ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+ # set root volume size
+ VirtletRootVolumeSize: 1Gi
+spec:
+ # This nodeAffinity specification tells Kubernetes to run this
+ # pod only on the nodes that have extraRuntime=virtlet label.
+ # This label is used by Virtlet DaemonSet to select nodes
+ # that must have Virtlet runtime
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: extraRuntime
+ operator: In
+ values:
+ - virtlet
+ containers:
+ - name: cirros-vm
+ # This specifies the image to use.
+ # virtlet.cloud/ prefix is used by CRI proxy, the remaining part
+ # of the image name is prepended with https:// and used to download the image
+ image: virtlet.cloud/cirros
+ imagePullPolicy: IfNotPresent
+ # tty and stdin required for `kubectl attach -t` to work
+ tty: true
+ stdin: true
+ resources:
+ limits:
+ # This memory limit is applied to the libvirt domain definition
+ memory: 160Mi
diff --git a/src/vagrant/kubeadm/virtlet/images.yaml b/src/vagrant/kubeadm/virtlet/images.yaml
new file mode 100644
index 0000000..1541ca7
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/images.yaml
@@ -0,0 +1,3 @@
+translations:
+ - name: cirros
+ url: https://github.com/mirantis/virtlet/releases/download/v0.9.3/cirros.img
diff --git a/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml
new file mode 100644
index 0000000..1bb4882
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml
@@ -0,0 +1,521 @@
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ creationTimestamp: null
+ name: virtlet
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ runtime: virtlet
+ template:
+ metadata:
+ creationTimestamp: null
+ labels:
+ runtime: virtlet
+ name: virtlet
+ spec:
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: extraRuntime
+ operator: In
+ values:
+ - virtlet
+ containers:
+ - command:
+ - /libvirt.sh
+ image: mirantis/virtlet:v1.4.1
+ imagePullPolicy: IfNotPresent
+ name: libvirt
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null
+ resources: {}
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /sys/fs/cgroup
+ name: cgroup
+ - mountPath: /lib/modules
+ name: modules
+ readOnly: true
+ - mountPath: /boot
+ name: boot
+ readOnly: true
+ - mountPath: /run
+ name: run
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
+ - mountPath: /var/run/libvirt
+ name: libvirt-sockets
+ - mountPath: /var/log/vms
+ name: vms-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
+ - mountPath: /dev
+ name: dev
+ - image: mirantis/virtlet:v1.4.1
+ imagePullPolicy: IfNotPresent
+ name: virtlet
+ readinessProbe:
+ exec:
+ command:
+ - /bin/sh
+ - -c
+ - socat - UNIX:/run/virtlet.sock </dev/null
+ resources: {}
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /run
+ name: run
+ - mountPath: /lib/modules
+ name: modules
+ readOnly: true
+ - mountPath: /boot
+ name: boot
+ readOnly: true
+ - mountPath: /dev
+ name: dev
+ - mountPath: /var/lib/virtlet
+ mountPropagation: Bidirectional
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
+ - mountPath: /var/run/libvirt
+ name: libvirt-sockets
+ - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+ name: k8s-flexvolume-plugins-dir
+ - mountPath: /var/lib/kubelet/pods
+ mountPropagation: Bidirectional
+ name: k8s-pods-dir
+ - mountPath: /var/log/vms
+ name: vms-log
+ - mountPath: /etc/virtlet/images
+ name: image-name-translations
+ - mountPath: /var/log/pods
+ name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
+ - mountPath: /var/run/netns
+ mountPropagation: Bidirectional
+ name: netns-dir
+ - command:
+ - /vms.sh
+ image: mirantis/virtlet:v1.4.1
+ imagePullPolicy: IfNotPresent
+ name: vms
+ resources: {}
+ volumeMounts:
+ - mountPath: /var/lib/virtlet
+ mountPropagation: HostToContainer
+ name: virtlet
+ - mountPath: /var/lib/libvirt
+ name: libvirt
+ - mountPath: /var/log/vms
+ name: vms-log
+ - mountPath: /var/lib/kubelet/pods
+ mountPropagation: HostToContainer
+ name: k8s-pods-dir
+ - mountPath: /dev
+ name: dev
+ - mountPath: /lib/modules
+ name: modules
+ dnsPolicy: ClusterFirstWithHostNet
+ hostNetwork: true
+ hostPID: true
+ initContainers:
+ - command:
+ - /prepare-node.sh
+ env:
+ - name: KUBE_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: VIRTLET_DISABLE_KVM
+ valueFrom:
+ configMapKeyRef:
+ key: disable_kvm
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_SRIOV_SUPPORT
+ valueFrom:
+ configMapKeyRef:
+ key: sriov_support
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_DOWNLOAD_PROTOCOL
+ valueFrom:
+ configMapKeyRef:
+ key: download_protocol
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_LOGLEVEL
+ valueFrom:
+ configMapKeyRef:
+ key: loglevel
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_CALICO_SUBNET
+ valueFrom:
+ configMapKeyRef:
+ key: calico-subnet
+ name: virtlet-config
+ optional: true
+ - name: IMAGE_REGEXP_TRANSLATION
+ valueFrom:
+ configMapKeyRef:
+ key: image_regexp_translation
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_RAW_DEVICES
+ valueFrom:
+ configMapKeyRef:
+ key: raw_devices
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_DISABLE_LOGGING
+ valueFrom:
+ configMapKeyRef:
+ key: disable_logging
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_CPU_MODEL
+ valueFrom:
+ configMapKeyRef:
+ key: cpu-model
+ name: virtlet-config
+ optional: true
+ - name: KUBELET_ROOT_DIR
+ valueFrom:
+ configMapKeyRef:
+ key: kubelet_root_dir
+ name: virtlet-config
+ optional: true
+ - name: VIRTLET_IMAGE_TRANSLATIONS_DIR
+ value: /etc/virtlet/images
+ image: mirantis/virtlet:v1.4.1
+ imagePullPolicy: IfNotPresent
+ name: prepare-node
+ resources: {}
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /kubelet-volume-plugins
+ name: k8s-flexvolume-plugins-dir
+ - mountPath: /run
+ name: run
+ - mountPath: /var/run/docker.sock
+ name: dockersock
+ - mountPath: /hostlog
+ name: log
+ - mountPath: /host-var-lib
+ name: var-lib
+ - mountPath: /dev
+ name: dev
+ - mountPath: /var/lib/virtlet
+ name: virtlet
+ serviceAccountName: virtlet
+ volumes:
+ - hostPath:
+ path: /dev
+ name: dev
+ - hostPath:
+ path: /sys/fs/cgroup
+ name: cgroup
+ - hostPath:
+ path: /lib/modules
+ name: modules
+ - hostPath:
+ path: /boot
+ name: boot
+ - hostPath:
+ path: /run
+ name: run
+ - hostPath:
+ path: /var/run/docker.sock
+ name: dockersock
+ - hostPath:
+ path: /var/lib/virtlet
+ name: virtlet
+ - hostPath:
+ path: /var/lib/libvirt
+ name: libvirt
+ - hostPath:
+ path: /var/log
+ name: log
+ - hostPath:
+ path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+ name: k8s-flexvolume-plugins-dir
+ - hostPath:
+ path: /var/lib/kubelet/pods
+ name: k8s-pods-dir
+ - hostPath:
+ path: /var/lib
+ name: var-lib
+ - hostPath:
+ path: /var/log/virtlet/vms
+ name: vms-log
+ - hostPath:
+ path: /var/log/libvirt
+ name: libvirt-log
+ - hostPath:
+ path: /var/run/libvirt
+ name: libvirt-sockets
+ - hostPath:
+ path: /var/log/pods
+ name: pods-log
+ - hostPath:
+ path: /var/run/netns
+ name: netns-dir
+ - configMap:
+ name: virtlet-image-translations
+ name: image-name-translations
+ updateStrategy: {}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ creationTimestamp: null
+ name: virtlet
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: virtlet
+subjects:
+- kind: ServiceAccount
+ name: virtlet
+ namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ creationTimestamp: null
+ name: virtlet
+ namespace: kube-system
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ - nodes
+ verbs:
+ - create
+ - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ creationTimestamp: null
+ name: configmap-reader
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ creationTimestamp: null
+ name: virtlet-userdata-reader
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ - secrets
+ verbs:
+ - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ creationTimestamp: null
+ name: kubelet-node-binding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: configmap-reader
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+ kind: Group
+ name: system:nodes
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ creationTimestamp: null
+ name: vm-userdata-binding
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: virtlet-userdata-reader
+subjects:
+- kind: ServiceAccount
+ name: virtlet
+ namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+ creationTimestamp: null
+ name: virtlet-crd
+rules:
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - create
+- apiGroups:
+ - virtlet.k8s
+ resources:
+ - virtletimagemappings
+ - virtletconfigmappings
+ verbs:
+ - list
+ - get
+
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ creationTimestamp: null
+ name: virtlet-crd
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: virtlet-crd
+subjects:
+- kind: ServiceAccount
+ name: virtlet
+ namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ creationTimestamp: null
+ name: virtlet
+ namespace: kube-system
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ creationTimestamp: null
+ labels:
+ virtlet.cloud: ""
+ name: virtletimagemappings.virtlet.k8s
+spec:
+ group: virtlet.k8s
+ names:
+ kind: VirtletImageMapping
+ plural: virtletimagemappings
+ shortNames:
+ - vim
+ singular: virtletimagemapping
+ scope: Namespaced
+ version: v1
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ creationTimestamp: null
+ labels:
+ virtlet.cloud: ""
+ name: virtletconfigmappings.virtlet.k8s
+spec:
+ group: virtlet.k8s
+ names:
+ kind: VirtletConfigMapping
+ plural: virtletconfigmappings
+ shortNames:
+ - vcm
+ singular: virtletconfigmapping
+ scope: Namespaced
+ validation:
+ openAPIV3Schema:
+ properties:
+ spec:
+ properties:
+ config:
+ properties:
+ calicoSubnetSize:
+ maximum: 32
+ minimum: 0
+ type: integer
+ cniConfigDir:
+ type: string
+ cniPluginDir:
+ type: string
+ cpuModel:
+ type: string
+ criSocketPath:
+ type: string
+ databasePath:
+ type: string
+ disableKVM:
+ type: boolean
+ disableLogging:
+ type: boolean
+ downloadProtocol:
+ pattern: ^https?$
+ type: string
+ enableRegexpImageTranslation:
+ type: boolean
+ enableSriov:
+ type: boolean
+ fdServerSocketPath:
+ type: string
+ imageDir:
+ type: string
+ imageTranslationConfigsDir:
+ type: string
+ kubeletRootDir:
+ type: string
+ libvirtURI:
+ type: string
+ logLevel:
+ maximum: 2147483647
+ minimum: 0
+ type: integer
+ rawDevices:
+ type: string
+ skipImageTranslation:
+ type: boolean
+ streamPort:
+ maximum: 65535
+ minimum: 1
+ type: integer
+ nodeName:
+ type: string
+ nodeSelector:
+ type: object
+ priority:
+ type: integer
+ version: v1
+
diff --git a/src/vagrant/kubeadm/virtlet/virtlet.sh b/src/vagrant/kubeadm/virtlet/virtlet.sh
new file mode 100755
index 0000000..505d7f0
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -ex
+
+kubectl label node worker1 extraRuntime=virtlet
+kubectl label node worker2 extraRuntime=virtlet
+kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y
+kubectl create configmap -n kube-system virtlet-image-translations --from-file /vagrant/virtlet/images.yaml
+kubectl create -f /vagrant/virtlet/virtlet-ds.yaml
+
+kubectl delete pod --all
+kubectl create -f /vagrant/virtlet/cirros-vm.yaml
+r="0"
+while [ $r -ne "1" ]
+do
+ r=$(kubectl get pods | grep Running | wc -l)
+ sleep 60
+done
+sleep 60
+kubectl get pods -o json | grep podIP | cut -f4 -d'"' | xargs ping -c 4
+echo 'login by user:cirros & password:gocubsgo'
diff --git a/src/vagrant/kubeadm/virtlet/virtlet_setup.sh b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh
new file mode 100644
index 0000000..b2dfaa0
--- /dev/null
+++ b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+wget https://github.com/Mirantis/criproxy/releases/download/v0.14.0/criproxy_0.14.0_amd64.deb
+echo "criproxy criproxy/primary_cri select containerd" | sudo debconf-set-selections
+sudo dpkg -i criproxy_0.14.0_amd64.deb
+sudo sed -i "s/EnvironmentFile/#EnvironmentFile/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+sudo systemctl daemon-reload
+sudo systemctl restart kubelet
diff --git a/src/vagrant/kubeadm/worker_setup.sh b/src/vagrant/kubeadm/worker_setup.sh
index 5f8d846..6b08712 100644
--- a/src/vagrant/kubeadm/worker_setup.sh
+++ b/src/vagrant/kubeadm/worker_setup.sh
@@ -3,5 +3,6 @@
set -ex
bash /vagrant/kata/kata_setup.sh
+bash /vagrant/virtlet/virtlet_setup.sh
sleep 120
sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443