diff options
| -rw-r--r-- | src/vagrant/kubeadm/Vagrantfile | 2 | ||||
| -rwxr-xr-x | src/vagrant/kubeadm/deploy.sh | 4 | ||||
| -rw-r--r-- | src/vagrant/kubeadm/virtlet/cirros-vm.yaml | 42 | ||||
| -rw-r--r-- | src/vagrant/kubeadm/virtlet/images.yaml | 3 | ||||
| -rw-r--r-- | src/vagrant/kubeadm/virtlet/virtlet-ds.yaml | 521 | ||||
| -rwxr-xr-x | src/vagrant/kubeadm/virtlet/virtlet.sh | 21 | ||||
| -rw-r--r-- | src/vagrant/kubeadm/virtlet/virtlet_setup.sh | 10 | ||||
| -rw-r--r-- | src/vagrant/kubeadm/worker_setup.sh | 1 |
8 files changed, 601 insertions, 3 deletions
diff --git a/src/vagrant/kubeadm/Vagrantfile b/src/vagrant/kubeadm/Vagrantfile index e861859..dc5efb1 100644 --- a/src/vagrant/kubeadm/Vagrantfile +++ b/src/vagrant/kubeadm/Vagrantfile @@ -11,7 +11,7 @@ Vagrant.configure("2") do |config| config.vm.define "registry" do |config| config.vm.hostname = "registry" - config.vm.provision "shell", path: "registry_setup.sh", privileged: false + #config.vm.provision "shell", path: "registry_setup.sh", privileged: false config.vm.network :private_network, ip: "192.168.1.5" end diff --git a/src/vagrant/kubeadm/deploy.sh b/src/vagrant/kubeadm/deploy.sh index 0154031..82b2483 100755 --- a/src/vagrant/kubeadm/deploy.sh +++ b/src/vagrant/kubeadm/deploy.sh @@ -6,6 +6,6 @@ DIR="$(dirname `readlink -f $0`)" cd $DIR ../cleanup.sh vagrant up -vagrant ssh master -c "/vagrant/istio/istio.sh" -vagrant ssh master -c "/vagrant/multus/multus.sh" vagrant ssh master -c "/vagrant/kata/nginx-app.sh" +vagrant ssh master -c "/vagrant/virtlet/virtlet.sh" +vagrant ssh master -c "/vagrant/istio/istio.sh" diff --git a/src/vagrant/kubeadm/virtlet/cirros-vm.yaml b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml new file mode 100644 index 0000000..334142b --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/cirros-vm.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cirros-vm + annotations: + # This tells CRI Proxy that this pod belongs to Virtlet runtime + kubernetes.io/target-runtime: virtlet.cloud + # CirrOS doesn't load nocloud data from SCSI CD-ROM for some reason + VirtletDiskDriver: virtio + # inject ssh keys via cloud-init + VirtletSSHKeys: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost + # set root volume size + VirtletRootVolumeSize: 1Gi +spec: + # This nodeAffinity specification tells Kubernetes to run this + # pod only on the nodes that have extraRuntime=virtlet label. + # This label is used by Virtlet DaemonSet to select nodes + # that must have Virtlet runtime + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + containers: + - name: cirros-vm + # This specifies the image to use. + # virtlet.cloud/ prefix is used by CRI proxy, the remaining part + # of the image name is prepended with https:// and used to download the image + image: virtlet.cloud/cirros + imagePullPolicy: IfNotPresent + # tty and stdin required for `kubectl attach -t` to work + tty: true + stdin: true + resources: + limits: + # This memory limit is applied to the libvirt domain definition + memory: 160Mi diff --git a/src/vagrant/kubeadm/virtlet/images.yaml b/src/vagrant/kubeadm/virtlet/images.yaml new file mode 100644 index 0000000..1541ca7 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/images.yaml @@ -0,0 +1,3 @@ +translations: + - name: cirros + url: https://github.com/mirantis/virtlet/releases/download/v0.9.3/cirros.img diff --git a/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml new file mode 100644 index 0000000..1bb4882 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet-ds.yaml @@ -0,0 +1,521 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system +spec: + selector: + matchLabels: + runtime: virtlet + template: + metadata: + creationTimestamp: null + labels: + runtime: virtlet + name: virtlet + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: extraRuntime + operator: In + values: + - virtlet + containers: + - command: + - /libvirt.sh + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: libvirt + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/var/run/libvirt/libvirt-sock-ro </dev/null + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /run + name: run + - mountPath: /var/lib/virtlet + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/run/libvirt + name: libvirt-sockets + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /dev + name: dev + - image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: virtlet + readinessProbe: + exec: + command: + - /bin/sh + - -c + - socat - UNIX:/run/virtlet.sock </dev/null + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /run + name: run + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /boot + name: boot + readOnly: true + - mountPath: /dev + name: dev + - mountPath: /var/lib/virtlet + mountPropagation: Bidirectional + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/run/libvirt + name: libvirt-sockets + - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + - mountPath: /var/lib/kubelet/pods + mountPropagation: Bidirectional + name: k8s-pods-dir + - mountPath: /var/log/vms + name: vms-log + - mountPath: /etc/virtlet/images + name: image-name-translations + - mountPath: /var/log/pods + name: pods-log + - mountPath: /var/log/libvirt + name: libvirt-log + - mountPath: /var/run/netns + mountPropagation: Bidirectional + name: netns-dir + - command: + - /vms.sh + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: vms + resources: {} + volumeMounts: + - mountPath: /var/lib/virtlet + mountPropagation: HostToContainer + name: virtlet + - mountPath: /var/lib/libvirt + name: libvirt + - mountPath: /var/log/vms + name: vms-log + - mountPath: /var/lib/kubelet/pods + mountPropagation: HostToContainer + name: k8s-pods-dir + - mountPath: /dev + name: dev + - mountPath: /lib/modules + name: modules + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + hostPID: true + initContainers: + - command: + - /prepare-node.sh + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: VIRTLET_DISABLE_KVM + valueFrom: + configMapKeyRef: + key: disable_kvm + name: virtlet-config + optional: true + - name: VIRTLET_SRIOV_SUPPORT + valueFrom: + configMapKeyRef: + key: sriov_support + name: virtlet-config + optional: true + - name: VIRTLET_DOWNLOAD_PROTOCOL + valueFrom: + configMapKeyRef: + key: download_protocol + name: virtlet-config + optional: true + - name: VIRTLET_LOGLEVEL + valueFrom: + configMapKeyRef: + key: loglevel + name: virtlet-config + optional: true + - name: VIRTLET_CALICO_SUBNET + valueFrom: + configMapKeyRef: + key: calico-subnet + name: virtlet-config + optional: true + - name: IMAGE_REGEXP_TRANSLATION + valueFrom: + configMapKeyRef: + key: image_regexp_translation + name: virtlet-config + optional: true + - name: VIRTLET_RAW_DEVICES + valueFrom: + configMapKeyRef: + key: raw_devices + name: virtlet-config + optional: true + - name: VIRTLET_DISABLE_LOGGING + valueFrom: + configMapKeyRef: + key: disable_logging + name: virtlet-config + optional: true + - name: VIRTLET_CPU_MODEL + valueFrom: + configMapKeyRef: + key: cpu-model + name: virtlet-config + optional: true + - name: KUBELET_ROOT_DIR + valueFrom: + configMapKeyRef: + key: kubelet_root_dir + name: virtlet-config + optional: true + - name: VIRTLET_IMAGE_TRANSLATIONS_DIR + value: /etc/virtlet/images + image: mirantis/virtlet:v1.4.1 + imagePullPolicy: IfNotPresent + name: prepare-node + resources: {} + securityContext: + privileged: true + volumeMounts: + - mountPath: /kubelet-volume-plugins + name: k8s-flexvolume-plugins-dir + - mountPath: /run + name: run + - mountPath: /var/run/docker.sock + name: dockersock + - mountPath: /hostlog + name: log + - mountPath: /host-var-lib + name: var-lib + - mountPath: /dev + name: dev + - mountPath: /var/lib/virtlet + name: virtlet + serviceAccountName: virtlet + volumes: + - hostPath: + path: /dev + name: dev + - hostPath: + path: /sys/fs/cgroup + name: cgroup + - hostPath: + path: /lib/modules + name: modules + - hostPath: + path: /boot + name: boot + - hostPath: + path: /run + name: run + - hostPath: + path: /var/run/docker.sock + name: dockersock + - hostPath: + path: /var/lib/virtlet + name: virtlet + - hostPath: + path: /var/lib/libvirt + name: libvirt + - hostPath: + path: /var/log + name: log + - hostPath: + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec + name: k8s-flexvolume-plugins-dir + - hostPath: + path: /var/lib/kubelet/pods + name: k8s-pods-dir + - hostPath: + path: /var/lib + name: var-lib + - hostPath: + path: /var/log/virtlet/vms + name: vms-log + - hostPath: + path: /var/log/libvirt + name: libvirt-log + - hostPath: + path: /var/run/libvirt + name: libvirt-sockets + - hostPath: + path: /var/log/pods + name: pods-log + - hostPath: + path: /var/run/netns + name: netns-dir + - configMap: + name: virtlet-image-translations + name: image-name-translations + updateStrategy: {} + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: virtlet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system +rules: +- apiGroups: + - "" + resources: + - configmaps + - nodes + verbs: + - create + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: configmap-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet-userdata-reader +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: kubelet-node-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: configmap-reader +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:nodes + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: vm-userdata-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-userdata-reader +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: virtlet-crd +rules: +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create +- apiGroups: + - virtlet.k8s + resources: + - virtletimagemappings + - virtletconfigmappings + verbs: + - list + - get + +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + creationTimestamp: null + name: virtlet-crd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: virtlet-crd +subjects: +- kind: ServiceAccount + name: virtlet + namespace: kube-system + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + creationTimestamp: null + name: virtlet + namespace: kube-system + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletimagemappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletImageMapping + plural: virtletimagemappings + shortNames: + - vim + singular: virtletimagemapping + scope: Namespaced + version: v1 + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + labels: + virtlet.cloud: "" + name: virtletconfigmappings.virtlet.k8s +spec: + group: virtlet.k8s + names: + kind: VirtletConfigMapping + plural: virtletconfigmappings + shortNames: + - vcm + singular: virtletconfigmapping + scope: Namespaced + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + properties: + calicoSubnetSize: + maximum: 32 + minimum: 0 + type: integer + cniConfigDir: + type: string + cniPluginDir: + type: string + cpuModel: + type: string + criSocketPath: + type: string + databasePath: + type: string + disableKVM: + type: boolean + disableLogging: + type: boolean + downloadProtocol: + pattern: ^https?$ + type: string + enableRegexpImageTranslation: + type: boolean + enableSriov: + type: boolean + fdServerSocketPath: + type: string + imageDir: + type: string + imageTranslationConfigsDir: + type: string + kubeletRootDir: + type: string + libvirtURI: + type: string + logLevel: + maximum: 2147483647 + minimum: 0 + type: integer + rawDevices: + type: string + skipImageTranslation: + type: boolean + streamPort: + maximum: 65535 + minimum: 1 + type: integer + nodeName: + type: string + nodeSelector: + type: object + priority: + type: integer + version: v1 + diff --git a/src/vagrant/kubeadm/virtlet/virtlet.sh b/src/vagrant/kubeadm/virtlet/virtlet.sh new file mode 100755 index 0000000..505d7f0 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +kubectl label node worker1 extraRuntime=virtlet +kubectl label node worker2 extraRuntime=virtlet +kubectl create configmap -n kube-system virtlet-config --from-literal=download_protocol=http --from-literal=image_regexp_translation=1 --from-literal=disable_kvm=y +kubectl create configmap -n kube-system virtlet-image-translations --from-file /vagrant/virtlet/images.yaml +kubectl create -f /vagrant/virtlet/virtlet-ds.yaml + +kubectl delete pod --all +kubectl create -f /vagrant/virtlet/cirros-vm.yaml +r="0" +while [ $r -ne "1" ] +do + r=$(kubectl get pods | grep Running | wc -l) + sleep 60 +done +sleep 60 +kubectl get pods -o json | grep podIP | cut -f4 -d'"' | xargs ping -c 4 +echo 'login by user:cirros & password:gocubsgo' diff --git a/src/vagrant/kubeadm/virtlet/virtlet_setup.sh b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh new file mode 100644 index 0000000..b2dfaa0 --- /dev/null +++ b/src/vagrant/kubeadm/virtlet/virtlet_setup.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -ex + +wget https://github.com/Mirantis/criproxy/releases/download/v0.14.0/criproxy_0.14.0_amd64.deb +echo "criproxy criproxy/primary_cri select containerd" | sudo debconf-set-selections +sudo dpkg -i criproxy_0.14.0_amd64.deb +sudo sed -i "s/EnvironmentFile/#EnvironmentFile/" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +sudo systemctl daemon-reload +sudo systemctl restart kubelet diff --git a/src/vagrant/kubeadm/worker_setup.sh b/src/vagrant/kubeadm/worker_setup.sh index 5f8d846..6b08712 100644 --- a/src/vagrant/kubeadm/worker_setup.sh +++ b/src/vagrant/kubeadm/worker_setup.sh @@ -3,5 +3,6 @@ set -ex bash /vagrant/kata/kata_setup.sh +bash /vagrant/virtlet/virtlet_setup.sh sleep 120 sudo kubeadm join --discovery-token-unsafe-skip-ca-verification --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 |