From 87e6c1419cab82ce5b0f828c360ac38120103993 Mon Sep 17 00:00:00 2001
From: trevortao <trevor.tao@arm.com>
Date: Thu, 25 Jan 2018 14:22:12 +0800
Subject: Add Kubernetes deployment scenario for arm server

1. README.rst: installation guide
2. Scenario1: Kubernetes_basic, a basic Kubernetes cluster
   with 2 nodes and Flannel CNI plugin, including the
   deployment scripts,  and an example use case

We may add more Kubernetes scenarios in the future

Change-Id: I3a0189533fdf99f0a80fd76c9e6cf96ff4a7e9f0
Signed-off-by: trevortao <trevor.tao@arm.com>
---
 src/arm/README.rst                               | 85 ++++++++++++++++++++++++
 src/arm/kubernetes_basic/examples/nginx-app.sh   | 42 ++++++++++++
 src/arm/kubernetes_basic/examples/nginx-app.yaml | 31 +++++++++
 src/arm/kubernetes_basic/k8s-build.sh            | 31 +++++++++
 src/arm/kubernetes_basic/k8s-deploy.sh           | 34 ++++++++++
 src/arm/kubernetes_basic/setup.sh                |  7 ++
 6 files changed, 230 insertions(+)
 create mode 100644 src/arm/README.rst
 create mode 100755 src/arm/kubernetes_basic/examples/nginx-app.sh
 create mode 100644 src/arm/kubernetes_basic/examples/nginx-app.yaml
 create mode 100755 src/arm/kubernetes_basic/k8s-build.sh
 create mode 100755 src/arm/kubernetes_basic/k8s-deploy.sh
 create mode 100755 src/arm/kubernetes_basic/setup.sh

diff --git a/src/arm/README.rst b/src/arm/README.rst
new file mode 100644
index 0000000..5e3325b
--- /dev/null
+++ b/src/arm/README.rst
@@ -0,0 +1,85 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, arm Limited.
+
+.. _Compass4nfv: https://wiki.opnfv.org/display/compass4nfv/Compass4nfv
+.. _repo: https://gerrit.opnfv.org/gerrit/#/admin/projects/compass4nfv
+.. _Flannel: https://github.com/coreos/flannel
+
+===============================================
+Kubernetes Deployment on arm Installation Guide
+===============================================
+
+Kubernetes deployement on arm64 server is now supported. This document give the basic installation
+guide for common things between different scenarios. The user can refer the installation script in
+each scenario for details.
+Kubernetes deployment scenarios on arm is based on Compass4nfv_. Arm has given patches to enable
+some certain scenarios to the Compass4nfv repo_.
+Typically it would deploy 2 VMs(hosts) in which one(host1) is deployed as Kubernetes Master
+and node, the other(host2) as Kubernetes node.
+The basic Kubernetes cluster would use Flannel_ as the container networking scheme.
+
+
+Installation Enviroment Preparation
+===================================
+
+.. _architecture: 
+   https://github.com/opnfv/compass4nfv/blob/master/docs/release/installation/k8s-deploy.rst
+
+The installed Kubernetes cluster architecture could refer the deployment architecture_ of
+Compass4nfv.
+
+Jumphost: Ubuntu16.04(xenial) aarch64
+
+* *1* Enable password-less sudo for current user
+* *2* Disable DHCP of libvirt default network
+
+::
+     Libvirt creates a default network at intallation, which enables DHCP and occupies port 67.
+     It conflicts with compass-cobbler container.
+     $ sudo virsh net-edit default
+     <!-- remove below lines and save/quit ->
+     <dhcp>
+     <range start='192.168.122.2' end='192.168.122.254'/>
+     </dhcp>
+     $ sudo virsh net-destroy default
+     $ sudo virsh net-start default
+
+* *3* Make sure ports 67,69,80,443 are free
+
+::
+    Compass-cobber requires ports 67, 69 to provide DHCP and TFTP services. Compass-deck provides
+    HTTP and HTTPS through ports 80, 443. All these ports should be free before deployment.
+
+* *4* Teardown apparmo "sudo service apparmor teardown"
+
+
+Installation Steps
+==================
+
+Run the setup script to deploy your needed scenario:
+
+./setup.sh
+
+which would call k8s-build.sh and k8s-deploy.sh to deploy a Kubernetes cluster
+
+Or you can run them separately:
+
+./k8s-build.sh
+./k8s-deploy.sh
+
+The whole installation would take 30min~2hours which mainly depends on how long you get the needed
+images from the network.
+
+
+Contacts
+========
+
+Trevor Tao(Zijin Tao), Yibo Cai, Di Xu and Bin Lu from Arm have made contributions to this document.
+
+Trevor Tao: trevor.tao@arm.com
+Yibo Cai:   yibo.cai@arm.com
+Di Xu:      di.xu@arm.com
+Bin Lu:     bin.lu@arm.com
+
diff --git a/src/arm/kubernetes_basic/examples/nginx-app.sh b/src/arm/kubernetes_basic/examples/nginx-app.sh
new file mode 100755
index 0000000..18aaf93
--- /dev/null
+++ b/src/arm/kubernetes_basic/examples/nginx-app.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright (c) 2017 arm Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Ref: src/vagrant/kubeadm_basic/examples/nginx-app.sh
+#Revised a little
+
+set -ex
+
+kubectl create -f ./examples/nginx-app.yaml
+kubectl get nodes
+kubectl get services
+kubectl get pods
+kubectl get rc
+
+r="0"
+while [ $r -ne "2" ]
+do
+   r=$(kubectl get pods | grep Running | wc -l)
+   sleep 60
+done
+
+svcip=$(kubectl get services nginx  -o json | grep clusterIP | cut -f4 -d'"')
+sleep 10
+wget http://$svcip
+kubectl delete -f ./examples/nginx-app.yaml
+kubectl get rc
+kubectl get pods
+kubectl get services
diff --git a/src/arm/kubernetes_basic/examples/nginx-app.yaml b/src/arm/kubernetes_basic/examples/nginx-app.yaml
new file mode 100644
index 0000000..9b3c54b
--- /dev/null
+++ b/src/arm/kubernetes_basic/examples/nginx-app.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: nginx
+  labels:
+    app: nginx
+spec:
+  type: NodePort
+  ports:
+  - port: 80
+    protocol: TCP
+    name: http
+  selector:
+    app: nginx
+---
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: nginx
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: nginx
+    spec:
+      containers:
+      - name: nginx
+        image: arm64v8/nginx:stable
+        ports:
+        - containerPort: 80
diff --git a/src/arm/kubernetes_basic/k8s-build.sh b/src/arm/kubernetes_basic/k8s-build.sh
new file mode 100755
index 0000000..832a120
--- /dev/null
+++ b/src/arm/kubernetes_basic/k8s-build.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+sudo apt-get install -y docker.io libvirt-bin virt-manager qemu qemu-efi
+
+WORKSPACE=`pwd`
+if [ ! -d "$WORKSPACE/compass4nfv" ]; then
+	git clone https://gerrit.opnfv.org/gerrit/compass4nfv
+fi
+
+cd compass4nfv
+
+WORKSPACE=`pwd`
+
+COMPASS_WORK_DIR=$WORKSPACE/../compass-work
+mkdir -p $COMPASS_WORK_DIR
+if [ ! -d "$WORKSPACE/work" ]; then
+	ln -s $COMPASS_WORK_DIR work
+fi
+
+#TODO: remove workaround after patches merged
+if [ ! -f "$WORKSPACE/patched" ]; then
+
+	git checkout a360411cb8c775dffa24a4157cec2b566cbde6f3
+	curl http://people.linaro.org/~yibo.cai/compass/0001-deploy-cobbler-drop-tcp_tw_recycle-in-sysctl.conf.patch | git apply || true
+	curl http://people.linaro.org/~yibo.cai/compass/0002-docker-compose-support-aarch64.patch | git apply || true
+	touch "$WORKSPACE/patched"
+fi
+
+# build tarball
+COMPASS_ISO_REPO='http://people.linaro.org/~yibo.cai/compass' ./build.sh
diff --git a/src/arm/kubernetes_basic/k8s-deploy.sh b/src/arm/kubernetes_basic/k8s-deploy.sh
new file mode 100755
index 0000000..3b8f6f3
--- /dev/null
+++ b/src/arm/kubernetes_basic/k8s-deploy.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+#sudo apt-get install -y docker.io libvirt-bin virt-manager qemu qemu-efi
+
+dir=`pwd`
+echo "Current dir: "$dir
+curdir=${dir##*/}
+result=$(echo $curdir | grep "compass4nfv")
+if [[ "$result" == "" ]]
+then
+    cd compass4nfv
+fi
+
+
+#export CONTAINER4NFV_SCENARIO={scenario}
+
+export ADAPTER_OS_PATTERN='(?i)CentOS-7.*arm.*'
+export OS_VERSION="centos7"
+export KUBERNETES_VERSION="v1.7.3"
+export DHA="deploy/conf/vm_environment/k8-nosdn-nofeature-noha.yml"
+export NETWORK="deploy/conf/vm_environment/network.yml"
+export VIRT_NUMBER=2 VIRT_CPUS=2 VIRT_MEM=4096 VIRT_DISK=50G
+
+./deploy.sh
+
+set -ex
+
+# basic test: ssh to master, check k8s node status
+sshpass -p root ssh root@10.1.0.50 kubectl get nodes 2>/dev/null | grep -i ready
+
+# Copy examples to master, run the example of nginx
+sshpass -p root scp -r ./examples root@10.1.0.50:~
+sshpass -p root ssh root@10.1.0.50 ~/examples/nginx-app.sh
diff --git a/src/arm/kubernetes_basic/setup.sh b/src/arm/kubernetes_basic/setup.sh
new file mode 100755
index 0000000..b33e990
--- /dev/null
+++ b/src/arm/kubernetes_basic/setup.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+./k8s-build.sh
+
+sleep 2
+
+./k8s-deploy.sh
-- 
cgit 1.2.3-korg