Add csi plugin ansible and tutorial document

As discussed on Compass4NFV meeting this week, we need to add the
support for csi-plugin. So in this patch I extend nbp-ansible to
support integrating with k8s using csi. All the detailed information
can be found in documents.
Besides, since opensds community has updated something in opensds
ansible, we should update ours to keep up with the latest code.

Change-Id: I238c7f7652b6a9c86cc9fcb24d9575d625c5f80a
Signed-off-by: leonwang <wanghui71@huawei.com>

17 files changed, 184 insertions, 36 deletions

diff --git a/ci/ansible/README.md b/ci/ansible/README.md
index 37a22f4..2bf2a80 100644
--- a/ci/ansible/README.md
+++ b/ci/ansible/README.md
@@ -165,7 +165,7 @@ sudo ansible-playbook clean.yml -i local.hosts
 

 ### Run ceph-ansible playbook to clean ceph cluster if ceph is deployed

 ```bash

-cd /tmp/ceph-ansible

+cd /opt/ceph-ansible

 sudo ansible-playbook infrastructure-playbooks/purge-cluster.yml -i ceph.hosts

 ```

 

@@ -174,5 +174,5 @@ In addition, clean up the logical partition on the physical block device used by
 ### Remove ceph-ansible source code (optional)

 ```bash

 cd ..

-sudo rm -rf /tmp/ceph-ansible

+sudo rm -rf /opt/ceph-ansible

 ```

diff --git a/ci/ansible/group_vars/common.yml b/ci/ansible/group_vars/common.yml
index 11fdc80..a67b8c8 100644
--- a/ci/ansible/group_vars/common.yml
+++ b/ci/ansible/group_vars/common.yml
@@ -15,7 +15,7 @@ dummy:
 # GENERAL #

 ###########

 

-workplace: /home/krej # Change this field according to your username

+workplace: /home/krej # Change this field according to your username, use '/root' if you login as root.

 

 # These fields are NOT suggested to be modified

 remote_url: https://github.com/opensds/opensds.git

@@ -23,3 +23,13 @@ opensds_root_dir: "{{ workplace }}/gopath/src/github.com/opensds/opensds"
 opensds_build_dir: "{{ opensds_root_dir }}/build"

 opensds_config_dir: /etc/opensds

 opensds_log_dir: /var/log/opensds

+

+###########

+# GOLANG  #

+###########

+

+golang_release: 1.9.2

+

+# These fields are NOT suggested to be modified

+golang_tarball: go{{ golang_release }}.linux-amd64.tar.gz

+golang_download_url: https://storage.googleapis.com/golang/{{ golang_tarball }}

diff --git a/ci/ansible/group_vars/osdsdb.yml b/ci/ansible/group_vars/osdsdb.yml
index 2ec40fe..f5dfdef 100644
--- a/ci/ansible/group_vars/osdsdb.yml
+++ b/ci/ansible/group_vars/osdsdb.yml
@@ -28,4 +28,4 @@ etcd_release: v3.2.0
 # These fields are not suggested to be modified

 etcd_tarball: etcd-{{ etcd_release }}-linux-amd64.tar.gz

 etcd_download_url: https://github.com/coreos/etcd/releases/download/{{ etcd_release }}/{{ etcd_tarball }}

-etcd_dir: /tmp/etcd-{{ etcd_release }}-linux-amd64

+etcd_dir: /opt/etcd-{{ etcd_release }}-linux-amd64

diff --git a/ci/ansible/roles/cleaner/tasks/main.yml b/ci/ansible/roles/cleaner/tasks/main.yml
index 93aeb59..334d5bb 100644
--- a/ci/ansible/roles/cleaner/tasks/main.yml
+++ b/ci/ansible/roles/cleaner/tasks/main.yml
@@ -1,4 +1,11 @@
 ---

+- name: remove golang tarball

+  file:

+    path: "/opt/{{ golang_tarball }}"

+    state: absent

+    force: yes

+  ignore_errors: yes

+

 - name: kill etcd daemon service

   shell: killall etcd

   ignore_errors: yes

@@ -14,7 +21,7 @@
 

 - name: remove etcd tarball

   file:

-    path: "/tmp/{{ etcd_tarball }}"

+    path: "/opt/{{ etcd_tarball }}"

     state: absent

     force: yes

   ignore_errors: yes

@@ -29,11 +36,9 @@
   ignore_errors: yes

 

 - name: clean all opensds build files

-  file:

-    path: "{{ opensds_build_dir }}"

-    state: absent

-    force: yes

-  ignore_errors: yes

+  shell: . /etc/profile; make clean

+  args:

+    chdir: "{{ opensds_root_dir }}"

 

 - name: clean all opensds configuration files

   file:

diff --git a/ci/ansible/roles/common/tasks/main.yml b/ci/ansible/roles/common/tasks/main.yml
index 20f5381..7c9a49a 100644
--- a/ci/ansible/roles/common/tasks/main.yml
+++ b/ci/ansible/roles/common/tasks/main.yml
@@ -13,8 +13,8 @@
       set -e

       set -x

 

-      wget https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz

-      tar xvf go1.9.linux-amd64.tar.gz -C /usr/local/

+      wget {{ golang_download_url }} -P /opt/

+      tar xvf /opt/{{ golang_tarball }} -C /usr/local/

       cat >> /etc/profile <<GOLANG__CONFIG_DOC

       export GOROOT=/usr/local/go

       export GOPATH=\$HOME/gopath

diff --git a/ci/ansible/roles/osdsdb/scenarios/etcd.yml b/ci/ansible/roles/osdsdb/scenarios/etcd.yml
index 79dc444..86f3a67 100644
--- a/ci/ansible/roles/osdsdb/scenarios/etcd.yml
+++ b/ci/ansible/roles/osdsdb/scenarios/etcd.yml
@@ -8,14 +8,14 @@
 - name: download etcd

   get_url:

     url={{ etcd_download_url }}

-    dest=/tmp/{{ etcd_tarball }}

+    dest=/opt/{{ etcd_tarball }}

   when:

     - etcdexisted.stat.exists is undefined or etcdexisted.stat.exists == false

 

 - name: extract the etcd tarball

   unarchive:

-    src=/tmp/{{ etcd_tarball }}

-    dest=/tmp/

+    src=/opt/{{ etcd_tarball }}

+    dest=/opt/

   when:

     - etcdexisted.stat.exists is undefined or etcdexisted.stat.exists == false

 

diff --git a/ci/ansible/roles/osdsdock/scenarios/ceph.yml b/ci/ansible/roles/osdsdock/scenarios/ceph.yml
index 2f6348e..2fa66b2 100644
--- a/ci/ansible/roles/osdsdock/scenarios/ceph.yml
+++ b/ci/ansible/roles/osdsdock/scenarios/ceph.yml
@@ -6,48 +6,48 @@
 

 - name: check for ceph-ansible source code existed

   stat:

-    path: /tmp/ceph-ansible

+    path: /opt/ceph-ansible

   ignore_errors: yes

   register: cephansibleexisted

 

 - name: download ceph-ansible source code

   git:

     repo: https://github.com/ceph/ceph-ansible.git

-    dest: /tmp/ceph-ansible

+    dest: /opt/ceph-ansible

   when:

     - cephansibleexisted.stat.exists is undefined or cephansibleexisted.stat.exists == false

 

 - name: copy ceph inventory host into ceph-ansible directory

   copy:

     src: ../../../group_vars/ceph/ceph.hosts

-    dest: /tmp/ceph-ansible/ceph.hosts

+    dest: /opt/ceph-ansible/ceph.hosts

 

 - name: copy ceph all.yml file into ceph-ansible group_vars directory

   copy:

     src: ../../../group_vars/ceph/all.yml

-    dest: /tmp/ceph-ansible/group_vars/all.yml

+    dest: /opt/ceph-ansible/group_vars/all.yml

 

 - name: copy ceph osds.yml file into ceph-ansible group_vars directory

   copy:

     src: ../../../group_vars/ceph/osds.yml

-    dest: /tmp/ceph-ansible/group_vars/osds.yml

+    dest: /opt/ceph-ansible/group_vars/osds.yml

 

 - name: copy site.yml.sample to site.yml in ceph-ansible

   copy:

-    src: /tmp/ceph-ansible/site.yml.sample

-    dest: /tmp/ceph-ansible/site.yml

+    src: /opt/ceph-ansible/site.yml.sample

+    dest: /opt/ceph-ansible/site.yml

 

 - name: ping all hosts

   shell: ansible all -m ping -i ceph.hosts

   become: true

   args:

-    chdir: /tmp/ceph-ansible

+    chdir: /opt/ceph-ansible

 

 - name: run ceph-ansible playbook

   shell: ansible-playbook site.yml -i ceph.hosts

   become: true

   args:

-    chdir: /tmp/ceph-ansible

+    chdir: /opt/ceph-ansible

 

 - name: Check if ceph osd is running

   shell: ps aux | grep ceph-osd | grep -v grep

diff --git a/ci/ansible/roles/osdsdock/scenarios/cinder_standalone.yml b/ci/ansible/roles/osdsdock/scenarios/cinder_standalone.yml
index 4ad5cea..7939d75 100644
--- a/ci/ansible/roles/osdsdock/scenarios/cinder_standalone.yml
+++ b/ci/ansible/roles/osdsdock/scenarios/cinder_standalone.yml
@@ -114,7 +114,7 @@
     sed -i "s/TAG ?= debian-cinder:latest/TAG ?= {{ cinder_image_tag }}:latest/g" Makefile

 

     sed -i "s/image: debian-cinder/image: {{ cinder_image_tag }}/g" docker-compose.yml

-    sed -i "s/image: lvm-debian-cinder/image: {{ cinder_image_tag }}/g" docker-compose.yml

+    sed -i "s/image: lvm-debian-cinder/image: lvm-{{ cinder_image_tag }}/g" docker-compose.yml

 

     sed -i "s/volume_group = cinder-volumes /volume_group = {{ cinder_volume_group }}/g" etc/cinder.conf

   become: true

diff --git a/ci/nbp-ansible/README.md b/ci/nbp-ansible/README.md
index 391bcc3..5a4c5ab 100644
--- a/ci/nbp-ansible/README.md
+++ b/ci/nbp-ansible/README.md
@@ -18,9 +18,6 @@ ssh-keygen -t rsa
 ssh-copy-id -i ~/.ssh/id_rsa.pub <ip_address> # IP address of the target machine of the installation

 ```

 

-### Install docker

-If use a standalone cinder as backend, you also need to install docker to run cinder service. Please see the [docker installation document](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/) for details.

-

 ### Install ansible tool

 ```bash

 sudo add-apt-repository ppa:ansible/ansible # This step is needed to upgrade ansible to version 2.4.2 which is required for the ceph backend.

@@ -29,6 +26,13 @@ sudo apt-get install ansible
 ansible --version # Ansible version 2.4.2 or higher is required for ceph; 2.0.0.2 or higher is needed for other backends.

 ```

 

+### Configure nbp plugin variable

+##### Common environment:

+Configure the ```nbp_plugin_type``` in `group_vars/common.yml` according to your environment:

+```yaml

+nbp_plugin_type: flexvolume # flexvolume is the default integration way, but you can change it from 'csi', 'flexvolume'

+```

+

 ### Check if the hosts can be reached

 ```bash

 sudo ansible all -m ping -i nbp.hosts

diff --git a/ci/nbp-ansible/group_vars/common.yml b/ci/nbp-ansible/group_vars/common.yml
index 344cab7..3860660 100644
--- a/ci/nbp-ansible/group_vars/common.yml
+++ b/ci/nbp-ansible/group_vars/common.yml
@@ -15,9 +15,19 @@ dummy:
 # GENERAL #

 ###########

 

-# These fields are not suggested to be modified

-nbp_download_url: https://github.com/opensds/nbp/releases/download/v0.1.0/opensds-k8s-linux-amd64.tar.gz

-nbp_tarball_url: /opt/opensds-k8s-linux-amd64.tar.gz

-nbp_dir: /opt/opensds-k8s-linux-amd64

+nbp_release: v0.1.0

+

+# These fields are not suggested to be modified 

+nbp_download_url: https://github.com/opensds/nbp/releases/download/{{ nbp_release }}/opensds-k8s-{{ nbp_release }}-linux-amd64.tar.gz

+nbp_tarball_url: /opt/opensds-k8s-{{ nbp_release }}-linux-amd64.tar.gz

+nbp_dir: /opt/opensds-k8s-{{ nbp_release }}-linux-amd64

+

+

+###########

+# PLUGIN  #

+###########

+

+nbp_plugin_type: flexvolume # flexvolume is the default integration way, but you can change it from 'csi', 'flexvolume'

 

 flexvolume_plugin_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/opensds.io~opensds

+

diff --git a/ci/nbp-ansible/roles/cleaner/tasks/main.yml b/ci/nbp-ansible/roles/cleaner/tasks/main.yml
index 38678b0..9e81756 100644
--- a/ci/nbp-ansible/roles/cleaner/tasks/main.yml
+++ b/ci/nbp-ansible/roles/cleaner/tasks/main.yml
@@ -5,6 +5,7 @@
     state: absent

     force: yes

   ignore_errors: yes

+  when: nbp_plugin_type == "flexvolume"

 

 - name: clean nbp release files

   file:

diff --git a/ci/nbp-ansible/roles/installer/scenarios/csi.yml b/ci/nbp-ansible/roles/installer/scenarios/csi.yml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/ci/nbp-ansible/roles/installer/scenarios/csi.yml
diff --git a/ci/nbp-ansible/roles/flexvolume/tasks/main.yml b/ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml
index 0bba93b..0bba93b 100644
--- a/ci/nbp-ansible/roles/flexvolume/tasks/main.yml
+++ b/ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml
diff --git a/ci/nbp-ansible/roles/installer/tasks/main.yml b/ci/nbp-ansible/roles/installer/tasks/main.yml
new file mode 100644
index 0000000..58057f1
--- /dev/null
+++ b/ci/nbp-ansible/roles/installer/tasks/main.yml
@@ -0,0 +1,8 @@
+---

+- name: include scenarios/flexvolume.yml

+  include: scenarios/flexvolume.yml

+  when: nbp_plugin_type == "flexvolume"

+

+- name: include scenarios/csi.yml

+  include: scenarios/csi.yml

+  when: nbp_plugin_type == "csi"

diff --git a/ci/nbp-ansible/site.yml b/ci/nbp-ansible/site.yml
index 42c0790..7e22f83 100644
--- a/ci/nbp-ansible/site.yml
+++ b/ci/nbp-ansible/site.yml
@@ -10,4 +10,4 @@
   become: True

   roles:

     - common

-    - flexvolume

+    - installer

diff --git a/tutorials/csi-plugin.md b/tutorials/csi-plugin.md
new file mode 100644
index 0000000..e3b0174
--- /dev/null
+++ b/tutorials/csi-plugin.md
@@ -0,0 +1,108 @@
+## Prerequisite ##

+

+### ubuntu

+* Version information

+

+	```

+	root@proxy:~# cat /etc/issue

+	Ubuntu 16.04.2 LTS \n \l

+	```

+

+### docker

+* Version information

+

+	```

+	root@proxy:~# docker version

+	Client:

+	 Version:      1.12.6

+	 API version:  1.24

+	 Go version:   go1.6.2

+	 Git commit:   78d1802

+	 Built:        Tue Jan 31 23:35:14 2017

+	 OS/Arch:      linux/amd64

+	

+	Server:

+	 Version:      1.12.6

+	 API version:  1.24

+	 Go version:   go1.6.2

+	 Git commit:   78d1802

+	 Built:        Tue Jan 31 23:35:14 2017

+	 OS/Arch:      linux/amd64

+	```

+

+### [kubernetes](https://github.com/kubernetes/kubernetes) local cluster

+* You can startup the lastest k8s local cluster by executing commands blow:

+

+	```

+	cd $HOME

+	git clone https://github.com/kubernetes/kubernetes.git

+	cd $HOME/kubernetes

+	make

+	echo alias kubectl='$HOME/kubernetes/cluster/kubectl.sh' >> /etc/profile

+	ALLOW_PRIVILEGED=true FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" LOG_LEVEL=5 hack/local-up-cluster.sh

+	```

+

+### [opensds](https://github.com/opensds/opensds) local cluster

+* For testing purposes you can deploy OpenSDS referring the [OpenSDS Cluster Installation through Ansible](https://github.com/opensds/opensds/wiki/OpenSDS-Cluster-Installation-through-Ansible) wiki. Besides, you need to deploy opensds csi plugin refering to ```nbp-ansible/README.md```.

+

+## Testing steps ##

+

+* Change the workplace

+

+	```

+	cd /opt/opensds-k8s-v0.1.0-linux-amd64

+	```

+

+* Configure opensds endpoint IP

+

+	```

+	vim csi/deploy/kubernetes/csi-configmap-opensdsplugin.yaml

+	```

+

+	The IP (127.0.0.1) should be replaced with the opensds actual endpoint IP.

+	```yaml

+	kind: ConfigMap

+	apiVersion: v1

+	    metadata:

+	name: csi-configmap-opensdsplugin

+	    data:

+	    opensdsendpoint: http://127.0.0.1:50040

+	```

+

+* Create opensds CSI pods.

+

+	```

+	kubectl create -f csi/deploy/kubernetes

+	```

+

+	After this three pods can be found by ```kubectl get pods``` like below:

+

+	- csi-provisioner-opensdsplugin

+	- csi-attacher-opensdsplugin

+	- csi-nodeplugin-opensdsplugin

+

+	You can find more design details from

+    [CSI Volume Plugins in Kubernetes Design Doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md)

+

+* Create example nginx application

+

+	```

+	kubectl create -f csi/examples/kubernetes/nginx.yaml

+	```

+

+	This example will mount a opensds volume into ```/var/lib/www/html```.

+

+	You can use the following command to inspect into nginx container to verify it.

+

+	```

+	docker exec -it <nginx container id> /bin/bash

+	```

+

+## Clean up steps ##

+

+Clean up example nginx application and opensds CSI pods by the following commands.

+

+```

+kubectl delete -f csi/examples/kubernetes/nginx.yaml

+kubectl delete -f csi/deploy/kubernetes

+```

diff --git a/tutorials/flexvolume-plugin.md b/tutorials/flexvolume-plugin.md
index 450acd5..c85d752 100644
--- a/tutorials/flexvolume-plugin.md
+++ b/tutorials/flexvolume-plugin.md
@@ -1,4 +1,5 @@
 ## Prerequisite ##

+

 ### ubuntu

 * Version information

 

@@ -6,6 +7,7 @@
 	root@proxy:~# cat /etc/issue

 	Ubuntu 16.04.2 LTS \n \l

 	```

+

 ### docker

 * Version information

 

@@ -55,14 +57,14 @@
 

 * Create service account, role and bind them.

 	```

-	cd /opt/opensds-k8s-linux-amd64/provisioner

+	cd /opt/opensds-k8s-{release version}-linux-amd64/provisioner

 	kubectl create -f serviceaccount.yaml

 	kubectl create -f clusterrole.yaml

 	kubectl create -f clusterrolebinding.yaml

 	```

 

 * Change the opensds endpoint IP in pod-provisioner.yaml

-The IP (192.168.56.106) should be replaced with the OpenSDS osdslet actual endpoint IP.

+The IP ```192.168.56.106``` should be replaced with the OpenSDS osdslet actual endpoint IP.

     ```yaml

     kind: Pod

     apiVersion: v1

@@ -72,7 +74,7 @@ The IP (192.168.56.106) should be replaced with the OpenSDS osdslet actual endpo
       serviceAccount: opensds-provisioner

       containers:

         - name: opensds-provisioner

-          image: opensdsio/opensds-provisioner

+          image: opensdsio/opensds-provisioner:latest

           securityContext:

           args:

             - "-endpoint=http://192.168.56.106:50040" # should be replaced