From 561c0708b78eed8120b90f0221cd66730c9e8a74 Mon Sep 17 00:00:00 2001 From: leonwang Date: Sat, 3 Feb 2018 09:27:38 +0000 Subject: Add csi plugin ansible and tutorial document As discussed on Compass4NFV meeting this week, we need to add the support for csi-plugin. So in this patch I extend nbp-ansible to support integrating with k8s using csi. All the detailed information can be found in documents. Besides, since opensds community has updated something in opensds ansible, we should update ours to keep up with the latest code. Change-Id: I238c7f7652b6a9c86cc9fcb24d9575d625c5f80a Signed-off-by: leonwang --- ci/ansible/README.md | 4 +- ci/ansible/group_vars/common.yml | 12 ++- ci/ansible/group_vars/osdsdb.yml | 2 +- ci/ansible/roles/cleaner/tasks/main.yml | 17 ++-- ci/ansible/roles/common/tasks/main.yml | 4 +- ci/ansible/roles/osdsdb/scenarios/etcd.yml | 6 +- ci/ansible/roles/osdsdock/scenarios/ceph.yml | 18 ++-- .../roles/osdsdock/scenarios/cinder_standalone.yml | 2 +- ci/nbp-ansible/README.md | 10 +- ci/nbp-ansible/group_vars/common.yml | 18 +++- ci/nbp-ansible/roles/cleaner/tasks/main.yml | 1 + ci/nbp-ansible/roles/flexvolume/tasks/main.yml | 11 --- ci/nbp-ansible/roles/installer/scenarios/csi.yml | 0 .../roles/installer/scenarios/flexvolume.yml | 11 +++ ci/nbp-ansible/roles/installer/tasks/main.yml | 8 ++ ci/nbp-ansible/site.yml | 2 +- tutorials/csi-plugin.md | 108 +++++++++++++++++++++ tutorials/flexvolume-plugin.md | 8 +- 18 files changed, 195 insertions(+), 47 deletions(-) delete mode 100644 ci/nbp-ansible/roles/flexvolume/tasks/main.yml create mode 100644 ci/nbp-ansible/roles/installer/scenarios/csi.yml create mode 100644 ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml create mode 100644 ci/nbp-ansible/roles/installer/tasks/main.yml create mode 100644 tutorials/csi-plugin.md diff --git a/ci/ansible/README.md b/ci/ansible/README.md index 37a22f4..2bf2a80 100644 --- a/ci/ansible/README.md +++ b/ci/ansible/README.md @@ -165,7 +165,7 @@ sudo ansible-playbook clean.yml -i local.hosts ### Run ceph-ansible playbook to clean ceph cluster if ceph is deployed ```bash -cd /tmp/ceph-ansible +cd /opt/ceph-ansible sudo ansible-playbook infrastructure-playbooks/purge-cluster.yml -i ceph.hosts ``` @@ -174,5 +174,5 @@ In addition, clean up the logical partition on the physical block device used by ### Remove ceph-ansible source code (optional) ```bash cd .. -sudo rm -rf /tmp/ceph-ansible +sudo rm -rf /opt/ceph-ansible ``` diff --git a/ci/ansible/group_vars/common.yml b/ci/ansible/group_vars/common.yml index 11fdc80..a67b8c8 100644 --- a/ci/ansible/group_vars/common.yml +++ b/ci/ansible/group_vars/common.yml @@ -15,7 +15,7 @@ dummy: # GENERAL # ########### -workplace: /home/krej # Change this field according to your username +workplace: /home/krej # Change this field according to your username, use '/root' if you login as root. # These fields are NOT suggested to be modified remote_url: https://github.com/opensds/opensds.git @@ -23,3 +23,13 @@ opensds_root_dir: "{{ workplace }}/gopath/src/github.com/opensds/opensds" opensds_build_dir: "{{ opensds_root_dir }}/build" opensds_config_dir: /etc/opensds opensds_log_dir: /var/log/opensds + +########### +# GOLANG # +########### + +golang_release: 1.9.2 + +# These fields are NOT suggested to be modified +golang_tarball: go{{ golang_release }}.linux-amd64.tar.gz +golang_download_url: https://storage.googleapis.com/golang/{{ golang_tarball }} diff --git a/ci/ansible/group_vars/osdsdb.yml b/ci/ansible/group_vars/osdsdb.yml index 2ec40fe..f5dfdef 100644 --- a/ci/ansible/group_vars/osdsdb.yml +++ b/ci/ansible/group_vars/osdsdb.yml @@ -28,4 +28,4 @@ etcd_release: v3.2.0 # These fields are not suggested to be modified etcd_tarball: etcd-{{ etcd_release }}-linux-amd64.tar.gz etcd_download_url: https://github.com/coreos/etcd/releases/download/{{ etcd_release }}/{{ etcd_tarball }} -etcd_dir: /tmp/etcd-{{ etcd_release }}-linux-amd64 +etcd_dir: /opt/etcd-{{ etcd_release }}-linux-amd64 diff --git a/ci/ansible/roles/cleaner/tasks/main.yml b/ci/ansible/roles/cleaner/tasks/main.yml index 93aeb59..334d5bb 100644 --- a/ci/ansible/roles/cleaner/tasks/main.yml +++ b/ci/ansible/roles/cleaner/tasks/main.yml @@ -1,4 +1,11 @@ --- +- name: remove golang tarball + file: + path: "/opt/{{ golang_tarball }}" + state: absent + force: yes + ignore_errors: yes + - name: kill etcd daemon service shell: killall etcd ignore_errors: yes @@ -14,7 +21,7 @@ - name: remove etcd tarball file: - path: "/tmp/{{ etcd_tarball }}" + path: "/opt/{{ etcd_tarball }}" state: absent force: yes ignore_errors: yes @@ -29,11 +36,9 @@ ignore_errors: yes - name: clean all opensds build files - file: - path: "{{ opensds_build_dir }}" - state: absent - force: yes - ignore_errors: yes + shell: . /etc/profile; make clean + args: + chdir: "{{ opensds_root_dir }}" - name: clean all opensds configuration files file: diff --git a/ci/ansible/roles/common/tasks/main.yml b/ci/ansible/roles/common/tasks/main.yml index 20f5381..7c9a49a 100644 --- a/ci/ansible/roles/common/tasks/main.yml +++ b/ci/ansible/roles/common/tasks/main.yml @@ -13,8 +13,8 @@ set -e set -x - wget https://storage.googleapis.com/golang/go1.9.linux-amd64.tar.gz - tar xvf go1.9.linux-amd64.tar.gz -C /usr/local/ + wget {{ golang_download_url }} -P /opt/ + tar xvf /opt/{{ golang_tarball }} -C /usr/local/ cat >> /etc/profile < # IP address of the target machine of the installation ``` -### Install docker -If use a standalone cinder as backend, you also need to install docker to run cinder service. Please see the [docker installation document](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/) for details. - ### Install ansible tool ```bash sudo add-apt-repository ppa:ansible/ansible # This step is needed to upgrade ansible to version 2.4.2 which is required for the ceph backend. @@ -29,6 +26,13 @@ sudo apt-get install ansible ansible --version # Ansible version 2.4.2 or higher is required for ceph; 2.0.0.2 or higher is needed for other backends. ``` +### Configure nbp plugin variable +##### Common environment: +Configure the ```nbp_plugin_type``` in `group_vars/common.yml` according to your environment: +```yaml +nbp_plugin_type: flexvolume # flexvolume is the default integration way, but you can change it from 'csi', 'flexvolume' +``` + ### Check if the hosts can be reached ```bash sudo ansible all -m ping -i nbp.hosts diff --git a/ci/nbp-ansible/group_vars/common.yml b/ci/nbp-ansible/group_vars/common.yml index 344cab7..3860660 100644 --- a/ci/nbp-ansible/group_vars/common.yml +++ b/ci/nbp-ansible/group_vars/common.yml @@ -15,9 +15,19 @@ dummy: # GENERAL # ########### -# These fields are not suggested to be modified -nbp_download_url: https://github.com/opensds/nbp/releases/download/v0.1.0/opensds-k8s-linux-amd64.tar.gz -nbp_tarball_url: /opt/opensds-k8s-linux-amd64.tar.gz -nbp_dir: /opt/opensds-k8s-linux-amd64 +nbp_release: v0.1.0 + +# These fields are not suggested to be modified +nbp_download_url: https://github.com/opensds/nbp/releases/download/{{ nbp_release }}/opensds-k8s-{{ nbp_release }}-linux-amd64.tar.gz +nbp_tarball_url: /opt/opensds-k8s-{{ nbp_release }}-linux-amd64.tar.gz +nbp_dir: /opt/opensds-k8s-{{ nbp_release }}-linux-amd64 + + +########### +# PLUGIN # +########### + +nbp_plugin_type: flexvolume # flexvolume is the default integration way, but you can change it from 'csi', 'flexvolume' flexvolume_plugin_dir: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/opensds.io~opensds + diff --git a/ci/nbp-ansible/roles/cleaner/tasks/main.yml b/ci/nbp-ansible/roles/cleaner/tasks/main.yml index 38678b0..9e81756 100644 --- a/ci/nbp-ansible/roles/cleaner/tasks/main.yml +++ b/ci/nbp-ansible/roles/cleaner/tasks/main.yml @@ -5,6 +5,7 @@ state: absent force: yes ignore_errors: yes + when: nbp_plugin_type == "flexvolume" - name: clean nbp release files file: diff --git a/ci/nbp-ansible/roles/flexvolume/tasks/main.yml b/ci/nbp-ansible/roles/flexvolume/tasks/main.yml deleted file mode 100644 index 0bba93b..0000000 --- a/ci/nbp-ansible/roles/flexvolume/tasks/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Create flexvolume plugin directory if not existed - file: - path: "{{ flexvolume_plugin_dir }}" - state: directory - mode: 0755 - -- name: Copy opensds flexvolume plugin binary file into flexvolume plugin dir - copy: - src: "{{ nbp_dir }}/flexvolume/opensds" - dest: "{{ flexvolume_plugin_dir }}/opensds" diff --git a/ci/nbp-ansible/roles/installer/scenarios/csi.yml b/ci/nbp-ansible/roles/installer/scenarios/csi.yml new file mode 100644 index 0000000..e69de29 diff --git a/ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml b/ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml new file mode 100644 index 0000000..0bba93b --- /dev/null +++ b/ci/nbp-ansible/roles/installer/scenarios/flexvolume.yml @@ -0,0 +1,11 @@ +--- +- name: Create flexvolume plugin directory if not existed + file: + path: "{{ flexvolume_plugin_dir }}" + state: directory + mode: 0755 + +- name: Copy opensds flexvolume plugin binary file into flexvolume plugin dir + copy: + src: "{{ nbp_dir }}/flexvolume/opensds" + dest: "{{ flexvolume_plugin_dir }}/opensds" diff --git a/ci/nbp-ansible/roles/installer/tasks/main.yml b/ci/nbp-ansible/roles/installer/tasks/main.yml new file mode 100644 index 0000000..58057f1 --- /dev/null +++ b/ci/nbp-ansible/roles/installer/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: include scenarios/flexvolume.yml + include: scenarios/flexvolume.yml + when: nbp_plugin_type == "flexvolume" + +- name: include scenarios/csi.yml + include: scenarios/csi.yml + when: nbp_plugin_type == "csi" diff --git a/ci/nbp-ansible/site.yml b/ci/nbp-ansible/site.yml index 42c0790..7e22f83 100644 --- a/ci/nbp-ansible/site.yml +++ b/ci/nbp-ansible/site.yml @@ -10,4 +10,4 @@ become: True roles: - common - - flexvolume + - installer diff --git a/tutorials/csi-plugin.md b/tutorials/csi-plugin.md new file mode 100644 index 0000000..e3b0174 --- /dev/null +++ b/tutorials/csi-plugin.md @@ -0,0 +1,108 @@ +## Prerequisite ## + +### ubuntu +* Version information + + ``` + root@proxy:~# cat /etc/issue + Ubuntu 16.04.2 LTS \n \l + ``` + +### docker +* Version information + + ``` + root@proxy:~# docker version + Client: + Version: 1.12.6 + API version: 1.24 + Go version: go1.6.2 + Git commit: 78d1802 + Built: Tue Jan 31 23:35:14 2017 + OS/Arch: linux/amd64 + + Server: + Version: 1.12.6 + API version: 1.24 + Go version: go1.6.2 + Git commit: 78d1802 + Built: Tue Jan 31 23:35:14 2017 + OS/Arch: linux/amd64 + ``` + +### [kubernetes](https://github.com/kubernetes/kubernetes) local cluster +* You can startup the lastest k8s local cluster by executing commands blow: + + ``` + cd $HOME + git clone https://github.com/kubernetes/kubernetes.git + cd $HOME/kubernetes + make + echo alias kubectl='$HOME/kubernetes/cluster/kubectl.sh' >> /etc/profile + ALLOW_PRIVILEGED=true FEATURE_GATES=CSIPersistentVolume=true,MountPropagation=true RUNTIME_CONFIG="storage.k8s.io/v1alpha1=true" LOG_LEVEL=5 hack/local-up-cluster.sh + ``` + +### [opensds](https://github.com/opensds/opensds) local cluster +* For testing purposes you can deploy OpenSDS referring the [OpenSDS Cluster Installation through Ansible](https://github.com/opensds/opensds/wiki/OpenSDS-Cluster-Installation-through-Ansible) wiki. Besides, you need to deploy opensds csi plugin refering to ```nbp-ansible/README.md```. + +## Testing steps ## + +* Change the workplace + + ``` + cd /opt/opensds-k8s-v0.1.0-linux-amd64 + ``` + +* Configure opensds endpoint IP + + ``` + vim csi/deploy/kubernetes/csi-configmap-opensdsplugin.yaml + ``` + + The IP (127.0.0.1) should be replaced with the opensds actual endpoint IP. + ```yaml + kind: ConfigMap + apiVersion: v1 + metadata: + name: csi-configmap-opensdsplugin + data: + opensdsendpoint: http://127.0.0.1:50040 + ``` + +* Create opensds CSI pods. + + ``` + kubectl create -f csi/deploy/kubernetes + ``` + + After this three pods can be found by ```kubectl get pods``` like below: + + - csi-provisioner-opensdsplugin + - csi-attacher-opensdsplugin + - csi-nodeplugin-opensdsplugin + + You can find more design details from + [CSI Volume Plugins in Kubernetes Design Doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/storage/container-storage-interface.md) + +* Create example nginx application + + ``` + kubectl create -f csi/examples/kubernetes/nginx.yaml + ``` + + This example will mount a opensds volume into ```/var/lib/www/html```. + + You can use the following command to inspect into nginx container to verify it. + + ``` + docker exec -it /bin/bash + ``` + +## Clean up steps ## + +Clean up example nginx application and opensds CSI pods by the following commands. + +``` +kubectl delete -f csi/examples/kubernetes/nginx.yaml +kubectl delete -f csi/deploy/kubernetes +``` diff --git a/tutorials/flexvolume-plugin.md b/tutorials/flexvolume-plugin.md index 450acd5..c85d752 100644 --- a/tutorials/flexvolume-plugin.md +++ b/tutorials/flexvolume-plugin.md @@ -1,4 +1,5 @@ ## Prerequisite ## + ### ubuntu * Version information @@ -6,6 +7,7 @@ root@proxy:~# cat /etc/issue Ubuntu 16.04.2 LTS \n \l ``` + ### docker * Version information @@ -55,14 +57,14 @@ * Create service account, role and bind them. ``` - cd /opt/opensds-k8s-linux-amd64/provisioner + cd /opt/opensds-k8s-{release version}-linux-amd64/provisioner kubectl create -f serviceaccount.yaml kubectl create -f clusterrole.yaml kubectl create -f clusterrolebinding.yaml ``` * Change the opensds endpoint IP in pod-provisioner.yaml -The IP (192.168.56.106) should be replaced with the OpenSDS osdslet actual endpoint IP. +The IP ```192.168.56.106``` should be replaced with the OpenSDS osdslet actual endpoint IP. ```yaml kind: Pod apiVersion: v1 @@ -72,7 +74,7 @@ The IP (192.168.56.106) should be replaced with the OpenSDS osdslet actual endpo serviceAccount: opensds-provisioner containers: - name: opensds-provisioner - image: opensdsio/opensds-provisioner + image: opensdsio/opensds-provisioner:latest securityContext: args: - "-endpoint=http://192.168.56.106:50040" # should be replaced -- cgit 1.2.3-korg