From 4bc89660111fbd2fe8a7edc1cbc053b2e297ec56 Mon Sep 17 00:00:00 2001 From: ramamani yeleswarapu Date: Wed, 7 Feb 2018 21:33:40 -0800 Subject: Add plugin for Stor4nfv for k8s deploy - Adds 'stor4nfv' plugin which installs OpenSDS using stor4nfv. - Adds k8s-nosdn-stor4nfv-ha scenario file. - Adds k8s-nosdn-stor4nfv-noha scenario file. Change-Id: Iee718a48994ad4e55f4b7a01933bd4e7773ac67f Signed-off-by: ramamani yeleswarapu --- .../ansible/kubernetes/ansible-kubernetes.yml | 12 ++++ deploy/compass_conf/flavor/kubernetes.conf | 2 +- .../package_installer/ansible-kubernetes.conf | 2 +- deploy/compass_conf/role/kubernetes_ansible.conf | 9 ++- .../kubernetes/vars/ansible-kubernetes.tmpl | 1 + .../conf/vm_environment/k8-nosdn-stor4nfv-ha.yml | 45 ++++++++++++ .../conf/vm_environment/k8-nosdn-stor4nfv-noha.yml | 28 ++++++++ plugins/stor4nfv/plugin.desc | 45 ++++++++++++ .../roles/stor4nfv/files/configure_vars.sh | 39 ++++++++++ .../roles/stor4nfv/files/install_ansible.sh | 18 +++++ plugins/stor4nfv/roles/stor4nfv/tasks/main.yml | 13 ++++ plugins/stor4nfv/roles/stor4nfv/tasks/opensds.yml | 84 ++++++++++++++++++++++ .../roles/stor4nfv/templates/ceph.hosts.j2 | 14 ++++ .../roles/stor4nfv/templates/opensds.hosts.j2 | 9 +++ 14 files changed, 318 insertions(+), 3 deletions(-) create mode 100644 deploy/conf/vm_environment/k8-nosdn-stor4nfv-ha.yml create mode 100644 deploy/conf/vm_environment/k8-nosdn-stor4nfv-noha.yml create mode 100644 plugins/stor4nfv/plugin.desc create mode 100644 plugins/stor4nfv/roles/stor4nfv/files/configure_vars.sh create mode 100644 plugins/stor4nfv/roles/stor4nfv/files/install_ansible.sh create mode 100644 plugins/stor4nfv/roles/stor4nfv/tasks/main.yml create mode 100644 plugins/stor4nfv/roles/stor4nfv/tasks/opensds.yml create mode 100644 plugins/stor4nfv/roles/stor4nfv/templates/ceph.hosts.j2 create mode 100644 plugins/stor4nfv/roles/stor4nfv/templates/opensds.hosts.j2 diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml index bfdc8958..1eae8a97 100755 --- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml +++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml @@ -42,3 +42,15 @@ max_fail_percentage: 0 roles: - post-k8s + +- hosts: kube_node + remote_user: root + max_fail_percentage: 0 + roles: + - storage + +- hosts: storage_master + remote_user: root + max_fail_percentage: 0 + roles: + - stor4nfv diff --git a/deploy/compass_conf/flavor/kubernetes.conf b/deploy/compass_conf/flavor/kubernetes.conf index 71acadff..e7e8ffc8 100755 --- a/deploy/compass_conf/flavor/kubernetes.conf +++ b/deploy/compass_conf/flavor/kubernetes.conf @@ -4,7 +4,7 @@ FLAVORS = [{ 'display_name': 'ansible-kubernetes', 'template': 'ansible-kubernetes.tmpl', 'roles': [ - 'kube_master', 'etcd', 'kube_node', 'ha' + 'kube_master', 'etcd', 'kube_node', 'ha', 'storage_master', 'storage_node' ], }] diff --git a/deploy/compass_conf/package_installer/ansible-kubernetes.conf b/deploy/compass_conf/package_installer/ansible-kubernetes.conf index 820691b7..044af9a9 100755 --- a/deploy/compass_conf/package_installer/ansible-kubernetes.conf +++ b/deploy/compass_conf/package_installer/ansible-kubernetes.conf @@ -7,7 +7,7 @@ SETTINGS = { 'playbook_file': 'site.yml', 'inventory_file': 'inventory.py', 'inventory_json_file': 'inventory.json', - 'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha'], + 'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha', 'ceph_adm', 'ceph_mon', 'ceph_osd', 'storage_master', 'storage_node'], 'group_variable': 'all', 'etc_hosts_path': 'roles/pre-k8s/templates/hosts', 'runner_dirs': ['roles','kubernetes/roles'] diff --git a/deploy/compass_conf/role/kubernetes_ansible.conf b/deploy/compass_conf/role/kubernetes_ansible.conf index c27779ad..f86902e3 100755 --- a/deploy/compass_conf/role/kubernetes_ansible.conf +++ b/deploy/compass_conf/role/kubernetes_ansible.conf @@ -15,6 +15,13 @@ ROLES = [{ 'role': 'ha', 'display_name': 'ha', 'description': 'ha' +}, { + 'role': 'storage_master', + 'display_name': 'storage master', + 'description': 'storage master' +}, { + 'role': 'storage_node', + 'display_name': 'storage node', + 'description': 'storage node' } - ] diff --git a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl index 59130fec..f6d96c78 100644 --- a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl +++ b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl @@ -40,6 +40,7 @@ deploy_type: $getVar('deploy_type', 'virtual') public_cidr: $kube_nodes[0]['install']['subnet'] storage_cidr: "{{ ip_settings[inventory_hostname]['storage']['cidr'] }}" mgmt_cidr: "{{ ip_settings[inventory_hostname]['mgmt']['cidr'] }}" +external_cidr: "{{ ip_settings[inventory_hostname]['external']['cidr'] }}" public_net_info: "{{ network_cfg.public_net_info }}" host_ip_settings: "{{ ip_settings[inventory_hostname] }}" diff --git a/deploy/conf/vm_environment/k8-nosdn-stor4nfv-ha.yml b/deploy/conf/vm_environment/k8-nosdn-stor4nfv-ha.yml new file mode 100644 index 00000000..e5e458f7 --- /dev/null +++ b/deploy/conf/vm_environment/k8-nosdn-stor4nfv-ha.yml @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +TYPE: virtual +FLAVOR: cluster + +plugins: + - stor4nfv: "Enable" + +hosts: + - name: host1 + roles: + - kube_master + - etcd + - ha + + - name: host2 + roles: + - kube_master + - etcd + - ha + + - name: host3 + roles: + - kube_master + - etcd + - ha + - storage_master + + - name: host4 + roles: + - kube_node + - storage_node + + - name: host5 + roles: + - kube_node + - storage_node diff --git a/deploy/conf/vm_environment/k8-nosdn-stor4nfv-noha.yml b/deploy/conf/vm_environment/k8-nosdn-stor4nfv-noha.yml new file mode 100644 index 00000000..f8c29b3e --- /dev/null +++ b/deploy/conf/vm_environment/k8-nosdn-stor4nfv-noha.yml @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +--- +TYPE: virtual +FLAVOR: cluster + +plugins: + - stor4nfv: "Enable" + +hosts: + - name: host1 + roles: + - kube_master + - etcd + - storage_master + - ha + + - name: host2 + roles: + - kube_node + - storage_node diff --git a/plugins/stor4nfv/plugin.desc b/plugins/stor4nfv/plugin.desc new file mode 100644 index 00000000..ae589a60 --- /dev/null +++ b/plugins/stor4nfv/plugin.desc @@ -0,0 +1,45 @@ +# Plugin for Stor4nfv service for Compass4nfv. +# Stor4NFV provides a storage solution based on Ceph and +# OpenSDS, and focuses on the optimization for storage +# intensive use cases of NFV, like I/O performance improvements. +# +# More details can be found in the development document. +# ############################################################## +--- +plugin: + # plugin name,it is also as the switch to enable/disable plugin in scenario + # files + name: stor4nfv + + description: provide a storage solution based on Ceph and OpenSDS + + maintainers: + + # host os type: ubuntu/centos + os_version: ubuntu + + # true: this plugin is deployed separately on a new node + # false: this plugin is deployed on controller or compute node + independent_hosts: false + + # artifact: package download url for this plugin + artifacts: + url: + + # orchestration + # A plugin can have mutiple components, each component may need to be + # installed on different inventory or have its own configuration. + # due to Compass4nfv currently only supports ansible, so each component + # of the installation and configuration script need to be use ansible. + # cm : congfiguration management tool : only ansible support + # role: each component corresponds to ansible script that locates in the same + # directory as plugin.desc. + # phrase: + # inventory: + orchestration: + cm: ansible + roles: + - role: stor4nfv + phrase: post_k8s + inventory: + - localhost diff --git a/plugins/stor4nfv/roles/stor4nfv/files/configure_vars.sh b/plugins/stor4nfv/roles/stor4nfv/files/configure_vars.sh new file mode 100644 index 00000000..878f1542 --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/files/configure_vars.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# ############################################################################# +# Copyright (c) 2018 Intel Corp. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################# + +cd $HOME/gopath/src/github.com/stor4nfv/stor4nfv/ci/ansible + +sed -i 's/^workplace.*/workplace: \/root/g' group_vars/common.yml + +sed -i 's/^enabled_backend.*/enabled_backend: ceph/g' group_vars/osdsdock.yml + +sed -i 's/^ceph_pool_name.*/ceph_pool_name: "rbd"/g' group_vars/osdsdock.yml + +sed -i 's/^ceph_origin.*/ceph_origin: repository/g' group_vars/ceph/all.yml + +sed -i 's/^ceph_repository.*/ceph_repository: community/g' group_vars/ceph/all.yml + +sed -i 's/^ceph_stable_release.*/ceph_stable_release: luminous/g' group_vars/ceph/all.yml + +sed -i 's|^public_network.*|public_network: '"$1"'|g' group_vars/ceph/all.yml + +sed -i 's|^cluster_network.*|cluster_network: '"$2"'|g' group_vars/ceph/all.yml + +sed -i 's/^monitor_interface.*/monitor_interface: eth0/g' group_vars/ceph/all.yml + +sed -i 's/^devices:.*/devices: [\/dev\/loop0, \/dev\/loop1, \/dev\/loop2]/g' group_vars/ceph/osds.yml + +sed -i 's/^osd_scenario.*/osd_scenario: collocated/g' group_vars/ceph/osds.yml + +sed -i 's/^db_endpoint.*/db_endpoint: localhost:62379,localhost:62380/g' group_vars/osdsdb.yml + +sed -i 's/^etcd_port.*/etcd_port: 62379/g' group_vars/osdsdb.yml + +sed -i 's/^etcd_peer_port.*/etcd_peer_port: 62380/g' group_vars/osdsdb.yml diff --git a/plugins/stor4nfv/roles/stor4nfv/files/install_ansible.sh b/plugins/stor4nfv/roles/stor4nfv/files/install_ansible.sh new file mode 100644 index 00000000..b4a3a30a --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/files/install_ansible.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# ############################################################################# +# Copyright (c) 2018 Intel Corp. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################# + +add-apt-repository ppa:ansible/ansible + +apt-get update +apt-get install -y ansible +sleep 5 + +ansible --version + diff --git a/plugins/stor4nfv/roles/stor4nfv/tasks/main.yml b/plugins/stor4nfv/roles/stor4nfv/tasks/main.yml new file mode 100644 index 00000000..353282f9 --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/tasks/main.yml @@ -0,0 +1,13 @@ +# ############################################################################# +# Copyright (c) 2018 Intel Corp. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################# +--- +- include: opensds.yml + when: + - ansible_distribution == 'Ubuntu' + - stor4nfv is defined and stor4nfv == "Enable" diff --git a/plugins/stor4nfv/roles/stor4nfv/tasks/opensds.yml b/plugins/stor4nfv/roles/stor4nfv/tasks/opensds.yml new file mode 100644 index 00000000..afd65bc7 --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/tasks/opensds.yml @@ -0,0 +1,84 @@ +# ############################################################################# +# Copyright (c) 2018 Intel Corp. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################# +--- + +- name: install prerequisites package + apt: + name: "{{ item }}" + state: present + with_items: + - make + - gcc + +- name: copy install_ansible script + remote_user: root + copy: + src: install_ansible.sh + dest: /opt/install_ansible.sh + mode: 0777 + +- name: install ansible + command: su -s /bin/sh -c "/opt/install_ansible.sh" + +- name: install ceph-ansible code + remote_user: root + shell: | + cd /opt; + git clone https://github.com/ceph/ceph-ansible.git + +- name: fetch stor4nfv source code + remote_user: root + shell: | + mkdir -p $HOME/gopath/src/github.com/stor4nfv && cd $HOME/gopath/src/github.com/stor4nfv; + git clone https://gerrit.opnfv.org/gerrit/stor4nfv.git + +- name: copy configure_vars script + remote_user: root + copy: + src: configure_vars.sh + dest: /opt/configure_vars.sh + mode: 0777 + +- name: configure variables for ceph, opensds + remote_user: root + shell: | + cd /opt; + ./configure_vars.sh {{ mgmt_cidr }} {{ external_cidr }} + +- name: render ceph inventory + remote_user: root + template: + src: ceph.hosts.j2 + dest: $HOME/gopath/src/github.com/stor4nfv/stor4nfv/ci/ansible/group_vars/ceph/ceph.hosts + +- name: render opensds inventory + remote_user: root + template: + src: opensds.hosts.j2 + dest: $HOME/gopath/src/github.com/stor4nfv/stor4nfv/ci/ansible/local.hosts + +- name: force to run the second etcd cluster + remote_user: root + lineinfile: + dest: $HOME/gopath/src/github.com/stor4nfv/stor4nfv/ci/ansible/roles/osdsdb/scenarios/etcd.yml + state: absent + regexp: 'when: service_etcd_status.rc != 0' + +- name: run playbook + remote_user: root + shell: | + cd $HOME/gopath/src/github.com/stor4nfv/stor4nfv/ci/ansible; + ansible-playbook site.yml -i local.hosts | tee /var/log/stor4nfv.log + register: stor4nfv_result + +- name: export opensds endpoint + remote_user: root + shell: | + export OPENSDS_ENDPOINT=http://127.0.0.1:50040 + when: stor4nfv_result|succeeded diff --git a/plugins/stor4nfv/roles/stor4nfv/templates/ceph.hosts.j2 b/plugins/stor4nfv/roles/stor4nfv/templates/ceph.hosts.j2 new file mode 100644 index 00000000..fda313d4 --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/templates/ceph.hosts.j2 @@ -0,0 +1,14 @@ +[mons] +{% for controller in hostvars[inventory_hostname]['groups']['kube_master'] %} +{{ controller }} ansible_ssh_host={{ hostvars[controller]['ansible_ssh_host'] }} +{% endfor %} + +[osds] +{% for controller in hostvars[inventory_hostname]['groups']['storage_node'] %} +{{ controller }} ansible_ssh_host={{ hostvars[controller]['ansible_ssh_host'] }} +{% endfor %} + +[mgrs] +{% for controller in hostvars[inventory_hostname]['groups']['kube_master'] %} +{{ controller }} ansible_ssh_host={{ hostvars[controller]['ansible_ssh_host'] }} +{% endfor %} diff --git a/plugins/stor4nfv/roles/stor4nfv/templates/opensds.hosts.j2 b/plugins/stor4nfv/roles/stor4nfv/templates/opensds.hosts.j2 new file mode 100644 index 00000000..97068b90 --- /dev/null +++ b/plugins/stor4nfv/roles/stor4nfv/templates/opensds.hosts.j2 @@ -0,0 +1,9 @@ +[controllers] +{% for controller in hostvars[inventory_hostname]['groups']['storage_master'] %} +{{ controller }} ansible_ssh_host={{ hostvars[controller]['ansible_ssh_host'] }} +{% endfor %} + +[docks] +{% for dock in hostvars[inventory_hostname]['groups']['storage_master'] %} +{{ dock }} ansible_ssh_host={{ hostvars[dock]['ansible_ssh_host'] }} +{% endfor %} -- cgit 1.2.3-korg