9 files changed, 211 insertions, 223 deletions
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml
index 26e3fa75..ae3dce76 100644..100755
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml
@@ -3,7 +3,7 @@
 
 # roles/download/defaults/main.yml
 etcd_version: v3.2.4-arm64
-flannel_version: "v0.8.0-arm64"
+flannel_version: "v0.9.1-arm64"
 flannel_cni_image_repo: "linaro/flannel-cni-arm64"
 hyperkube_image_repo: "gcr.io/google-containers/hyperkube-arm64"
 hyperkube_image_tag: "{{ kube_version }}"
@@ -14,20 +14,8 @@ dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-arm64"
 dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-arm64"
 kubednsautoscaler_image_repo: "gcr.io/google_containers/\
 cluster-proportional-autoscaler-arm64"
+dashboard_image_repo: "gcr.io/google_containers/kubernetes-dashboard-arm64"
 
 # inventory/group_vars/k8s-cluster.yml
 kube_network_plugin: flannel
 helm_enabled: false
-docker_options: "--insecure-registry={{ kube_service_addresses }} \
---graph={{ docker_daemon_graph }}  {{ docker_log_opts }} \
---add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \
---default-runtime=docker-runc \
---exec-opt native.cgroupdriver=systemd \
---userland-proxy-path=/usr/libexec/docker/docker-proxy-current \
---signature-verification=false"
-
-# roles/docker/vars/redhat.yml
-docker_package_info:
-  pkg_mgr: yum
-  pkgs:
-    - name: docker
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml
deleted file mode 100644
index e13e33ca..00000000
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-# Override default kubespray variables
-
-# Just a placeholder to satisfy ansible
-dummy_var: 0
-
-# helm_enabled: true
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py b/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py
deleted file mode 100755
index 2ffb4cae..00000000
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import yaml
-import sys
-import os
-from jinja2 import Environment
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-INVENTORY_TEMPLATE = """
-[all]
-{% for host, vales in hostvars.iteritems() %}
-{{ host }} ansible_ssh_host={{ vales['ansible_ssh_host'] }} \
-ansible_ssh_pass=root  ansible_user=root
-{% endfor %}
-[kube-master]
-{% for host in kube_master %}
-{{ host }}
-{% endfor %}
-
-[etcd]
-{% for host in etcd %}
-{{ host }}
-{% endfor %}
-
-[kube-node]
-{% for host in kube_node %}
-{{ host }}
-{% endfor %}
-
-[k8s-cluster:children]
-kube-node
-kube-master
-
-[calico-rr]
-[vault]
-"""
-
-
-def _byteify(data, ignore_dicts=False):
-
-    if isinstance(data, unicode):
-        return data.encode('utf-8')
-    if isinstance(data, list):
-        return [_byteify(item, ignore_dicts=True) for item in data]
-    if isinstance(data, dict) and not ignore_dicts:
-        return {
-            _byteify(key, ignore_dicts=True):
-            _byteify(value, ignore_dicts=True)
-            for key, value in data.iteritems()
-        }
-    return data
-
-
-def load_inventory(inventory):
-    if not os.path.exists(inventory):
-        raise RuntimeError('file: %s not exist' % inventory)
-    with open(inventory, 'r') as fd:
-        return json.load(fd, object_hook=_byteify)
-
-
-def create_inventory_file(inventories_path,
-                          hostvars, kube_master, etcd, kube_node):
-    content = Environment().from_string(INVENTORY_TEMPLATE).render(
-              hostvars=hostvars, kube_master=kube_master,
-              etcd=etcd, kube_node=kube_node)
-    with open(inventories_path, 'w+') as f:
-        f.write(content)
-
-
-def main(inventories_path, local_inventory):
-    inventory_data = load_inventory(local_inventory)
-    hostvars = inventory_data['_meta']['hostvars']
-    kube_node = inventory_data['kube_node']['hosts']
-    kube_master = inventory_data['kube_master']['hosts']
-    etcd = inventory_data['etcd']['hosts']
-
-    create_inventory_file(inventories_path,
-                          hostvars, kube_master, etcd, kube_node)
-
-
-if __name__ == "__main__":
-    path = yaml.load(sys.argv[1])
-    local_inventory = yaml.load(sys.argv[2])
-
-    main(path, local_inventory)
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo b/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo
deleted file mode 100644
index 4900db69..00000000
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo
+++ /dev/null
@@ -1,32 +0,0 @@
-[base]
-name=CentOS-$releasever - Base
-mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra
-#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-
-#released updates
-[updates]
-name=CentOS-$releasever - Updates
-mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra
-#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-
-#additional packages that may be useful
-[extras]
-name=CentOS-$releasever - Extras
-mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra
-#baseurl=http://mirror.centos.org/centos/$releasever/extras/$basearch/
-gpgcheck=1
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-
-#additional packages that extend functionality of existing packages
-[centosplus]
-name=CentOS-$releasever - Plus
-mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra
-#baseurl=http://mirror.centos.org/centos/$releasever/centosplus/$basearch/
-gpgcheck=1
-enabled=0
-gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
-
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo b/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo
index 1d622d3c..1d622d3c 100644..100755
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
index 2763e53e..512121e2 100644..100755
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
@@ -7,107 +7,165 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 ---
-- name: clean local repo conf
-  file:
-    path: /etc/yum.repos.d
-    state: absent
-  run_once: "True"
-  when: ansible_os_family == 'RedHat'
-
-- name: create local repo conf dir
-  file:
-    path: /etc/yum.repos.d
-    state: directory
-  run_once: "True"
-  when: ansible_os_family == 'RedHat'
-
-- name: configure local mirror repo
-  copy:
-    src: "{{ item }}"
-    dest: /etc/yum.repos.d/mirrors.repo
-  with_first_found:
-    - mirrors_{{ ansible_architecture }}.repo
-    - mirrors.repo
-  run_once: "True"
-  when: ansible_os_family == 'RedHat'
-
-- name: clean local pip conf to use official pip repo
-  file:
-    path: /root/.pip/pip.conf
-    state: absent
-  run_once: "True"
-
-- name: install dependency for ansible update
-  yum:
-    name: "{{ item }}"
-    state: latest
-  with_items:
-    - git
-    - libffi-devel
-    - openssl-devel
-    - python-devel
-  run_once: "True"
-  when: ansible_os_family == 'RedHat'
-
-- name: update python packages
-  pip:
-    name: "{{ item }}"
-    state: latest
-  with_items:
-    - netaddr
-    - jinja2
+- name: check the kubespray sample path
+  stat: path=/opt/kargo_k8s/inventory/sample
+  register: sample_stat
 
+- name: Move kubespray group_vars folder
+  command: mv /opt/kargo_k8s/inventory/sample/group_vars /opt/kargo_k8s/inventory/
+  when: sample_stat.stat.exists
 
-- name: copy inventories generate script
-  copy:
-    src: generate_inventories.py
-    dest: /tmp/generate_inventories.py
+- name: generate kubespray inventory configure file
+  template:
+    src: "inventory.j2"
+    dest: "/opt/kargo_k8s/inventory/inventory.cfg"
   tags:
     - ansible
 
-- name: copy inventoriy.json file
+- name: copy overrided variables for arm architecture
   copy:
-    src: /var/ansible/run/kubernetes-opnfv2/inventories/inventory.json
-    dest: /tmp/inventory.json
-  tags:
-    - ansible
+    src: "{{ item }}"
+    dest: /opt/kargo_k8s/extra-vars.yml
+  with_first_found:
+    - extra-vars-{{ ansible_architecture }}.yml
+    - extra-vars.yml
+    - skip: true
 
-- name: generate kargo inventories
-  shell: >
-    python /tmp/generate_inventories.py \
-           "/opt/kargo_k8s/inventory/inventory.cfg" \
-           "/tmp/inventory.json"
+- name: copy overrided variables for kubespray
+  template:
+    src: "{{ item }}"
+    dest: "/opt/kargo_k8s/extra-vars.yml"
+  with_first_found:
+    - extra-vars-{{ ansible_architecture }}.yml.j2
+    - extra-vars.yml.j2
   tags:
     - ansible
 
-- name: configure target hosts
-  shell: |
-    cd /opt/kargo_k8s
-    ansible -i inventory/inventory.cfg -m ping all
-    ansible -i inventory/inventory.cfg all -m shell -a "rm /etc/yum.repos.d/*"
-    ansible -i inventory/inventory.cfg all -m copy -a \
-        "src=/etc/yum.repos.d/mirrors.repo dest=/etc/yum.repos.d"
-  tags:
-    - ansible
+- name: copy 2flannel playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/2flannel"
+    dest: /opt/kargo_k8s/roles/network_plugin
+
+- name: copy 2flannel-apps playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/2flannel-apps/"
+    dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/2flannel
 
-- name: enable helm
+- name: append 2flannel to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml
+    block: "  - role: network_plugin/2flannel\n    \
+      when: kube_network_plugin == '2flannel'\n    tags: 2flannel\n"
+
+- name: append 2flannel apps to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml
+    block: "  - role: kubernetes-apps/network_plugin/2flannel\n    \
+      when: kube_network_plugin == '2flannel'\n    tags: 2flannel\n"
+
+- name: append 2flannel to valid kube_network_plugin list
+  replace:
+    path: "/opt/kargo_k8s/roles/kubernetes/{{ item.path }}"
+    regexp: "{{ item.regexp }}"
+    replace: "{{ item.replace }}"
+  with_items:
+    - {path: 'master/templates/manifests/kube-controller-manager.manifest.j2',
+       regexp: '"cloud", "flannel"',
+       replace: '"cloud", "flannel", "2flannel"'}
+    - {path: 'node/templates/kubelet.kubeadm.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "2flannel"'}
+    - {path: 'node/templates/kubelet.standard.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "2flannel"'}
+    - {path: 'node/templates/kubelet.rkt.service.j2',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "2flannel"'}
+    - {path: 'preinstall/tasks/main.yml',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "2flannel"'}
+
+# yamllint disable rule:line-length
+- name: enable CSI plugin feature
+  lineinfile:
+    dest: "/opt/kargo_k8s/roles/kubespray-defaults/defaults/main.yaml"
+    regexp: "^kube_feature_gates:"
+    line: "{% raw %}kube_feature_gates: ['Initializers={{ istio_enabled|string }}', 'PersistentLocalVolumes={{ local_volumes_enabled|string }}', 'CSIPersistentVolume=True', 'MountPropagation=True']{% endraw %}"
+  when:
+    - stor4nfv is defined and stor4nfv == "Enable"
+# yamllint enable rule:line-length
+
+- name: enable CSI plugin runtime_config
   lineinfile:
-    dest: /opt/kargo_k8s/inventory/group_vars/k8s-cluster.yml
-    regexp: '^helm_enabled:'
-    line: 'helm_enabled: {{ helm_flag }}'
+    dest: /opt/kargo_k8s/roles/kubernetes/master/defaults/main.yml
+    insertafter: '^  - admissionregistration.k8s.io/v1alpha1'
+    line: '  - storage.k8s.io/v1alpha1'
+  when:
+    - stor4nfv is defined and stor4nfv == "Enable"
 
-- name: copy overrided variables
+- name: copy sriov playbook to kargo
   copy:
-    src: "{{ item }}"
-    dest: /opt/kargo_k8s/extra-vars.yml
-  with_first_found:
-    - extra-vars-{{ ansible_architecture }}.yml
-    - extra-vars.yml
+    src: "{{ run_dir }}/roles/sriov"
+    dest: /opt/kargo_k8s/roles/network_plugin
+
+- name: copy sriov-apps playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/sriov-apps/"
+    dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/sriov
+
+- name: append sriov to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml
+    block: "  - role: network_plugin/sriov\n    when: kube_network_plugin == 'sriov'\n    \
+      tags: sriov\n"
+
+- name: append sriov apps to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml
+    block: "  - role: kubernetes-apps/network_plugin/sriov\n    \
+      when: kube_network_plugin == 'sriov'\n    tags: sriov\n"
+
+- name: append sriov to valid kube_network_plugin list
+  replace:
+    path: "{{ item.path }}"
+    regexp: "{{ item.regexp }}"
+    replace: "{{ item.replace }}"
+  with_items:
+    - {path: "/opt/kargo_k8s/roles/kubernetes/master/templates/manifests/\
+kube-controller-manager.manifest.j2",
+       regexp: '"cloud", "flannel"',
+       replace: '"cloud", "flannel", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.standard.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.rkt.service.j2',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/preinstall/tasks/main.yml',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "sriov"'}
 
 - name: run kargo playbook
   shell: |
-    cd /opt/kargo_k8s
     ansible-playbook -i inventory/inventory.cfg cluster.yml \
                      -e "@extra-vars.yml" -b -v 2>&1  | tee kargo.log
+  args:
+    chdir: "/opt/kargo_k8s"
   tags:
     - ansible
+
+- name: read the ansible log file
+  shell: "cat /opt/kargo_k8s/kargo.log | tail -n 1000"
+  register: setup_kargo_result
+
+- fail:
+    msg: "some task failed when setup kargo."
+  when: setup_kargo_result.stdout.find('failed=1') != -1
+
+- fail:
+    msg: "some host are unreachable."
+  when: setup_kargo_result.stdout.find('unreachable=1') != -1
+  run_once: true
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2
new file mode 100755
index 00000000..1d7a2fa2
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2
@@ -0,0 +1,40 @@
+---
+# Override default kubespray variables
+
+#dashboard_port: "{{dashboard_port|default('31746')}}"
+
+# kubespray configure
+apiserver_loadbalancer_domain_name: "{{ public_vip.ip  }}"
+loadbalancer_apiserver:
+  address: "{{ public_vip.ip  }}"
+  port: {{ loadbalancer_apiserver_port|default(8383) }}
+loadbalancer_apiserver_localhost: {{ loadbalancer_apiserver_localhost|default(true) }}
+
+kube_basic_auth: {{ kube_basic_auth |default(true) }}
+kube_network_plugin: {{ kube_network_plugin|default('calico') }}
+# Monitoring apps for k8s
+efk_enabled: {{ efk_enabled |default(true)}}
+# Helm deployment
+helm_enabled: {{ helm_enabled |default(true)}}
+# Istio deployment
+istio_enabled: {{ istio_enabled |default(false)}}
+supplementary_addresses_in_ssl_keys: ["{{ public_vip.ip  }}"]
+#storage
+local_volume_provisioner_enabled: {{local_volume_provisioner_enabled |default(false) }}
+# local_volume_provisioner_namespace: "system_namespace"
+# local_volume_provisioner_base_dir: /mnt/disks
+# local_volume_provisioner_mount_dir: /mnt/disks
+# local_volume_provisioner_storage_class: local-storage
+
+# CephFS provisioner deployment
+cephfs_provisioner_enabled: {{ cephfs_provisioner_enabled |default(false)}}
+# cephfs_provisioner_namespace: "cephfs-provisioner"
+# cephfs_provisioner_cluster: ceph
+# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# cephfs_provisioner_admin_id: admin
+# cephfs_provisioner_secret: secret
+# cephfs_provisioner_storage_class: cephfs
+# cephfs_provisioner_reclaim_policy: Delete
+# cephfs_provisioner_claim_root: /volumes
+# cephfs_provisioner_deterministic_names: true
+
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2
new file mode 100644
index 00000000..0120ae18
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2
@@ -0,0 +1,26 @@
+[all]
+{% for host, vales in hostvars.iteritems() %}
+{{ host }} ansible_ssh_host={{ vales['ansible_ssh_host'] }} ansible_ssh_pass=root  ansible_user=root
+{% endfor %}
+
+[kube-master]
+{% for host in hostvars[inventory_hostname]['groups']['kube_master'] %}
+{{ host }}
+{% endfor %}
+
+[etcd]
+{% for host in hostvars[inventory_hostname]['groups']['etcd'] %}
+{{ host }}
+{% endfor %}
+
+[kube-node]
+{% for host in hostvars[inventory_hostname]['groups']['kube_node'] %}
+{{ host }}
+{% endfor %}
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[calico-rr]
+[vault]
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
index 2d396d06..af9c9675 100644..100755
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
@@ -1,2 +1,3 @@
 ---
-helm_flag: true
+http_proxy: "{{ proxy }}"
+https_proxy: "{{ proxy }}"