Update BMRA to v2.1

Updated Kuberef to use BMRA v2.1 instead of v2.0.
Removed a few workarounds used for BMRA v2.0.
Updated a few dependencies in deployment container.
Updated Kuberef templates for BMRA configuration.

Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: I091da09277ca9ecc0d3d7574dc3c4f000e73432d
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72064
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>

9 files changed, 17 insertions, 397 deletions

diff --git a/functions.sh b/functions.sh
index cb099d5..03e2284 100755
--- a/functions.sh
+++ b/functions.sh
@@ -194,7 +194,7 @@ provision_k8s() {
     ansible_cmd="/bin/bash -c '"
     if [[ "$DEPLOYMENT" == "k8s" ]]; then
         ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml
-        ansible_cmd+="pip install --upgrade pip==9.0.3; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
+        ansible_cmd+="yum -y remove python-netaddr; pip install --upgrade pip; pip install ansible==2.9.6; ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/k8s/patch_kubespray.yml;"
     fi
     ansible_cmd+="ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/${BMRA_PROFILE}.yml'"
 
@@ -210,7 +210,7 @@ if ! command -v docker; then
     done
 fi
 if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then
-    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.0.0 ${PROJECT_ROOT}/container-experience-kits/
+    git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v2.1.0 ${PROJECT_ROOT}/container-experience-kits/
     cp -r ${PROJECT_ROOT}/container-experience-kits/examples/${BMRA_PROFILE}/group_vars ${PROJECT_ROOT}/container-experience-kits/
 # NOTE The following condition/workaround will be removed once the reported issue https://github.com/intel/container-experience-kits/issues/68
 # is fixed upstream
@@ -225,12 +225,6 @@ cp ${PROJECT_ROOT}/${INSTALLER}/inventory.ini \
     ${PROJECT_ROOT}/container-experience-kits/
 cp ${PROJECT_ROOT}/${INSTALLER}/{all.yml,kube-node.yml} \
     ${PROJECT_ROOT}/container-experience-kits/group_vars/
-cp ${PROJECT_ROOT}/${INSTALLER}/dpdk_patch.yml \
-    ${PROJECT_ROOT}/container-experience-kits/roles/install_dpdk/tasks/main.yml
-cp ${PROJECT_ROOT}/${INSTALLER}/patched_tas.yml \
-    ${PROJECT_ROOT}/container-experience-kits/roles/tas_install/tasks/tas.yml
-cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk.yml \
-    ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/charts/cpu-manager-for-kubernetes/templates/job.yml
 cp ${PROJECT_ROOT}/${INSTALLER}/patched_cmk_build.yml \
     ${PROJECT_ROOT}/container-experience-kits/roles/cmk_install/tasks/main.yml
 sudo docker run --rm \
diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml
index 434ada2..d085c7c 100644
--- a/hw_config/equinix-metal/idf.yaml
+++ b/hw_config/equinix-metal/idf.yaml
@@ -77,5 +77,5 @@ bmra:
       policy: "best-effort"
     tas:
       enable: true
-      demo_policy: true
+      demo_policy: false
     psp: true
diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml
index 13b8054..07c45cc 100644
--- a/hw_config/intel/idf.yaml
+++ b/hw_config/intel/idf.yaml
@@ -106,5 +106,5 @@ bmra:
       policy: "best-effort"
     tas:
       enable: true
-      demo_policy: true
+      demo_policy: false
     psp: true
diff --git a/playbooks/roles/bmra-config/templates/all.j2 b/playbooks/roles/bmra-config/templates/all.j2
index 1017331..6dc074e 100644
--- a/playbooks/roles/bmra-config/templates/all.j2
+++ b/playbooks/roles/bmra-config/templates/all.j2
@@ -65,7 +65,7 @@ topology_manager_policy: "{{ bmra.features.topology_manager.policy }}"
 sriov_net_dp_enabled: {{ bmra.features.sriov_net_dp }}
 sriov_net_dp_namespace: kube-system
 # whether to build and store image locally or use one from public external registry
-sriov_net_dp_build_image_locally: false
+sriov_net_dp_build_image_locally: true
 # SR-IOV network device plugin configuration.
 # For more information on supported configuration refer to: https://github.com/intel/sriov-network-device-plugin#configurations
 sriovdp_config_data: |
diff --git a/playbooks/roles/bmra-config/templates/inventory.j2 b/playbooks/roles/bmra-config/templates/inventory.j2
index 28581e0..6008179 100644
--- a/playbooks/roles/bmra-config/templates/inventory.j2
+++ b/playbooks/roles/bmra-config/templates/inventory.j2
@@ -2,7 +2,7 @@
 {% for node in nodes %}
 {{ idf.kubespray.hostnames[node.name] }}   ansible_host={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }} ip={{ node.interfaces[idf.net_config[engine.pxe_network].interface].address }}
 {% endfor %}
-localhost
+localhost ansible_python_interpreter=/usr/bin/python2
 
 {% for role in ['kube-master', 'etcd', 'kube-node'] %}
 [{{ role }}]
diff --git a/playbooks/roles/bmra-config/templates/kube-node.j2 b/playbooks/roles/bmra-config/templates/kube-node.j2
index 302fa8f..f32fbdb 100644
--- a/playbooks/roles/bmra-config/templates/kube-node.j2
+++ b/playbooks/roles/bmra-config/templates/kube-node.j2
@@ -16,11 +16,11 @@
 ---
 # Kubernetes node configuration
 
-# Enable SR-IOV networking related setup
-sriov_enabled: {{ bmra.features.sriov.enable }}
+# Enable IOMMU (required for SR-IOV networking)
+iommu_enabled: {{ bmra.features.sriov.enable }}
 
-# SR-IOV PF specific configuration list
-sriov_nics:
+# dataplane interface configuration list
+dataplane_interfaces:
 {% for intf in bmra.network_roles.sriov %}
   - name: {{ intf.name }}
     bus_info: "{{ intf.bus_info }}"
@@ -41,8 +41,8 @@ sriov_cni_enabled: {{ bmra.features.sriov_cni }}
 # Bond CNI
 bond_cni_enabled: false
 
-# install DPDK
-install_dpdk: true # DPDK installation is required for sriov_enabled:true; default to false
+# Install DPDK (required for SR-IOV networking)
+install_dpdk: true
 
 # Userspace networking
 userspace_cni_enabled: false
@@ -62,10 +62,10 @@ update_nic_drivers: false
 update_nic_firmware: false
 
 # Additional list of NIC interfaces that the FW update will be executed on
-# NOTE: FW update will be also executed on all NICs listed in "sriov_nics[*].name"
-firmware_update_nics:
-  - intf1
-  - intf2
+# NOTE: FW update will be also executed on all NICs listed in "dataplane_interfaces[*].name"
+firmware_update_nics: []
+#  - enp24s0f0
+#  - enp24s0f1
 
 # install Intel x700 & x800 series NICs DDP packages
 install_ddp_packages: false
@@ -93,7 +93,7 @@ cpu_count: 144
 # Enable/Disable Intel PState scaling driver
 intel_pstate_enabled: true
 # Config options for intel_pstate: disable, passive, force, no_hwp, hwp_only, support_acpi_ppc, per_cpu_perf_limits
-intel_pstate: disable
+intel_pstate: hwp_only
 # Enable/Disable Intel Turbo Boost PState attribute
 turbo_boost_enabled: false
 
diff --git a/sw_config/bmra/dpdk_patch.yml b/sw_config/bmra/dpdk_patch.yml
deleted file mode 100644
index ec77b39..0000000
--- a/sw_config/bmra/dpdk_patch.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-##
-##   Copyright (c) 2020 Intel Corporation.
-##
-##   Licensed under the Apache License, Version 2.0 (the "License");
-##   you may not use this file except in compliance with the License.
-##   You may obtain a copy of the License at
-##
-##       http://www.apache.org/licenses/LICENSE-2.0
-##
-##   Unless required by applicable law or agreed to in writing, software
-##   distributed under the License is distributed on an "AS IS" BASIS,
-##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-##   See the License for the specific language governing permissions and
-##   limitations under the License.
-##
----
-- name: install dependencies
-  include_role:
-    name: install_dependencies
-
-- name: download DPDK
-  unarchive:
-    src: "{{ dpdk_url }}"
-    dest: "/usr/src"
-    remote_src: yes
-    list_files: yes
-    mode: 0755
-  register: dpdk_download
-
-- name: set local dpdk directory path
-  set_fact:
-    dpdk_dir: "{{ dpdk_download.dest  }}/{{ dpdk_download.files[0] }}"
-
-- name: run make config
-  make:
-    chdir: "{{ dpdk_dir }}"
-    target: config
-    params:
-      T: "{{ dpdk_target }}"
-
-- name: update ansible_kernel fact
-  setup:
-    filter: 'ansible_kernel'
-
-- name: patch DPDK (kni)
-  lineinfile:
-    path: "{{ dpdk_dir }}/kernel/linux/kni/kni_net.c"
-    regexp: '^\s*kni_net_tx_timeout'
-    line: 'kni_net_tx_timeout(struct net_device *dev, unsigned int txqueue)'
-  when:
-    - ansible_distribution == "CentOS"
-    - ansible_distribution_major_version == '8'
-    - ansible_kernel is version('4.18.0-240','>=')
-
-- name: enable virtio-user support
-  lineinfile:
-    path: "{{ dpdk_dir }}/build/.config"
-    regexp: '^CONFIG_RTE_VIRTIO_USER'
-    line: 'CONFIG_RTE_VIRTIO_USER=y'
-    mode: 0600
-
-- name: enable PCAP PMD support
-  lineinfile:
-    path: "{{ dpdk_dir }}/build/.config"
-    regexp: '^CONFIG_RTE_LIBRTE_PMD_PCAP'
-    line: 'CONFIG_RTE_LIBRTE_PMD_PCAP=y'
-    mode: 0600
-
-- name: build DPDK
-  make:
-    target: install
-    chdir: "{{ dpdk_dir }}"
-    params:
-      T: "{{ dpdk_target }}"
-      DESTDIR: install
-      prefix: "/usr"
-
-- name: find dpdk tools
-  find:
-    path: "{{ dpdk_dir }}"
-    patterns: "dpdk-devbind.py"
-    recurse: yes
-  register: dpdk_tools_dir
-
-- name: set path to dpdk usertools directory
-  set_fact:
-    dpdk_tools: "{{ dpdk_tools_dir.files[0].path }}"
-
-- name: load userspace modules
-  modprobe:
-    name: "{{ item }}"
-    state: present
-  with_items:
-    - vfio-pci
-    - uio
-
-- name: install dpdk-devbind.py in /usr/local/bin
-  copy:
-    remote_src: yes
-    src: "{{ dpdk_tools }}"
-    dest: "/usr/local/bin/dpdk-devbind.py"
-    mode: 0700
-    owner: root
-    group: root
-  become: yes
-
-- name: load intel module
-  command: "insmod {{ dpdk_dir }}/{{ dpdk_target }}/kmod/igb_uio.ko"
-  register: result
-  failed_when: "'No such file or directory' in result.stderr"
-  changed_when: "'already bound' not in result.stderr"
diff --git a/sw_config/bmra/patched_cmk.yml b/sw_config/bmra/patched_cmk.yml
deleted file mode 100644
index e97083a..0000000
--- a/sw_config/bmra/patched_cmk.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-{{- $fullName := include "cmk.fullname" . -}}
-{{- $exclusiveMode := .Values.exclusiveMode -}}
-{{- $numExclusiveCores := .Values.numExclusiveCores -}}
-{{- $sharedMode := .Values.sharedMode -}}
-{{- $numSharedCores := .Values.numSharedCores -}}
-{{- $pullPolicy := .Values.image.pullPolicy -}}
-{{- $image := .Values.image.repository -}}
-{{- $tag := .Values.image.tag -}}
-{{- $noTaint := .Values.noTaint }}
-{{ range splitList "," .Values.hosts.list }}
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  annotations:
-    helm.sh/hook: pre-install,pre-upgrade,pre-rollback
-    helm.sh/hook-weight: "10"
-  labels:
-    app: {{ $fullName }}-init-discover-{{ . }}
-  name: {{ $fullName }}-init-discover-{{ . }}
-spec:
-  template:
-    spec:
-      serviceAccountName: {{ $fullName }}
-      restartPolicy: Never
-      tolerations:
-      - key: cmk
-        operator: Exists
-      containers:
-        - name: install
-          image: {{ $image }}:{{ $tag }}
-          imagePullPolicy: {{ $pullPolicy }}
-          command: ["/bin/bash", "-c"]
-          args:
-          - "/cmk/cmk.py install"
-          volumeMounts:
-          - mountPath: /opt/bin
-            name: cmk-install-dir
-        - name: init
-          image: {{ $image }}:{{ $tag }}
-          imagePullPolicy: {{ $pullPolicy }}
-          env:
-          - name: CMK_PROC_FS
-            value: "/host/proc"
-          - name: NODE_NAME
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: spec.nodeName
-          command: ["/bin/bash", "-c"]
-          args:
-          - "/cmk/cmk.py init --exclusive-mode={{ $exclusiveMode }} --num-exclusive-cores={{ $numExclusiveCores }} --shared-mode={{ $sharedMode }} --num-shared-cores={{ $numSharedCores }}"
-          volumeMounts:
-          - mountPath: /host/proc
-            name: host-proc
-            readOnly: true
-        - name: discover
-          image: {{ $image }}:{{ $tag }}
-          imagePullPolicy: {{ $pullPolicy }}
-          env:
-          - name: CMK_PROC_FS
-            value: /host/proc
-          - name: NODE_NAME
-            valueFrom:
-              fieldRef:
-                apiVersion: v1
-                fieldPath: spec.nodeName
-          command: ["/bin/bash", "-c"]
-          args:
-          - "/cmk/cmk.py discover {{ if $noTaint }}--no-taint{{ end }}"
-          volumeMounts:
-          - mountPath: /host/proc
-            name: host-proc
-            readOnly: true
-      volumes:
-      - hostPath:
-          path: /proc
-          type: Directory
-        name: host-proc
-      - hostPath:
-          path: /opt/bin
-          type: DirectoryOrCreate
-        name: cmk-install-dir
-{{ end }}
diff --git a/sw_config/bmra/patched_tas.yml b/sw_config/bmra/patched_tas.yml
deleted file mode 100644
index 633f5d7..0000000
--- a/sw_config/bmra/patched_tas.yml
+++ /dev/null
@@ -1,179 +0,0 @@
-##
-##   Copyright (c) 2020 Intel Corporation.
-##
-##   Licensed under the Apache License, Version 2.0 (the "License");
-##   you may not use this file except in compliance with the License.
-##   You may obtain a copy of the License at
-##
-##       http://www.apache.org/licenses/LICENSE-2.0
-##
-##   Unless required by applicable law or agreed to in writing, software
-##   distributed under the License is distributed on an "AS IS" BASIS,
-##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-##   See the License for the specific language governing permissions and
-##   limitations under the License.
-##
----
-- name : install dependencies
-  include_role:
-    name: install_dependencies
-
-- name: update to git2 on RHEL 7 based distros
-  include_role:
-    name: git2_install
-  when:
-    - ansible_distribution == 'RedHat' or ansible_distribution == 'CentOS'
-    - ansible_distribution_version < '8'
-
-- name: check if stable repo has already been added
-  command: helm repo list
-  register: helm_repo_list
-  failed_when: false
-  changed_when: false
-
-- name: add Helm stable repo
-  command: helm repo add stable https://charts.helm.sh/stable
-  when: '"https://charts.helm.sh/stable" not in helm_repo_list.stdout'
-  register: helm_add_result
-  changed_when: '"has been added to your repositories" in helm_add_result.stdout'
-
-- name: update Helm repo before installation of public charts
-  command: helm repo update
-  register: helm_update_result
-  changed_when: '"Successfully got an update" in helm_update_result.stdout'
-
-
-- name: create Helm charts directory if needed
-  file:
-    path: /usr/src/charts
-    state: directory
-    mode: 0755
-
-- name: generate cert and key
-  include_role:
-    name: create_signed_k8s_certs
-  vars:
-    secret_name: "{{ tas_extender_secret_name }}"
-    service_name: tas-telemetry-aware-scheduling
-    key_pair_name: tas
-    host_secrets_folder: "{{ tas_ssl_mount_path }}"
-    k8s_namespace: "{{ tas_namespace }}"
-    csr_cluster_name: "{{ cluster_name | default('cluster.local') }}"
-  when: tas_tls_enabled
-
-- name: clone TAS repository
-  git:
-    repo: "{{ tas_git_url }}"
-    version: "{{ tas_git_version }}"
-    dest: "{{ tas_dir }}"
-    force: yes
-
-- name: make build and make image - TAS
-  make:
-    target: "{{ item }}"
-    chdir: "{{ tas_dir }}"
-  loop:
-    - build
-    - image
-
-- name: tag TAS-controller and TAS-extender
-  # TAS Makefile always creates ":latest" version images
-  command: docker tag {{ item }}:latest {{ registry_local_address }}/{{ item }}:{{ tas_version }}
-  loop:
-    - tas-controller
-    - tas-extender
-  changed_when: false
-
-- name: push TAS-controller and TAS-extender image to local registry
-  command: docker push {{ registry_local_address }}/{{ item }}:{{ tas_version }}
-  loop:
-    - tas-controller
-    - tas-extender
-  changed_when: true
-
-- name: create descheduler directory if needed
-  file:
-    path: "{{ sigs_k8s_io_dir }}"
-    state: directory
-    mode: 0755
-
-- name: clone Descheduler for Kubernetes
-  git:
-    repo: "{{ descheduler_git_url }}"
-    dest: "{{ descheduler_dir }}"
-    force: yes
-    version: "{{ descheduler_git_version }}"
-
-- name: install descheduler
-  make:
-    chdir: "{{ descheduler_dir }}"
-
-- name: copy Helm chart resource definition to controller node
-  copy:
-    src: "{{ role_path }}/charts/{{ item }}"
-    dest: "/usr/src/charts/"
-    mode: preserve
-  loop:
-    - telemetry-aware-scheduling
-    - tas-policy-crd.yml
-
-- name: populate tas Helm chart values template and push to controller node
-  template:
-    src: "tas-values.yml.j2"
-    dest: "/usr/src/charts/tas-values.yml"
-    force: yes
-    mode: preserve
-
-- name: create TASPolicy resource
-  command: kubectl apply -f tas-policy-crd.yml
-  args:
-    chdir: "/usr/src/charts"
-  changed_when: true
-
-- name: install TAS helm chart
-  command: helm upgrade --install --namespace {{ tas_namespace }} {{ tas_name }} -f tas-values.yml telemetry-aware-scheduling/
-  args:
-    chdir: "/usr/src/charts"
-  retries: 5
-  delay: 5
-  register: result
-  until: result.rc == 0
-  changed_when: true
-
-- name: Configure arguments from Kubernetes Scheduler file if they exist - dnsPolicy
-  lineinfile:
-    path: /etc/kubernetes/manifests/kube-scheduler.yaml
-    insertafter: "spec:"
-    line: "  dnsPolicy: ClusterFirstWithHostNet"
-    regexp: "  dnsPolicy: "
-    state: present
-    mode: 0600
-
-- name: Configure arguments to our kube-scheduler manifest - configmap
-  lineinfile:
-    path: /etc/kubernetes/manifests/kube-scheduler.yaml
-    insertafter: "    - kube-scheduler"
-    line: "{{ item.arg }}={{ item.value }}"
-    regexp: "{{ item.arg }}"
-    state: present
-    mode: 0600
-  with_items:
-  - { arg: "    - --policy-configmap", value: "{{ tas_name }}-telemetry-aware-scheduling-scheduler-extender-policy" }
-  - { arg: "    - --policy-configmap-namespace", value: "{{ tas_namespace }}" }
-
-# TAS Demo Policy
-- name: template TAS demo policy
-  template:
-    src: "tas-demo-policy.yml.j2"
-    dest: "/usr/src/charts/tas-demo-policy.yml"
-    force: yes
-    mode: preserve
-  when:
-    - tas_enable_demo_policy
-
-- name: create TAS demo policy resource
-  command: kubectl apply -f tas-demo-policy.yml
-  args:
-    chdir: "/usr/src/charts"
-  when:
-    - tas_enable_demo_policy