diff options
Diffstat (limited to 'sw_config/bmra')
| -rw-r--r-- | sw_config/bmra/Dockerfile | 2 | ||||
| -rw-r--r-- | sw_config/bmra/patched_cmk_build.yml | 65 | ||||
| -rw-r--r-- | sw_config/bmra/patched_k8s.yml | 30 | ||||
| -rw-r--r-- | sw_config/bmra/patched_kubespray_requirements.txt | 6 | ||||
| -rw-r--r-- | sw_config/bmra/patched_packages.yml | 7 | ||||
| -rw-r--r-- | sw_config/bmra/patched_preflight.yml | 544 | ||||
| -rw-r--r-- | sw_config/bmra/patched_rhel_packages.yml | 85 | ||||
| -rw-r--r-- | sw_config/bmra/patched_sriov_cni_install.yml | 44 |
8 files changed, 733 insertions, 50 deletions
diff --git a/sw_config/bmra/Dockerfile b/sw_config/bmra/Dockerfile index 3f21241..d464822 100644 --- a/sw_config/bmra/Dockerfile +++ b/sw_config/bmra/Dockerfile @@ -10,7 +10,7 @@ RUN yum -y update && \ yum -y install git epel-release python36 python-netaddr && \ yum -y install python-pip && \ pip install --no-cache-dir pip==9.0.3 && \ - pip install --no-cache-dir ansible==2.9.17 jmespath && \ + pip install --no-cache-dir ansible==2.9.20 jmespath && \ pip install --no-cache-dir jinja2 --upgrade CMD ["bash"] diff --git a/sw_config/bmra/patched_cmk_build.yml b/sw_config/bmra/patched_cmk_build.yml index 0b5c774..a424c55 100644 --- a/sw_config/bmra/patched_cmk_build.yml +++ b/sw_config/bmra/patched_cmk_build.yml @@ -1,37 +1,12 @@ -# SPDX-FileCopyrightText: 2020 Intel Corporation. +# SPDX-FileCopyrightText: 2021 Intel Corporation. # # SPDX-License-Identifier: Apache-2.0 --- -- name: install epel-release on Red Hat based OS - package: name=epel-release - when: ansible_os_family == 'RedHat' - -# note: on Ubuntu, pip is installed via install_dependencies -- name: install pip - package: - name: python-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version < '8' - -- name: install pip - package: - name: python3-pip - when: - - ansible_distribution in ["RedHat", "CentOS"] - - ansible_distribution_version >= '8' - - name: install dependencies include_role: name: install_dependencies -- name: install Python dependencies - pip: - name: - - setuptools - - docker - - name: clone CMK repository git: repo: "{{ cmk_git_url }}" @@ -61,25 +36,42 @@ - name: build CMK image make: chdir: "{{ cmk_dir }}" + when: container_runtime == "docker" # NOTE(przemeklal): this fixes problem in CMK with ImagePullPolicy hardcoded to Never and the pod is scheduled on controller node - name: tag CMK image command: docker tag cmk:{{ cmk_img_version }} {{ registry_local_address }}/cmk:{{ cmk_img_version }} changed_when: true + when: container_runtime == "docker" - name: push CMK image to local registry command: docker push {{ registry_local_address }}/cmk:{{ cmk_img_version }} + changed_when: true when: + - container_runtime == "docker" - inventory_hostname == groups['kube-node'][0] + +- name: build and tag CMK image + command: podman build -f Dockerfile -t {{ registry_local_address }}/cmk:{{ cmk_img_version }} + args: + chdir: "{{ cmk_dir }}" changed_when: true + when: '"docker" not in container_runtime' + +- name: push CMK image to local registry + command: podman push {{ registry_local_address }}/cmk:{{ cmk_img_version }} + changed_when: true + when: + - inventory_hostname == groups['kube-node'][0] + - '"docker" not in container_runtime' -- name: clean up any preexisting certs/key/CSR files +- name: clean up any pre-existing certs/key/CSR files file: path=/etc/ssl/cmk state=absent when: inventory_hostname == groups['kube-master'][0] failed_when: false become: yes -- name: delete any preexisting certs/key/CSR from Kubernetes +- name: delete any pre-existing certs/key/CSR from Kubernetes command: kubectl delete csr cmk-webhook-{{ item }}.{{ cmk_namespace }} when: inventory_hostname == groups['kube-master'][0] failed_when: false @@ -212,7 +204,7 @@ when: - inventory_hostname == groups['kube-master'][0] -- name: get approved server certificate +- name: get approved server certificate shell: kubectl get csr cmk-webhook-server.{{ cmk_namespace }} -o jsonpath='{.status.certificate}' args: chdir: "/etc/ssl/cmk/" @@ -272,8 +264,13 @@ - name: restart kube-apiserver after updating admission control configuration when: inventory_hostname == groups['kube-master'][0] block: - - name: remove kube-apiserver Docker container - shell: docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f + - name: remove kube-apiserver container + # noqa 305 - shell is used intentionally here + shell: >- + {{ (container_runtime == 'docker') | ternary('docker ps -af name=k8s_kube-apiserver* -q | + xargs --no-run-if-empty docker rm -f', + 'crictl ps -a --name=kube-apiserver* -q | + xargs --no-run-if-empty crictl rm -f') }} args: executable: /bin/bash register: remove_apiserver_container @@ -345,11 +342,11 @@ when: - inventory_hostname == groups['kube-master'][0] -# remove any preexisting configmaps before cmk redeployment -- name: remove any preexisting configmaps before CMK deployment +# remove any pre-existing configmaps before cmk redeployment +- name: remove any pre-existing configmaps before CMK deployment command: kubectl delete cm cmk-config-{{ inventory_hostname }} when: - - inventory_hostname in cmk_hosts_list.split(',') + - inventory_hostname in (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) delegate_to: "{{ groups['kube-master']|first }}" failed_when: false diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml index 5dfc3bd..fb0d43a 100644 --- a/sw_config/bmra/patched_k8s.yml +++ b/sw_config/bmra/patched_k8s.yml @@ -30,6 +30,10 @@ {%- endif -%} enable_admission_plugins_prepare: >- [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}] + bmra_docker_version: >- + {% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %} + flannel_backend_type: >- + {% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %} kube_config_dir: /etc/kubernetes - name: set kube_cert_dir set_fact: @@ -38,15 +42,32 @@ environment: "{{ proxy_env | d({}) }}" any_errors_fatal: true +- hosts: all + tasks: + - name: add docker runtime vars + set_fact: + container_manager: docker + docker_iptables_enabled: true + docker_dns_servers_strict: false + docker_version: "{{ bmra_docker_version }}" + when: container_runtime == "docker" + - name: add containerd runtime vars + set_fact: + container_manager: containerd + etcd_deployment_type: host + containerd_extra_args: |2 + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry_local_address }}"] + endpoint = ["https://{{ registry_local_address }}"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry_local_address }}".tls] + ca_file = "/etc/containers/certs.d/{{ registry_local_address }}/ca.crt" + when: container_runtime == "containerd" - name: run kubespray import_playbook: kubespray/cluster.yml vars: kubeadm_enabled: true multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf - docker_iptables_enabled: true - docker_dns_servers_strict: false + nginx_image_tag: 1.21.1 override_system_hostname: false - docker_version: '19.03' kube_proxy_mode: iptables enable_nodelocaldns: false system_reserved: true @@ -105,6 +126,7 @@ - name: restart docker daemon to recreate iptables rules systemd: name=docker state=restarted become: yes + when: container_runtime == "docker" - name: restart kubelet to trigger static pods recreation systemd: name=kubelet state=restarted become: yes @@ -144,7 +166,7 @@ roles: - role: cluster_defaults tags: defaults - - role: docker_registry + - role: container_registry tags: registry - role: dockerhub_credentials when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file" diff --git a/sw_config/bmra/patched_kubespray_requirements.txt b/sw_config/bmra/patched_kubespray_requirements.txt index b6cf112..cceb6ff 100644 --- a/sw_config/bmra/patched_kubespray_requirements.txt +++ b/sw_config/bmra/patched_kubespray_requirements.txt @@ -2,10 +2,12 @@ # # SPDX-License-Identifier: Apache-2.0 -ansible==2.9.17 -jinja2==2.11.1 +ansible==2.9.20 +cryptography==2.8 +jinja2==2.11.3 netaddr==0.7.19 pbr==5.4.4 jmespath==0.9.5 ruamel.yaml==0.16.10 +MarkupSafe==1.1.1 ruamel.yaml.clib==0.2.2 diff --git a/sw_config/bmra/patched_packages.yml b/sw_config/bmra/patched_packages.yml index 90a8519..14d7291 100644 --- a/sw_config/bmra/patched_packages.yml +++ b/sw_config/bmra/patched_packages.yml @@ -19,7 +19,7 @@ - setuptools<=44 extra_args: --upgrade -#pinned python package versions +#pinned python packages versions - name: install Python packages pip: name: @@ -27,7 +27,12 @@ - ruamel.yaml.clib==0.2.2 - ruamel.yaml==0.16.13 - cachetools=={{ (ansible_os_family == 'RedHat' and ansible_distribution_version < '8') | ternary('3.1.1', '4.2.1') }} + - markupsafe==1.1.1 + - jinja2==2.11.3 - openshift==0.11.2 + - six>=1.15.0 + - websocket-client==0.58.0 + - oauthlib==3.1.0 state: present register: pip_result retries: 5 diff --git a/sw_config/bmra/patched_preflight.yml b/sw_config/bmra/patched_preflight.yml new file mode 100644 index 0000000..41b7efd --- /dev/null +++ b/sw_config/bmra/patched_preflight.yml @@ -0,0 +1,544 @@ +# SPDX-FileCopyrightText: 2021 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 + +--- +# Preflight: ALL checks must PASS +# Only assert issues (do NOT change anything) +# +# Manual run: 'ansible-playbook -i inventory.ini playbooks/preflight.yml --flush-cache' +# +# Summary: +# On Ansible Host (localhost): +# - Check Ansible version (match) +# - Check Python version (min) +# - Check Group Vars (exist) +# - Check CMK Hosts (valid targets) +# On All targets (k8s-cluster): +# - Check Linux Distro +# - Check Hostnames (match Inventory) +# - Check CMK Config (isolcpus defined) +# - Check isolcpus Total (not more than actual) +# - Check isolcpus IDs (valid on system) +# - Check isolcpus OS Reserved (not 0,1,etc) +# On Worker Nodes Only (kube-node): +# - Check DP Interfaces (is not empty) +# - Check DP Interfaces Name (optional) +# - Check DP Interfaces Bus Info (pciid) +# - Check QAT Devices Bus Info (pciid) +# - Check QAT SRIOV VFs (max) +# - Check SGX configuration +# - Check OVS DPDK Dependencies (for 1G Hugepages) +# - Check VPP Dependencies (for 2M Hugepages) +# - Check CNI Dependencies (for OVS DPDK or VPP and Hugepages) +# - Check SST (not on RHEL 8.2 or old OSs) +# - Warn BIOS VT-d (should be enabled) +# - Warn BIOS Hyper-Threading (should be enabled) +# - Warn collectd (kernel update needed on old OSs) +# - Check OVS DPDK Version Compatability (for OVS support) + +# additional vars required: +# bmra_supported_ansible: # must be version +# bmra_supported_python: # min version +# bmra_supported_distros: [] # list +# bmra_supported_distros_versions: [] # list + + +################################## +# Prerequisites for Ansible Host # +################################## +- hosts: 127.0.0.1 + connection: local + vars: + bmra_supported_ansible: 2.9.20 + bmra_supported_python: 2.7 + + tasks: + + - debug: msg="Ansible version is {{ ansible_version.string }}" + - name: Check Ansible Version + assert: + that: (ansible_version.full is version_compare(bmra_supported_ansible, '==')) + msg: "Ansible version must be {{ bmra_supported_ansible }}. Please update" + + - debug: msg="Python version is {{ ansible_python_version }}" + - name: Check Python Version + assert: + that: (ansible_python_version is version_compare(bmra_supported_python, '>=')) + msg: "Python version must be at least {{ bmra_supported_python }}. Please update" + + - name: Read Group Vars + stat: + path: "{{ inventory_dir }}/group_vars/" + register: group_vars_details + + - name: Check Group Vars + assert: + that: "group_vars_details.stat.exists and group_vars_details.stat.isdir" + msg: "File group_vars/all.yml does NOT exist. Must be created per Guide" + + - debug: + msg: + - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) + - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml) + - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) + - all targets = {{ groups['all'] }} (inventory.ini) + when: cmk_enabled is defined # CMK expected true for all profiles except basic + + - name: Check Intel CMK Hosts + assert: + that: "item in groups['all']" + msg: "Hostname '{{ item }}' is NOT a valid target from inventory. Please correct the cmk_hosts_list or disable the CMK feature in group vars" + with_items: "{{ (cmk_hosts_list.split(',') if (cmk_hosts_list is defined and cmk_hosts_list | length > 0) else []) }}" + when: cmk_enabled is defined and cmk_enabled and not cmk_use_all_hosts + + +############################################## +# Prerequisites for Control and Worker Nodes # +############################################## +- hosts: k8s-cluster + vars: + bmra_supported_distros: [CentOS, RedHat, Ubuntu] + bmra_supported_distros_versions: ['7.6', '7.8', '7.9', '8.2', '8.3', '8.4', '18.04', '20.04', '21.04'] + isolcpus_ranges: [] + isolcpus_discretes: [] + + tasks: + + - debug: msg="Linux distribution on target is {{ ansible_distribution }} {{ ansible_distribution_version }} {{ ansible_distribution_release }}" + - name: Check Linux Distro and Version + assert: + that: "ansible_distribution in bmra_supported_distros and ansible_distribution_version in bmra_supported_distros_versions" + msg: + - Linux distribution {{ ansible_distribution }} {{ ansible_distribution_version }} on target '{{ inventory_hostname }}' is NOT supported + - Must be one of {{ bmra_supported_distros }} and version {{ bmra_supported_distros_versions }} + +# - name: Check Linux Across Cluster +# TODO ?? Linux OS must be the same on all targets (no mix-n-match) + + - name: regather network facts in case hostname recently changed + setup: + gather_subset: network + - debug: msg="Inventory target '{{ inventory_hostname }}' has the actual system hostname '{{ ansible_hostname }}'" + - name: Check Inventory Hostnames + debug: + msg: + - "Target '{{ inventory_hostname }}' in inventory does NOT match the actual system hostname '{{ ansible_hostname }}'." + - "If it's done intentionally, please ignore this message." + when: + - inventory_hostname != ansible_hostname + +# Early check if SELinux is configured properly + - block: + - name: "Collect packages facts" + package_facts: + - debug: + msg: + - "Current SELinux status:" + - "status: {{ ansible_selinux.status | default('') }}" + - "policy version: {{ ansible_selinux.policyvers | default('') }}" + - "type: {{ ansible_selinux.type | default('') }}" + - "mode: {{ ansible_selinux.mode | default('') }}" + - "config_mode: {{ ansible_selinux.config_mode | default('') }}" + + - name: check selinux condition possibly causing system boot failure + debug: + msg: + - "Current SELinux setup might cause the system possibly will not boot up on next reboot." + - "Please, check SELinux settings and set it up according to the documentation." + when: + - "'selinux-policy' not in ansible_facts.packages" + - "'selinux-policy-targeted' not in ansible_facts.packages" + when: + - ansible_os_family == "RedHat" + +# STORY: "cmk requires isolcpus to be configured" + - block: + - debug: + msg: + - cmk_enabled = {{ cmk_enabled }} (group_vars/all.yml) + - cmk_use_all_hosts = {{ cmk_use_all_hosts }} (group_vars/all.yml) + - cmk_hosts_list = {{ cmk_hosts_list | default('') }} (group_vars/all.yml) + - cmk_shared_num_cores = {{ cmk_shared_num_cores }} (group_vars/all.yml) + - cmk_exclusive_num_cores = {{ cmk_exclusive_num_cores }} (group_vars/all.yml) + - isolcpus_enabled = {{ isolcpus_enabled }} (host_vars) + - isolcpus = {{ isolcpus }} (host_vars) + - ansible_processor_count = {{ ansible_processor_count }} + - ansible_processor_cores = {{ ansible_processor_cores }} + - ansible_processor_threads_per_core = {{ ansible_processor_threads_per_core }} + - ansible_processor_vcpus = {{ ansible_processor_vcpus }} + - CPUs Reserved for OS = 0...{{ ansible_processor_count - 1 }} +# - CPUs Reserved for OS = {{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }} # [E207] Nested jinja pattern + + - name: Check Intel CMK Config + assert: + that: ({{ cmk_enabled }} and {{ isolcpus_enabled }} and "{{ isolcpus }}" | length > 0) + msg: + - Incorrect configuration pertaining Intel CMK. Conflicting or improper values detected + - When Intel CMK is enabled, CPUs isolation ('isolcpus') must be set according to the example file for host_vars. Please correct the configuration + + - name: Split isolcpus Groups + set_fact: + isolcpus_groups: "{{ isolcpus.split(',') }}" + + - debug: msg="isolcpus_groups = {{ isolcpus_groups }}" + + - name: Filter isolcpus Ranges + set_fact: + isolcpus_ranges: "{{ isolcpus_ranges + [item] }}" + with_items: "{{ isolcpus_groups }}" + when: ("-" in item) + + - debug: msg="isolcpus_ranges = {{ isolcpus_ranges }}" + + - name: Filter isolcpus Discretes + set_fact: + isolcpus_discretes: "{{ isolcpus_discretes + [item] }}" + with_items: "{{ isolcpus_groups }}" + when: ("-" not in item) + + - debug: msg="isolcpus_discretes = {{ isolcpus_discretes }}" + + - name: Build isolcpus List + set_fact: + isolcpus_list: "{{ isolcpus_list | default([]) | union(isolcpus_discretes) | union([item]) }}" + with_sequence: "{{ isolcpus_ranges }}" + + - debug: msg="isolcpus_list = {{ isolcpus_list }}" + + - name: Check isolcpus Total + assert: + that: "{{ isolcpus_list | length }} <= ansible_processor_vcpus" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The number of isolcpus {{ isolcpus_list | length }}, exceeds total CPUs on target {{ ansible_processor_vcpus }}. Please correct the configuration + when: isolcpus is defined + + - name: Check isolcpus IDs + assert: + that: "item | int <= ansible_processor_vcpus" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The CPU ID {{ item }} set for isolcpus is NOT actually present on target. Please correct the configuration + with_items: "{{ isolcpus_list }}" + when: isolcpus is defined + + - name: Check isolcpus OS Reserved + assert: + that: "item not in isolcpus_list" + msg: + - Incorrect configuration pertaining isolcpus. Conflicting or improper values detected + - The CPU ID 0...{{ ansible_processor_count - 1 }} should NOT be set for isolcpus. Please correct the configuration + with_items: "{{ lookup('sequence','0-{{ ansible_processor_count - 1 }}').split(',') }}" + when: isolcpus is defined + +#TODO relationship between cmk shared/exclusive cores and isolcpus + + when: + - cmk_enabled is defined + - (not cmk_hosts_list is defined) or (inventory_hostname in cmk_hosts_list) #CMK expected true for all profiles except basic + # {% if not cmk_use_all_hosts %} + - "'kube-node' in group_names" + # {% endif %} + + +#################################### +# Prerequisites for Worker Node(s) # +#################################### +- hosts: kube-node + vars: + phy_nics_pciids: [] + + tasks: + +# STORY: "nic bus info specified is present on system" + - debug: + msg: "Dataplane (DP) interface(s) defined in host_vars = {{ dataplane_interfaces }}" + when: dataplane_interfaces is defined + + - name: Check DP Interfaces + assert: + that: "dataplane_interfaces != []" + msg: "Dataplane (DP) interface(s) on target '{{ ansible_hostname }}' must be set in host_vars. Please correct the configuration" + when: + - dataplane_interfaces is defined + - (update_nic_drivers is defined and update_nic_drivers) or + (install_ddp_packages is defined and install_ddp_packages) or + (sriov_cni_enabled is defined and sriov_cni_enabled) or + (sriov_network_operator_enabled is defined and sriov_network_operator_enabled) + + - debug: + msg: "Network interfaces present on target '{{ ansible_hostname }}' = {{ ansible_interfaces }}" + + - name: Read Physical NICs PCIIDs + set_fact: + phy_nics_pciids: "{{ phy_nics_pciids + [ ansible_facts[item]['pciid'] ] }}" + with_items: "{{ ansible_interfaces }}" + when: ansible_facts[item]['pciid'] is defined and ansible_facts[item]['type'] == "ether" + + - debug: msg="PCI Slots for the NICs on target '{{ ansible_hostname }}' = {{ phy_nics_pciids }}" + + - name: Check DP Interfaces Names + assert: + that: ("{{ item.name }}" in {{ ansible_interfaces }}) + msg: "Dataplane interface '{{ item.name }}' defined in host_vars does NOT exist on target. Please correct the configuration" + with_items: "{{ dataplane_interfaces }}" + when: dataplane_interfaces is defined and dataplane_interfaces != [] + ignore_errors: True + + - name: Check DP Interfaces Bus Info + assert: + that: ("{{ item.bus_info }}" in "{{ phy_nics_pciids }}") + msg: "Dataplane interface '{{ item.name }}' defined with PCI ID '{{ item.bus_info }}' does NOT exist on target. Please correct the configuration" + with_items: "{{ dataplane_interfaces }}" + when: dataplane_interfaces is defined and dataplane_interfaces != [] + ignore_errors: True + + +# QAT Devices list is okay to be left empty (default), but if was defined, device(s) must exist on target + - debug: + msg: "QAT device(s) defined in host_vars = {{ qat_devices }}" + when: qat_devices is defined + + - name: Read QAT PCIIDs + shell: lshw -businfo -numeric | grep -i quickassist + register: lshw_qat + ignore_errors: True + when: qat_devices is defined + + - debug: + msg: "QAT devices found on target = {{ lshw_qat.stdout }}" + when: qat_devices is defined + + - name: Check QAT Devices' Bus Info + assert: + that: ("{{ item.qat_id }}" in """{{ lshw_qat.stdout }}""") + msg: "QAT device '{{ item.qat_dev }}' defined with PCI ID '{{ item.qat_id }}' does NOT exist on target. Please correct the configuration" + with_items: "{{ qat_devices }}" + when: qat_devices is defined and qat_devices != [] + ignore_errors: True + +# STORY: "qat_sriov_numvfs should not exceed max supported (16) per each dev_ID" + - debug: + msg: + - qat_sriov_numvfs for {{ item.qat_id }} = {{ item.qat_sriov_numvfs }} (host_vars) + - update_qat_drivers = {{ update_qat_drivers }} (host_vars) + with_items: "{{ qat_devices }}" + when: qat_devices is defined and qat_devices != [] # update_qat_drivers expected as 'true' for all profiles except basic + + - name: Check QAT SRIOV VFs + assert: + that: ({{ item.qat_sriov_numvfs }} <= 16) + msg: + - Incorrect configuration pertaining QAT SRIOV. Conflicting or improper values detected + - When SRIOV VFs are set for QAT, max value is 16 for each ID (max 48 total per card). Please correct the configuration + with_items: "{{ qat_devices }}" + when: + - update_qat_drivers is defined and update_qat_drivers + - qat_devices is defined and qat_devices != [] +# OpenSSL & OpenSSL*Engine must only be configured / installed when update_qat_drivers is set to 'true' and qat_devices is defined in host vars + - name: check OpenSSL and OpenSSL*Engine requirements + assert: + that: + - update_qat_drivers + - qat_devices is defined and qat_devices != [] + fail_msg: "OpenSSL & OpenSSL*Engine will only configured if update_qat_drivers is set to 'true' & qat_devices is defined in host vars" + success_msg: "OpenSSL & OpenSSL*Engine verification completed" + when: openssl_install is defined and openssl_install + + - name: check KMRA requirements + assert: + that: + - sgx_dp_enabled + fail_msg: "KMRA installation requires sgx_dp_enabled set to 'true'" + success_msg: "KMRA requirements verified" + when: kmra_enabled is defined and kmra_enabled + + - name: check SGX configuration + assert: + that: + - sgx_enabled + fail_msg: "SGX drivers installation requires sgx_enabled set to 'true'" + success_msg: "SGX configuration verified" + when: + - sgx_dp_enabled is defined and sgx_dp_enabled + - (ansible_distribution == 'Ubuntu' and ansible_distribution_version != '21.04') + or (ansible_os_family == 'RedHat' and ansible_distribution_version != '8.4') + + - name: check NFD configuration + assert: + that: + - nfd_enabled + fail_msg: "SGX DP requires nfd_enabled set to 'true'" + success_msg: "NFD configuration verified" + when: sgx_dp_enabled is defined and sgx_dp_enabled + + - name: check kmra_pccs_api_key presence + assert: + that: + - kmra_pccs_api_key is defined + fail_msg: + - "kmra_pccs_api_key is not defined" + success_msg: "kmra_pccs_api_key presence is verified" + when: + - kmra_enabled is defined and kmra_enabled + + - name: check PCCS API key length + assert: + that: + - kmra_pccs_api_key | length == 32 + fail_msg: "PCCS API Key should be 32 bytes long" + success_msg: "PCCS API key length verified" + when: + - kmra_enabled is defined and kmra_enabled + + - name: check PCCS API key is not a placeholder + assert: + that: + - kmra_pccs_api_key is defined + - kmra_pccs_api_key != "ffffffffffffffffffffffffffffffff" + fail_msg: + - "Please, visit https://api.portal.trustedservices.intel.com/provisioning-certification and click on 'Subscribe'" + - "to generate PCCS API key." + - "PCCS API key is essential for KMRA deployment and usage." + success_msg: "PCCS API key verified" + when: + - kmra_enabled is defined and kmra_enabled + +# STORY: "vpp/ovsdpdk require hugepage enabled and configured" + - debug: + msg: + - vpp_enabled = {{ vpp_enabled }} (host_vars) + - example_net_attach_defs = {{ example_net_attach_defs }} (group_vars/all.yml) + - userspace_ovs_dpdk = {{ example_net_attach_defs['userspace_ovs_dpdk'] }} (group_vars/all.yml) + - userspace_vpp = {{ example_net_attach_defs['userspace_vpp'] }} (group_vars/all.yml) + - sriov_net_dp = {{ example_net_attach_defs['sriov_net_dp'] }} (group_vars/all.yml) + - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars) + - sriov_cni_enabled = {{ sriov_cni_enabled }} (host_vars) + - sriov_network_operator_enabled = {{ sriov_network_operator_enabled }} (host_vars) + - bond_cni_enabled = {{ bond_cni_enabled }} (host_vars) + - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars) + - userspace_cni_enabled = {{ userspace_cni_enabled }} (host_vars) + - hugepages_enabled = {{ hugepages_enabled }} (host_vars) + - default_hugepage_size = {{ default_hugepage_size }} (host_vars) + - number_of_hugepages = {{ number_of_hugepages }} (host_vars) + when: vpp_enabled is defined #host_vars + + - name: Check OVS DPDK Dependencies + assert: + that: >- + ({{ ovs_dpdk_enabled }} and not {{ vpp_enabled }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + or {{ vpp_enabled }} + msg: + - Incorrect configuration pertaining OVS DPDK. Conflicting or improper values detected + - When OVS DPDK is enabled, VPP must be disabled and Hugepages must be set to 1G according to host_vars example. Please correct the configuration + when: ovs_dpdk_enabled is defined and ovs_dpdk_enabled + + - name: Check VPP Dependencies + assert: + that: >- + ({{ vpp_enabled }} and not {{ ovs_dpdk_enabled }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + or {{ ovs_dpdk_enabled }} + msg: + - Incorrect configuration pertaining VPP. Conflicting or improper values detected + - When VPP is enabled, OVS DPDK must be disabled and Hugepages must be set to 2M according to host_vars example. Please correct the configuration + when: vpp_enabled is defined and vpp_enabled + + +# STORY: "cnis require net-attach-defs to be enabled" + - name: Check CNI Config + assert: + that: >- + ({{ userspace_cni_enabled }} and {{ ovs_dpdk_enabled }} and {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and not {{ vpp_enabled }} and + not {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "1G" and {{ number_of_hugepages }} >= 0) + or ({{ userspace_cni_enabled }} and not {{ ovs_dpdk_enabled }} and not {{ example_net_attach_defs['userspace_ovs_dpdk'] }} and {{ vpp_enabled }} + and {{ example_net_attach_defs['userspace_vpp'] }} and {{ hugepages_enabled }} and + "{{ default_hugepage_size }}" == "2M" and {{ number_of_hugepages }} >= 0) + msg: + - Incorrect configuration pertaining CNI. Conflicting or improper values detected. + - When CNI is enabled, either OVS DPDK either VPP must be enabled and Hugepages must be according to example files. Please correct the configuration + when: userspace_cni_enabled is defined and userspace_cni_enabled + + +# STORY: "If SST enabled, confirm minimum kernel or kernel_update specified" + - name: Check SST # see Jira NPF-1545 + assert: + that: (not sst_bf_configuration_enabled) + msg: "SST-BF is NOT supported on {{ ansible_distribution }} {{ ansible_distribution_version }}. Please use a different OS or disable this feature" + when: + - sst_bf_configuration_enabled is defined + - (ansible_distribution == "RedHat" and ansible_distribution_version == '8.2') or ansible_distribution_version in ['7.6', '7.8', '7.9', '18.04'] + ignore_errors: True + +# STORY: Intel VT-d should be enabled in BIOS + - name: Check Intel VT-d + shell: dmesg | grep DMAR | grep remapping + register: dmesg_dmar_remap + ignore_errors: True + changed_when: False + + - debug: msg="dmesg >> {{ dmesg_dmar_remap.stdout }}" + + - name: Warn about Intel VT-d + fail: + msg: "Warning: Intel VT-d appears DISABLED on target. Please check BIOS under 'Advanced > Integrated IO Configuration' and Enable if necessary" + when: dmesg_dmar_remap.stdout|length == 0 + ignore_errors: True + + +# STORY: CPU Hyper-Threading should be enabled in BIOS + - name: Warn about Hyper-Threading + fail: + msg: "Warning: CPU Hyper-Threading is DISABLED on target. Please check BIOS under 'Advanced > Processor Configuration' and Enable if necessary" + when: ansible_processor_threads_per_core != 2 + ignore_errors: True + + +# STORY: "check for collectd. See Jira NPF-1687" + - name: Warn about collectd + fail: + msg: "Warning: On {{ ansible_distribution }} {{ ansible_distribution_version }} collectd won't work unless 'update_kernel' is enabled in group_vars" + when: ansible_distribution_version in ['7.6', '18.04'] + ignore_errors: True + + +# STORY: TEMPORARY: "ovs dpdk version requirements" + - debug: + msg: + - install_dpdk = {{ install_dpdk }} (host_vars) + - dpdk_version = {{ dpdk_version }} (host_vars) + - ovs_dpdk_enabled = {{ ovs_dpdk_enabled }} (host_vars) + - ovs_version = {{ ovs_version }} (host_vars) + when: + - install_dpdk is defined #host_vars + - dpdk_version is defined #host_vars + - ovs_version is defined #host_vars + - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars + + - name: Check OVS DPDK compatibility + assert: + that: >- + "{{ ovs_version }} == \"v2.15.0\" and {{ dpdk_version }} >= \"20.11\"" + or "{{ ovs_version }} == \"v2.14.2\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.14.1\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.14.0\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.3\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.2\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.1\" and {{ dpdk_version }} == \"19.11.6\"" + or "{{ ovs_version }} == \"v2.13.0\" and {{ dpdk_version }} == \"19.11.6\"" + msg: "OVS {{ ovs_version }} does not build with DPDK version {{ dpdk_version }}. Please correct the host_vars configuration" + when: + - dpdk_version is defined #host_vars + - ovs_version is defined #host_vars + - ovs_dpdk_enabled is defined and ovs_dpdk_enabled #host_vars + + + - meta: end_play + +# - name: Print all variables/facts known for a host +# ansible.builtin.debug: +# var: hostvars[inventory_hostname] +# verbosity: 4 diff --git a/sw_config/bmra/patched_rhel_packages.yml b/sw_config/bmra/patched_rhel_packages.yml index 40b22b6..687142a 100644 --- a/sw_config/bmra/patched_rhel_packages.yml +++ b/sw_config/bmra/patched_rhel_packages.yml @@ -10,7 +10,7 @@ - ansible_distribution == "CentOS" - ansible_distribution_version >= '8' and ansible_distribution_version < '8.3' -- name: enable PowerTools repository on CentOS >= 8.3 +- name: enable powertools repository on CentOS >= 8.3 # noqa 303 - yum is called intenionallly here command: yum config-manager --set-enabled powertools when: @@ -88,7 +88,10 @@ package: name: "{{ item }}" state: present + retries: 5 + delay: 10 register: source_status + until: source_status is not failed with_items: - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-193.el8.x86_64.rpm" - "https://vault.centos.org/8.2.2004/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-193.el8.x86_64.rpm" @@ -103,22 +106,35 @@ state: present register: source_status with_items: - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" - - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" -# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" -# - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" + - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-240.el8.x86_64.rpm" + - "https://vault.centos.org/8.3.2011/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-240.el8.x86_64.rpm" when: - ansible_distribution == "CentOS" - ansible_distribution_version == '8.3' - not update_kernel +- name: pull matching kernel headers on CentOS 8.4 + package: + name: "{{ item }}" + state: present + register: source_status + with_items: + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.3.1.el8.x86_64.rpm" + - "http://mirror.centos.org/centos/8/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.3.1.el8.x86_64.rpm" +# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-headers-4.18.0-305.el8.x86_64.rpm" +# - "https://vault.centos.org/8.4.2105/BaseOS/x86_64/os/Packages/kernel-devel-4.18.0-305.el8.x86_64.rpm" + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version == '8.4' + - not update_kernel + # pull the matching kernel headers if kernel is not updated - name: pull matching kernel headers from configured repos # noqa 503 - more than one condition, can't be a handler package: name: - - kernel-headers-{{ ansible_kernel }} - - kernel-devel-{{ ansible_kernel }} + - kernel-headers-{{ ansible_kernel }} + - kernel-devel-{{ ansible_kernel }} register: kernel_source retries: 3 until: kernel_source is success @@ -196,7 +212,7 @@ - ansible_distribution in ["RedHat", "CentOS"] - ansible_distribution_version < '8' -- name: Set python is python3 +- name: set python is python3 alternatives: name: python path: /usr/bin/python3 @@ -214,6 +230,51 @@ state: present when: ansible_distribution in ["RedHat", "CentOS"] +- name: Add kubic yum repo and install updated version of podman + block: + - name: disable container-tools module + # noqa 305 - shell is used intentionally here + shell: dnf -y module disable container-tools + - name: enable rhcontainerbot/container-selinux repository + # noqa 305 - shell is used intentionally here + shell: dnf -y copr enable rhcontainerbot/container-selinux + - name: Add kubic yum repo + yum_repository: + name: devel_kubic_libcontainers_stable + description: Stable Releases of Upstream github.com/containers packages (CentOS_$releasever) + baseurl: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/ + gpgcheck: yes + gpgkey: https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_$releasever/repodata/repomd.xml.key + keepcache: '0' + - name: install runc + dnf: + name: runc + state: present + - name: install podman package + package: + name: podman + state: present + when: + - ansible_os_family == "RedHat" and ansible_distribution_version >= '8.2' + - '"docker" not in container_runtime' + +# SELINUX will be disabled later stage so, these packages are required when container_runtime is docker +- name: install packages in RHEL >= 8.4 when container_runtime is docker + dnf: + name: + - bridge-utils + - lsof + - lvm2 + - tcpdump + - iproute-tc + - openssh-server + - chrony + - iputils + when: + - ansible_distribution == "RedHat" and ansible_distribution_version >= '8.4' + - container_runtime == "docker" + +# Workaround - Set pip to a version that supports correct version of packages needed - name: use the correct pip version for CentOS 7 pip: name: @@ -221,3 +282,11 @@ when: - ansible_distribution == "CentOS" - ansible_distribution_version < '7.9' + +- name: Install pip3 required by dpdk + yum: + name: python3-pip + state: latest + when: + - ansible_distribution == "CentOS" + - ansible_distribution_version < '7.9' diff --git a/sw_config/bmra/patched_sriov_cni_install.yml b/sw_config/bmra/patched_sriov_cni_install.yml new file mode 100644 index 0000000..ab0dc52 --- /dev/null +++ b/sw_config/bmra/patched_sriov_cni_install.yml @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: 2021 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +--- +- name: install dependencies + include_role: + name: install_dependencies + +- name: clone sriov-cni repository + git: + repo: "{{ sriov_cni_url }}" + dest: "{{ sriov_cni_dir }}" + version: "{{ sriov_cni_version }}" + force: yes + +# Ignore errors as this has been failing +- name: build sriov-cni plugin + make: + chdir: /usr/src/sriov-cni + ignore_errors: true + +# missing go.sum entry for module providing package golang.org/x/sys/unix +- name: Force checkout as some files change after build error + shell: + cmd: git checkout -- . + chdir: /usr/src/sriov-cni + +- name: build sriov-cni plugin + make: + chdir: /usr/src/sriov-cni + +- name: create /opt/cni/bin + file: + path: "/opt/cni/bin" + state: directory + recurse: yes + mode: 0755 + +- name: install sriov-cni binary to /opt/cni/bin directory + copy: + src: /usr/src/sriov-cni/build/sriov + dest: /opt/cni/bin/sriov + mode: 0755 + remote_src: yes |