Added support for Docker Hub credentials (CI)

When credentials are available on the Ansible host (/opt/kuberef/docker_config)
they will be added to the K8s nodes.

Also includes a minor change to increase the resources in the Kuberef VM

Signed-off-by: Michael S. Pedersen <michaelx.pedersen@intel.com>
Change-Id: I9acb0df97d2b423449ee57ae3d8241b2d9567ba2
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/72768
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Rihab Banday <rihab.banday@ericsson.com>

3 files changed, 174 insertions, 0 deletions

diff --git a/sw_config/bmra/dockerhub_credentials/tasks/main.yml b/sw_config/bmra/dockerhub_credentials/tasks/main.yml
new file mode 100644
index 0000000..6531df8
--- /dev/null
+++ b/sw_config/bmra/dockerhub_credentials/tasks/main.yml
@@ -0,0 +1,8 @@
+---
+- name: Add Docker hub credentials on target nodes
+  docker_login:
+    username: "{{ dhub_user }}"
+    password: "{{ dhub_pass }}"
+    email: "{{ dhub_email }}"
+    registry_url: "{{ dhub_reg_url }}"
+  no_log: True
diff --git a/sw_config/bmra/dockerhub_credentials/vars/.gitignore b/sw_config/bmra/dockerhub_credentials/vars/.gitignore
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/sw_config/bmra/dockerhub_credentials/vars/.gitignore
diff --git a/sw_config/bmra/patched_k8s.yml b/sw_config/bmra/patched_k8s.yml
new file mode 100644
index 0000000..52239b0
--- /dev/null
+++ b/sw_config/bmra/patched_k8s.yml
@@ -0,0 +1,166 @@
+##
+##   Copyright (c) 2020-2021 Intel Corporation.
+##
+##   Licensed under the Apache License, Version 2.0 (the "License");
+##   you may not use this file except in compliance with the License.
+##   You may obtain a copy of the License at
+##
+##       http://www.apache.org/licenses/LICENSE-2.0
+##
+##   Unless required by applicable law or agreed to in writing, software
+##   distributed under the License is distributed on an "AS IS" BASIS,
+##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+##   See the License for the specific language governing permissions and
+##   limitations under the License.
+##
+---
+- hosts: 127.0.0.1
+  connection: local
+  tasks: []
+  roles:
+    - { role: kubespray_install }
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- hosts: k8s-cluster
+  tasks: []
+  roles:
+    - role: cluster_defaults
+    - role: kubespray_target_setup
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- hosts: all
+  gather_facts: false
+  tasks:
+    - name: prepare additional kubespray facts
+      set_fact:
+        kubelet_node_custom_flags_prepare: >-
+          {%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%}
+            --reserved-cpus={{ native_cpu_manager_reserved_cpus }}
+          {%- endif -%}
+        enable_admission_plugins_prepare: >-
+          [EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}]
+        kube_config_dir: /etc/kubernetes
+    - name: set kube_cert_dir
+      set_fact:
+        kube_cert_dir: "{{ kube_config_dir }}/ssl"
+        kube_csr_dir: "{{ kube_config_dir }}/csr"
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- name: run kubespray
+  import_playbook: kubespray/cluster.yml
+  vars:
+    kubeadm_enabled: true
+    multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf
+    docker_iptables_enabled: true
+    docker_dns_servers_strict: false
+    override_system_hostname: false
+    docker_version: '19.03'
+    kube_proxy_mode: iptables
+    enable_nodelocaldns: false
+    system_reserved: true
+    dashboard_enabled: true
+    system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}"
+    kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}"
+    kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}"
+    kube_api_anonymous_auth: true
+    kube_feature_gates:
+      - CPUManager=true # feature gate can be enabled by default, default policy is none in Kubernetes
+      - TopologyManager={{ topology_manager_enabled | default(true) }}
+      - RotateKubeletServerCertificate=true
+    # Kubernetes cluster hardening
+    kubernetes_audit: true
+    audit_log_maxbackups: 10
+    kube_controller_manager_bind_address: 127.0.0.1
+    kube_scheduler_bind_address: 127.0.0.1
+    kube_proxy_healthz_bind_address: 127.0.0.1
+    kube_proxy_metrics_bind_address: 127.0.0.1
+    kube_read_only_port: 0
+    kube_override_hostname: ""
+    kube_kubeadm_apiserver_extra_args:
+      service-account-lookup: true
+      service-account-key-file: "{{ kube_cert_dir }}/sa.key"
+      admission-control-config-file: "{{ kube_config_dir }}/admission-control/config.yaml"
+    kube_kubeadm_scheduler_extra_args:
+      address: 127.0.0.1
+      profiling: false
+    kube_kubeadm_controller_extra_args:
+      address: 127.0.0.1
+      service-account-private-key-file: "{{ kube_cert_dir }}/sa.key"
+    kubelet_config_extra_args:
+      protectKernelDefaults: true
+      cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}"
+      topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}"
+      eventRecordQPS: 0
+    kube_apiserver_request_timeout: 60s
+    kube_apiserver_enable_admission_plugins: "{{ enable_admission_plugins_prepare | from_yaml }}"
+    podsecuritypolicy_enabled: "{{ psp_enabled }}"
+    kube_encrypt_secret_data: true
+    apiserver_extra_volumes:
+      - name: admission-control-config
+        hostPath: /etc/kubernetes/admission-control/
+        mountPath: /etc/kubernetes/admission-control/
+        readOnly: true
+    preinstall_selinux_state: "{{ selinux_mode | default('disabled') }}"
+    tls_cipher_suites:
+      - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+      - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+      - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+    etcd_extra_vars:
+      ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
+
+- hosts: k8s-cluster
+  tasks:
+    - name: restart docker daemon to recreate iptables rules
+      systemd: name=docker state=restarted
+      become: yes
+    - name: restart kubelet to trigger static pods recreation
+      systemd: name=kubelet state=restarted
+      become: yes
+    # note: fix for the issue mentioned here:
+    # https://github.com/kubernetes-sigs/kubespray/blob/58f48500b1adac3f18466fa1c5cf8aa9d9838150/docs/flannel.md#flannel
+    - name: check if flannel.1 interface exists
+      stat:
+        path: /sys/class/net/flannel.1
+      when: kube_network_plugin == "flannel"
+      register: flannel_endpoint
+    - name: disable offloading features on flannel.1
+      command: ethtool --offload flannel.1 rx off tx off
+      become: yes
+      when:
+        - kube_network_plugin == "flannel"
+        - flannel_endpoint.stat.exists
+
+- hosts: etcd
+  tasks:
+    - name: change /var/lib/etcd owner
+      file:
+        path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
+        owner: etcd
+        group: etcd
+        recurse: true
+        state: directory
+        mode: 0700
+    - name: change /var/lib/etcd permissions
+      file:
+        path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
+        owner: etcd
+        group: etcd
+        mode: '0700'
+        state: directory
+
+- hosts: k8s-cluster
+  roles:
+    - role: cluster_defaults
+      tags: defaults
+    - role: docker_registry
+      tags: registry
+    - role: dockerhub_credentials
+      when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file"
+  environment: "{{ proxy_env | d({}) }}"
+  any_errors_fatal: true
+
+- name: run certificate generation for mTLS in kubelet
+  import_playbook: kubelet-certificates.yml