docker build, deploy: Switch tooling to python3

Python2.7 is deprecated and packages are starting to enforce py3
usage (e.g. dockermake recently started supporting only 3.6).

Switch pipenv to python3, but allow pyhton3.5 by pinning dockermake
to v0.8 since Ubuntu Xenial does not have python3.6 easily available.

While at it, switch deploy tooling (PDF/IDF configuration parsing)
from python2 to pyhton3 too and fix some jumphost package requirements.

Change-Id: Id66d08d0f51a1bc35c1d78c1956df832a5536bde
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

5 files changed, 39 insertions, 17 deletions

diff --git a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
index 34db789a2..75da21633 100644
--- a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
+++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch
@@ -18,11 +18,15 @@ Subject: [PATCH] OPNFV package installation, Ubuntu user
   have to switch to 'root' login;
 * Preinstall `salt_minion_dependency_packages` and
   `salt_minion_reclass_dependencies`;
+* Pin dockermake to v0.8 to allow using python3.5 for virtualenv,
+  since python 3.6 is not easily available for Ubuntu Xenial
+  jump/build hosts.
 
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
  DockerMake.yml | 38 +++++++++++++++++++++++++++++++++++++-
- 1 file changed, 37 insertions(+), 1 deletion(-)
+ Pipfile        |  4 ++--
+ 2 files changed, 39 insertions(+), 3 deletions(-)
 
 diff --git a/DockerMake.yml b/DockerMake.yml
 index 2c75586..9ab7195 100644
@@ -87,3 +91,18 @@ index 2c75586..9ab7195 100644
  wheel:
    requires:
      - base
+diff --git a/Pipfile b/Pipfile
+index d3e8d66..340e125 100644
+--- a/Pipfile
++++ b/Pipfile
+@@ -7,8 +7,8 @@ name = "pypi"
+
+ [packages]
+ pygithub = "*"
+-docker-make = {git = "https://github.com/avirshup/DockerMake"}
+-dockermake = {git = "https://github.com/avirshup/DockerMake"}
++docker-make = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"}
++dockermake = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"}
+
+ [requires]
+ python_version = "3.6"
diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh
index bec8e159e..822d14116 100644
--- a/mcp/scripts/lib_template.sh
+++ b/mcp/scripts/lib_template.sh
@@ -51,14 +51,14 @@ function do_templates_scenario {
     awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
   j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;)
   # shellcheck disable=SC2086
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
     notify_e '[ERROR] Could not convert j2 scenario definitions!'
   fi
   for _yaml in "${extra_yaml[@]}"; do
     awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}"
   done
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")" \
     -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
     notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
@@ -84,7 +84,7 @@ function do_templates_cluster {
            "$(readlink -f docker-compose)" $(readlink -f ./*j2) \
            -name '*.j2' -exec echo -j {} \;)
   # shellcheck disable=SC2086
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+  if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
     -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
     notify_e '[ERROR] Could not convert PDF to network definitions!'
   fi
diff --git a/mcp/scripts/requirements_deb.yaml b/mcp/scripts/requirements_deb.yaml
index 73374b6ef..58fc533b0 100644
--- a/mcp/scripts/requirements_deb.yaml
+++ b/mcp/scripts/requirements_deb.yaml
@@ -9,7 +9,10 @@
 build:
   # Common pkgs required for all builds, no matter the type, arch etc.
   common:
-    - python-pip
+    - curl
+    - git
+    - make
+    - python3-pip
 deploy:
   # Common pkgs required for all deploys, no matter the type, arch etc.
   common:
@@ -30,11 +33,10 @@ deploy:
     - uuid-runtime
     - virtinst
     # python is indirectly required for PDF parsing
-    - python
-    - python-ipaddress
-    - python-jinja2
-    - python-yaml
-    - python-jsonschema
+    - python3
+    - python3-jinja2
+    - python3-yaml
+    - python3-jsonschema
   # Optional, arch-specific requirements, matched by key name = $(uname -m)
   aarch64:
     # AArch64 VMs use AAVMF (guest UEFI)
diff --git a/mcp/scripts/requirements_rpm.yaml b/mcp/scripts/requirements_rpm.yaml
index ef9bfad9c..d21e49a9d 100644
--- a/mcp/scripts/requirements_rpm.yaml
+++ b/mcp/scripts/requirements_rpm.yaml
@@ -9,7 +9,9 @@
 build:
   # Common pkgs required for all builds, no matter the type, arch etc.
   common:
-    - python-pip
+    - git
+    - make
+    - python3-pip
 deploy:
   # Common pkgs required for all deploys, no matter the type, arch etc.
   common:
@@ -35,11 +37,10 @@ deploy:
     - virt-install
     - wget
     # For python is indirectly required for PDF parsing
-    - python
-    - python-ipaddress
-    - python-jinja2
-    - python-yaml
-    - python-jsonschema
+    - python3
+    - python3-jinja2
+    - python3-yaml
+    - python3-jsonschema
   # Optional, arch-specific requirements, matched by key name = $(uname -m)
   aarch64:
     # AArch64 VMs use AAVMF (guest UEFI)
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 79bc16c81..6aca36f32 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -39,7 +39,7 @@
         {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
         {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%}
         {%- if 'numa' in V[n] and cpu_topo -%}
-          {%- for k, v in V[n].numa.iteritems() -%}
+          {%- for k, v in V[n].numa.items() -%}
             {%- set c = pack([k, v.memory, v.cpus]) -%}
             {%- do V[n].update({'s_numa': c if 's_numa' not in V[n] else pack([c, V[n].s_numa])}) -%}
           {%- endfor -%}