[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

3 files changed, 41 insertions, 138 deletions

diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index f5affb0d3..8f7cccd26 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -454,10 +454,7 @@ function create_vms {
 
 function update_mcpcontrol_network {
   # set static ip address for salt master node, MaaS node
-  local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
   local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
-  virsh net-update "mcpcontrol" add ip-dhcp-host \
-    "<host mac='${cmac}' name='cfg01' ip='${SALT_MASTER}'/>" --live --config
   [ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \
     "<host mac='${amac}' name='mas01' ip='${MAAS_IP}'/>" --live --config
 }
@@ -489,6 +486,29 @@ function start_vms {
   done
 }
 
+function prepare_containers {
+  local image_dir=$1
+  [ -n "${image_dir}" ] || exit 1
+  [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1
+
+  "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml down
+  sudo rm -rf "${image_dir}/salt" "${image_dir}/nodes/"*
+  mkdir -p "${image_dir}/salt/"{master.d,minion.d}
+  # salt state does not properly configure file_roots in master.conf, hard set it
+  sed -e 's/user: salt/user: root\nfile_recv: True/' -e 's/auto_accept:/open_mode:/' \
+      "${MCP_REPO_ROOT_PATH}/docker/files/salt/master.conf" > \
+      "${image_dir}/salt/master.d/opnfv.conf"
+  echo 'master: localhost' > "${image_dir}/salt/minion.d/opnfv.conf"
+  cp "${MCP_REPO_ROOT_PATH}/mcp/scripts/docker-compose/files/hosts" \
+      "${image_dir}/hosts"
+}
+
+function start_containers {
+  local image_dir=$1
+  [ -n "${image_dir}" ] || exit 1
+  "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml up --quiet-pull -d
+}
+
 function check_connection {
   local total_attempts=60
   local sleep_time=5
@@ -584,12 +604,27 @@ function get_nova_compute_pillar_data {
 }
 
 function docker_install {
+  local image_dir=$1
   # Mininum effort attempt at installing Docker if missing
-  if ! which docker; then
+  if ! docker --version; then
     curl -fsSL https://get.docker.com -o get-docker.sh
     sudo sh get-docker.sh
     rm get-docker.sh
     # On RHEL distros, the Docker service should be explicitly started
     sudo systemctl start docker
+  else
+    DOCKER_VER=$(docker version --format '{{.Server.Version}}')
+    if [ "${DOCKER_VER%%.*}" -lt 2 ]; then
+      notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it."
+    fi
+  fi
+  # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7)
+  COMPOSE_BIN="${image_dir}/docker-compose"
+  COMPOSE_VERSION='1.22.0'
+  notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3
+  if [ ! -e "${COMPOSE_BIN}" ]; then
+    COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}"
+    sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}"
+    sudo chmod +x "${COMPOSE_BIN}"
   fi
 }
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
deleted file mode 100755
index 46ef6ffbf..000000000
--- a/mcp/scripts/salt.sh
+++ /dev/null
@@ -1,132 +0,0 @@
-#!/bin/bash -e
-##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-#
-# Deploy Salt Master
-#
-
-CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
-F_GIT_ROOT=$(git rev-parse --show-toplevel)
-F_GIT_DIR=$(cd "${F_GIT_ROOT}/mcp" && git rev-parse --git-dir)
-F_GIT_SUBD=${F_GIT_ROOT#${F_GIT_DIR%%/.git*}}
-OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv"
-OPNFV_GIT_DIR="/root/opnfv"
-OPNFV_FUEL_DIR="/root/fuel" # Should be in sync with patch.sh, scripts patches
-OPNFV_RDIR="reclass/classes/cluster/all-mcp-arch-common"
-OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img"
-OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images"
-LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}"
-LOCAL_PDF_RECLASS=$1; shift
-# shellcheck disable=SC2116,SC2086
-LOCAL_VIRT_NODES=$(echo ${*//cfg01/}) # unquoted to filter space
-NODE_MASK="${LOCAL_VIRT_NODES// /|}"
-
-# push to cfg01 current git repo first (including submodules), at ~ubuntu/opnfv
-# later we move it to ~root/opnfv (and ln as ~root/fuel); delete the temp clone
-remote_tmp="${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")"
-STORAGE_DIR=$(dirname "${LOCAL_PDF_RECLASS}")
-REL_STORAGE_DIR_PATH=${STORAGE_DIR#${LOCAL_GIT_DIR}}
-if [[ "${REL_STORAGE_DIR_PATH}" == "${STORAGE_DIR}" ]]
-then
-  REL_STORAGE_DIR_PATH=""
-fi
-rsync -Erl --delete -e "ssh ${SSH_OPTS}" \
-  --exclude={.gitignore,"$REL_STORAGE_DIR_PATH"} \
-  "${LOCAL_GIT_DIR}/" "${remote_tmp}/"
-if [ -n "${LOCAL_PDF_RECLASS}" ] && [ -f "${LOCAL_PDF_RECLASS}" ]; then
-  rsync -e "ssh ${SSH_OPTS}" "${LOCAL_PDF_RECLASS}" \
-    "${remote_tmp}${F_GIT_SUBD}/mcp/${OPNFV_RDIR}/opnfv/"
-fi
-local_vcp_img=$(dirname "${LOCAL_PDF_RECLASS}")/$(basename "${OPNFV_VCP_IMG}")
-if [ -e "${local_vcp_img}" ]; then
-  rsync -L -e "ssh ${SSH_OPTS}" "${local_vcp_img}" \
-    "${remote_tmp}${F_GIT_SUBD}/${OPNFV_VCP_IMG}"
-fi
-
-# ssh to cfg01
-# shellcheck disable=SC2086,2087
-ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END
-  sudo -i
-  set -e
-  export TERM=${TERM}
-  export CI_DEBUG=${CI_DEBUG}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
-
-  echo -n 'Checking out cloud-init has finished running ...'
-  while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done
-  echo ' done'
-
-  mkdir -p /srv/salt /usr/share/salt-formulas/reclass
-  rm -rf ${OPNFV_GIT_DIR} ${OPNFV_FUEL_DIR}
-  mv ${OPNFV_TMP_DIR} ${OPNFV_GIT_DIR} && chown -R root.root ${OPNFV_GIT_DIR}
-  find ${OPNFV_GIT_DIR} -name '.git' -type f | while read f_git; do
-    sed -i 's@${LOCAL_GIT_DIR}@${OPNFV_GIT_DIR}@g' \$f_git
-  done
-  ln -sf ${OPNFV_GIT_DIR}${F_GIT_SUBD} ${OPNFV_FUEL_DIR}
-  ln -sf ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt
-  ln -sf ${OPNFV_FUEL_DIR}/mcp/deploy/scripts /srv/salt
-  ln -sf ${OPNFV_FUEL_DIR}/mcp/scripts/mcp.rsa $(dirname "${OPNFV_FUEL_DIR}")
-
-  cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass
-  cd /srv/salt/reclass/classes/service && \
-    ln -sf /usr/share/salt-formulas/reclass/service/opendaylight
-
-  # Armband APT-MK nightly/extra repo for forked & extended reclass
-  wget -qO - https://linux.enea.com/apt-mk/public.gpg | apt-key add -
-  echo 'deb http://linux.enea.com/apt-mk/xenial nightly extra' > \
-    '/etc/apt/sources.list.d/armband_mcp_extra.list'
-  apt-get update
-
-  cd /srv/salt/scripts
-  export DEBIAN_FRONTEND=noninteractive
-  echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/90silence-dpkg
-  OLD_DOMAIN=\$(grep -sPzo "id: cfg01\.\K(\S*)" /etc/salt/minion.d/minion.conf) || true
-  BOOTSTRAP_SALTSTACK_OPTS=" -r -dX stable 2017.7 " \
-    MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly \
-      EXTRA_FORMULAS="nfs panko gnocchi oslo-templates" \
-        ./salt-master-init.sh
-  salt-key -Ay
-
-  cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env
-  cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas
-  cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass
-
-  source ${OPNFV_FUEL_DIR}/mcp/scripts/lib.sh
-  wait_for 3.0 "salt-call state.apply salt"
-
-  # In case scenario changed (and implicitly domain name), re-register minions
-  if [ -n "\${OLD_DOMAIN}" ] && [ "\${OLD_DOMAIN}" != "${CLUSTER_DOMAIN}" ]; then
-    salt "*.\${OLD_DOMAIN}" cmd.run "grep \${OLD_DOMAIN} -sRl /etc/salt | \
-      xargs --no-run-if-empty sed -i 's/\${OLD_DOMAIN}/${CLUSTER_DOMAIN}/g'; \
-        service salt-minion restart" || true
-    salt-key -yd "*.\${OLD_DOMAIN}"
-    salt-key -Ay
-  fi
-
-  # Init specific to VMs on FN (all for virtual, cfg|mas for baremetal)
-  wait_for 3.0 'salt -C "cfg01*" state.apply linux'
-  if [[ "${LOCAL_VIRT_NODES}" =~ mas ]]; then
-    wait_for 3.0 'salt -C "mas*" test.ping'
-  else
-    wait_for 3.0 '(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)'
-  fi
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" saltutil.sync_all'
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt'
-
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage'
-  wait_for 2.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.network'
-  salt -C "E@^(${NODE_MASK}).*" system.reboot
-  wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping'
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False dist_upgrade=True'
-
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp'
-
-  if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then
-    mkdir -p "${OPNFV_VCP_DIR}"
-    mv "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" "${OPNFV_VCP_DIR}/"
-  fi
-SALT_INSTALL_END
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index c0629fec4..b8c32135c 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -33,7 +33,7 @@
 {%- macro serialize_vnodes() -%}
   {%- set arr = [] -%}
   {%- for section in section_map -%}
-    {%- for n in V.nodes[section] -%}
+    {%- for n in V.nodes[section] or [] -%}
       {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%}
         {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
         {%- do arr.append(pack([n, V[n].ram or arch.default.ram,
@@ -48,7 +48,7 @@
 {%- macro filter_nodes(type, output_as_query = False, sections = section_map) -%}
   {%- set arr = [] -%}
   {%- for section in sections -%}
-    {%- for n in V.nodes[section] -%}
+    {%- for n in V.nodes[section] or [] -%}
       {%- if ( (section_map[section] < 0 and type == 'virtual') or
                (section_map[section] >= 0 and
                 conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%}