[lib] Create veths using systemd opnfv-fuel units

Create 2 systemd services on the jumphost that will handle veth
pairs creation, respectively adding them to virsh/real bridges.
This allows us to set docker containers restart policy to 'always',
enabling persistent Salt Master/MaaS containers across jumphost
reboots.

NOTE: libvirt creates virtual networks async, hence the need for
retrying hooking veths to them.

JIRA: FUEL-406

Change-Id: I1ca033cb5eb854b577b57bb2387a58bd9605a5bb
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

2 files changed, 43 insertions, 9 deletions

diff --git a/mcp/scripts/docker-compose/docker-compose.yaml.j2 b/mcp/scripts/docker-compose/docker-compose.yaml.j2
index bc8b3e80a..dbac901ca 100644
--- a/mcp/scripts/docker-compose/docker-compose.yaml.j2
+++ b/mcp/scripts/docker-compose/docker-compose.yaml.j2
@@ -14,6 +14,7 @@ services:
   opnfv-fuel-salt-master:
     container_name: "fuel"
     image: "opnfv/fuel:saltmaster-reclass-{{ conf.MCP_DOCKER_TAG }}"
+    restart: always
     networks:
       mcpcontrol:
         ipv4_address: {{ conf.SALT_MASTER }}
@@ -40,6 +41,7 @@ services:
   opnfv-fuel-maas:
     container_name: "maas"
     image: "opnfv/fuel:saltminion-maas-{{ conf.MCP_DOCKER_TAG }}"
+    restart: always
     networks:
       mcpcontrol:
         ipv4_address: {{ conf.MAAS_IP }}
@@ -83,7 +85,7 @@ networks:
     driver: macvlan
     driver_opts:
 {%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) %}
-      parent: {{ ma.interface_str('veth_mcp5', nm.vlan_mgmt) }}
+      parent: {{ ma.interface_str('veth_mcp3', nm.vlan_mgmt) }}
 {%- else %}
       parent: veth_mcp3  # Untagged by default
 {%- endif %}
diff --git a/mcp/scripts/lib_jump_deploy.sh b/mcp/scripts/lib_jump_deploy.sh
index bce54ade2..5efbfa667 100644
--- a/mcp/scripts/lib_jump_deploy.sh
+++ b/mcp/scripts/lib_jump_deploy.sh
@@ -292,14 +292,46 @@ function create_networks {
       ${VIRSH} net-start "${net}"
     fi
   done
-  # create veth pairs for relevant networks (pxebr, mgmt)
-  for i in $(seq 0 2 2); do
-    sudo ip link del "veth_mcp$i" || true
-    sudo ip link add "veth_mcp$i" type veth peer name "veth_mcp$((i+1))"
-    sudo ip link set "veth_mcp$i" up mtu 9000
-    sudo ip link set "veth_mcp$((i+1))" up mtu 9000
-    sudo brctl addif "${all_vnode_networks[$((i/2))]}" "veth_mcp$i"
-  done
+
+  sudo ip link del veth_mcp0 || true
+  sudo ip link del veth_mcp2 || true
+  # Create systemd service for veth creation after reboot
+  FUEL_VETHC_SERVICE="/etc/systemd/system/opnfv-fuel-vethc.service"
+  FUEL_VETHA_SERVICE="/etc/systemd/system/opnfv-fuel-vetha.service"
+  test -f /usr/sbin/ip && PREFIX=/usr/sbin || PREFIX=/sbin
+  cat <<-EOF | sudo tee "${FUEL_VETHC_SERVICE}"
+	[Unit]
+	After=libvirtd.service
+	Before=docker.service
+	[Service]
+	ExecStart=/bin/sh -ec '\
+	  ${PREFIX}/ip link add veth_mcp0 type veth peer name veth_mcp1 && \
+	  ${PREFIX}/ip link add veth_mcp2 type veth peer name veth_mcp3 && \
+	  ${PREFIX}/ip link set veth_mcp0 up mtu 9000 && \
+	  ${PREFIX}/ip link set veth_mcp1 up mtu 9000 && \
+	  ${PREFIX}/ip link set veth_mcp2 up mtu 9000 && \
+	  ${PREFIX}/ip link set veth_mcp3 up mtu 9000'
+	EOF
+  cat <<-EOF | sudo tee "${FUEL_VETHA_SERVICE}"
+	[Unit]
+	StartLimitInterval=200
+	StartLimitBurst=10
+	After=opnfv-fuel-vethc.service
+	[Service]
+	Restart=on-failure
+	RestartSec=10
+	ExecStartPre=/bin/sh -ec '\
+	  ${PREFIX}/brctl showstp ${all_vnode_networks[0]} > /dev/null 2>&1 && \
+	  ${PREFIX}/brctl showstp ${all_vnode_networks[1]} > /dev/null 2>&1'
+	ExecStart=/bin/sh -ec '\
+	  ${PREFIX}/brctl addif ${all_vnode_networks[0]} veth_mcp0 && \
+	  ${PREFIX}/brctl addif ${all_vnode_networks[1]} veth_mcp2'
+	EOF
+  sudo ln -sf "${FUEL_VETHC_SERVICE}" "/etc/systemd/system/multi-user.target.wants/"
+  sudo ln -sf "${FUEL_VETHA_SERVICE}" "/etc/systemd/system/multi-user.target.wants/"
+  sudo systemctl daemon-reload
+  sudo systemctl restart opnfv-fuel-vethc
+  sudo systemctl restart opnfv-fuel-vetha
 }
 
 function create_vms {