Merge "Add NOVCP HA OVS scenario (baremetal, virtual)"

27 files changed, 807 insertions, 11 deletions

diff --git a/mcp/config/scenario/baremetal/os-nosdn-nofeature-novcp-ha.yaml b/mcp/config/scenario/baremetal/os-nosdn-nofeature-novcp-ha.yaml
new file mode 100644
index 000000000..0c5744199
--- /dev/null
+++ b/mcp/config/scenario/baremetal/os-nosdn-nofeature-novcp-ha.yaml
@@ -0,0 +1,25 @@
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+cluster:
+  domain: mcp-pike-ovs-novcp-ha.local
+  states:
+    - maas
+    - baremetal_init
+    - openstack_ha
+    - networks
+virtual:
+  nodes:
+    - cfg01
+    - mas01
+  cfg01:
+    vcpus: 4
+    ram: 6144
+  mas01:
+    vcpus: 4
+    ram: 6144
diff --git a/mcp/config/scenario/virtual/os-nosdn-nofeature-novcp-ha.yaml b/mcp/config/scenario/virtual/os-nosdn-nofeature-novcp-ha.yaml
new file mode 100644
index 000000000..69d407b82
--- /dev/null
+++ b/mcp/config/scenario/virtual/os-nosdn-nofeature-novcp-ha.yaml
@@ -0,0 +1,31 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+cluster:
+  domain: mcp-pike-ovs-novcp-ha.local
+  states:
+    - openstack_ha
+    - networks
+virtual:
+  nodes:
+    - cfg01
+    - kvm01
+    - kvm02
+    - kvm03
+    - cmp001
+    - cmp002
+  # NOTE: We might need to add more RAM here
+  kvm01:
+    vcpus: 4
+    ram: 14336
+  kvm02:
+    vcpus: 4
+    ram: 14336
+  kvm03:
+    vcpus: 4
+    ram: 14336
diff --git a/mcp/patches/0011-routes-Skip-network-restart-on-noifupdown.patch b/mcp/patches/0011-routes-Skip-network-restart-on-noifupdown.patch
new file mode 100644
index 000000000..5ccd04aef
--- /dev/null
+++ b/mcp/patches/0011-routes-Skip-network-restart-on-noifupdown.patch
@@ -0,0 +1,29 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 8 Jan 2018 05:09:11 +0100
+Subject: [PATCH] routes: Skip network restart on 'noifupdown'
+
+Previously, setting up routes did not allow passing 'require_reboot',
+so each route change would lead to a networking service restart,
+rendering interface configuration options like 'noifupdown' useless.
+Allow disabling network restart per-interface using the existing
+'noifupdown' option.
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ linux/network/interface.sls | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/linux/network/interface.sls b/linux/network/interface.sls
+index 921ceac..6ebc670 100644
+--- a/linux/network/interface.sls
++++ b/linux/network/interface.sls
+@@ -338,6 +338,9 @@ linux_network_{{ interface_name }}_routes:
+       gateway: {{ route.gateway }}
+       {%- endif %}
+     {%- endfor %}
++  {%- if interface.noifupdown is defined %}
++  - require_reboot: {{ interface.noifupdown }}
++  {%- endif %}
+
+ {%- endif %}
+
diff --git a/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch b/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch
new file mode 100644
index 000000000..25159bc4a
--- /dev/null
+++ b/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch
@@ -0,0 +1,44 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 03 Dec 2017 22:03:01 +0200
+Subject: [PATCH] reclass.storage.node: Merge duplicate nodes
+
+Reclass does not support duplicate nodes in top pillar, so merge all
+nodes with the same name into a single node, inheriting classes from
+all instances.
+
+This allows using multiple "system.reclass.storage.system.*_cluster"
+classes for the same node, based on re-using the name (hostname).
+
+NOTE: defaults.merge module does not merge lists (e.g. for classes),
+so handle that case separately.
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+
+diff --git a/reclass/storage/node.sls b/reclass/storage/node.sls
+--- a/reclass/storage/node.sls
++++ b/reclass/storage/node.sls
+@@ -1,7 +1,15 @@
+ {%- from "reclass/map.jinja" import storage with context %}
+ {%- if storage.enabled %}
+
++{%- set storage_by_name = {} %}
+ {%- for node_name, node in storage.get('node', {}).iteritems() %}
++{%- set new_node_name = node.get('name', node_name) %}
++{%- set new_node = storage_by_name.get(new_node_name, {'classes': []}) %}
++{%- do salt['defaults.merge'](node, {'classes': new_node.classes + node.classes}) %}
++{%- do salt['defaults.merge'](storage_by_name, {new_node_name: node}) %}
++{%- endfor %}
++
++{%- for node_name, node in storage_by_name.iteritems() %}
+
+ {%- if node.repeat is defined %}
+
diff --git a/mcp/patches/patch.sh b/mcp/patches/patch.sh
index 1da3bc597..bb48dcd07 100755
--- a/mcp/patches/patch.sh
+++ b/mcp/patches/patch.sh
@@ -11,10 +11,9 @@ CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
 
 if [ -r "$1" ]; then
   while IFS=': ' read -r p_dest p_file; do
-    if [[ ! "${p_dest}" =~ '^#' ]] && [[ "${p_dest}" =~ $2 ]] && \
-      ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \
-        "/root/fuel/mcp/patches/${p_file}" > /dev/null; then
-          patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}"
+    if ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \
+      "/root/fuel/mcp/patches/${p_file}" > /dev/null; then
+        patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}"
     fi
-  done < "$1"
+  done < <(grep -vE '^#' "${1}" | grep -E "^.*${2}.*: ")
 fi
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 2eb45b272..703bd0587 100644
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -16,5 +16,6 @@
 /usr/share/salt-formulas/env: 0009-controller-Use-keystoneclient-to-check-project-ID.patch
 /usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch
 /usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch
+/usr/share/salt-formulas/env: 0011-routes-Skip-network-restart-on-noifupdown.patch
 /usr/share/salt-formulas/env: 0012-linux.storage.lvm-Disable-filter.patch
 /usr/share/salt-formulas/env: 0013-dpdk-Support-ovs-bridge-tagging.patch
diff --git a/mcp/patches/patches_init.list b/mcp/patches/patches_init.list
new file mode 100644
index 000000000..94488cf9a
--- /dev/null
+++ b/mcp/patches/patches_init.list
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+/usr/share/salt-formulas/env: 0014-reclass.storage.node-Merge-duplicate-nodes.patch
diff --git a/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch b/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch
new file mode 100644
index 000000000..978815020
--- /dev/null
+++ b/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch
@@ -0,0 +1,35 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Fri, 8 Dec 2017 20:30:46 +0100
+Subject: [PATCH] salt-master-init.sh: Apply OPNFV Fuel patches
+
+Some of Fuel@OPNFV patches need to be applied before the reclass
+storage.node state is ran for Salt Master, i.e. between installing
+salt-formula-* packages and configuring the Salt Master salt services.
+
+JIRA: FUEL-310
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ salt-master-init.sh | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/salt-master-init.sh b/salt-master-init.sh
+index 343324c..a4ec138 100755
+--- a/salt-master-init.sh
++++ b/salt-master-init.sh
+@@ -284,6 +284,7 @@ options
+     system_config
+
+     saltmaster_bootstrap &&\
++    /root/fuel/mcp/patches/patch.sh /root/fuel/mcp/patches/patches_init.list formulas &&\
+     saltmaster_init &&\
+
+     verify_salt_minions
diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
index 0d86a086e..120b6fb7e 100644
--- a/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
+++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/infra/kvm_pdf.yml.j2
@@ -35,6 +35,7 @@ parameters:
           name: {{ vlans[vlan] }}.{{ vlan }}
           use_interfaces:
             - {{ vlans[vlan] }}
+          noifupdown: true
     {%- endif %}
 {%- endfor %}
 
@@ -53,6 +54,7 @@ parameters:
           netmask: 255.255.255.0
           use_interfaces:
             - {{ nm.ctl01.nic_mgmt }}{% if nm.vlan_mgmt and nm.vlan_mgmt != 'native' %}.{{ nm.vlan_mgmt }}{% endif %}
+          noifupdown: true
         br-ex:
           enabled: true
           proto: manual
diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_control_novcp.yml b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_control_novcp.yml
new file mode 100644
index 000000000..dcf0463a3
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_control_novcp.yml
@@ -0,0 +1,144 @@
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - system.linux.system.repo.glusterfs
+  - system.ceilometer.client
+  - system.memcached.server.single
+  - system.keystone.server.cluster
+  - system.keystone.server.wsgi
+  - system.glance.control.cluster
+  - system.nova.control.cluster
+  - system.cinder.control.cluster
+  - system.cinder.control.backend.lvm
+  - system.heat.server.cluster
+  - system.designate.server.cluster
+  - system.designate.server.backend.bind
+  - system.bind.server.single
+  - system.haproxy.proxy.listen.openstack.nova-placement
+  - system.haproxy.proxy.listen.openstack.glare
+  - system.glusterfs.client.cluster
+  - system.glusterfs.client.volume.glance
+  - system.glusterfs.client.volume.keystone
+  # sync from kvm
+  - service.keepalived.cluster.single
+  - system.glusterfs.server.volume.glance
+  - system.glusterfs.server.volume.keystone
+  - system.glusterfs.server.cluster
+  # NOTE(armband): Disabled for novcp
+  # - system.salt.control.virt
+  # - system.salt.control.cluster.openstack_control_cluster
+  # - system.salt.control.cluster.openstack_proxy_cluster
+  # - system.salt.control.cluster.openstack_database_cluster
+  # - system.salt.control.cluster.openstack_message_queue_cluster
+  # - system.salt.control.cluster.openstack_telemetry_cluster
+  # - system.salt.control.cluster.stacklight_server_cluster
+  # - system.salt.control.cluster.stacklight_log_cluster
+  # - system.salt.control.cluster.stacklight_telemetry_cluster
+  - cluster.mcp-pike-common-ha.infra.kvm_pdf
+  - cluster.mcp-pike-common-ha.include.proxy
+parameters:
+  _param:
+    linux_system_codename: xenial  # sync from kvm
+    # For NOVCP, we switch keepalived VIPs, to keep cluster_vip_address in ctl
+    single_nic: br-ctl  # for keepalive_vip_interface interpolation
+    keepalived_vip_interface: ${_param:single_nic}  # sync from kvm
+    keepalived_vip_virtual_router_id: 50
+    keepalived_openstack_web_public_vip_address: ${_param:openstack_proxy_address}
+    keepalived_openstack_web_public_vip_interface: br-ex
+    cluster_vip_address: ${_param:openstack_control_address}
+    cluster_local_address: ${_param:single_address}
+    cluster_node01_hostname: ${_param:openstack_control_node01_hostname}
+    cluster_node01_address: ${_param:openstack_control_node01_address}
+    cluster_node02_hostname: ${_param:openstack_control_node02_hostname}
+    cluster_node02_address: ${_param:openstack_control_node02_address}
+    cluster_node03_hostname: ${_param:openstack_control_node03_hostname}
+    cluster_node03_address: ${_param:openstack_control_node03_address}
+    nova_vncproxy_url: https://${_param:cluster_public_host}:6080
+    glusterfs_version: '3.13'
+  libvirt:
+    server:
+      service: libvirtd
+      config_sys: /etc/default/libvirtd
+      unix_sock_group: libvirt
+  linux:
+    network:
+      remove_iface_files:
+        - '/etc/network/interfaces.d/50-cloud-init.cfg'
+      # Add public IPs here as overrides, no need to fork another kvm_pdf.j2
+      interface:
+        br-ex:
+          address: ${_param:external_address}
+          proto: static
+  neutron:
+    server:
+      vlan_aware_vms: true
+  keystone:
+    server:
+      cacert: /etc/ssl/certs/mcp_os_cacert
+  bind:
+    server:
+      control:
+        mgmt:
+          enabled: true
+          bind:
+            address: ${_param:single_address}
+            port: 953
+          allow:
+            - ${_param:openstack_control_node01_address}
+            - ${_param:openstack_control_node02_address}
+            - ${_param:openstack_control_node03_address}
+          keys:
+            - designate
+  designate:
+    server:
+      pools:
+        default:
+          description: 'test pool'
+          targets:
+            default:
+              description: 'test target1'
+            default1:
+              type: ${_param:designate_pool_target_type}
+              description: 'test target2'
+              masters: ${_param:designate_pool_target_masters}
+              options:
+                host: ${_param:openstack_control_node02_address}
+                port: 53
+                rndc_host: ${_param:openstack_control_node02_address}
+                rndc_port: 953
+                rndc_key_file: /etc/designate/rndc.key
+            default2:
+              type: ${_param:designate_pool_target_type}
+              description: 'test target3'
+              masters: ${_param:designate_pool_target_masters}
+              options:
+                host: ${_param:openstack_control_node03_address}
+                port: 53
+                rndc_host: ${_param:openstack_control_node03_address}
+                rndc_port: 953
+                rndc_key_file: /etc/designate/rndc.key
+  # sync from common-ha kvm role
+  glusterfs:
+    server:
+      service: glusterd
+      volumes:
+        nova_instances:
+          storage: /srv/glusterfs/nova_instances
+          replica: 3
+          bricks:
+            - ${_param:cluster_node01_address}:/srv/glusterfs/nova_instances
+            - ${_param:cluster_node02_address}:/srv/glusterfs/nova_instances
+            - ${_param:cluster_node03_address}:/srv/glusterfs/nova_instances
+          options:
+            cluster.readdir-optimize: 'True'
+            nfs.disable: 'True'
+            network.remote-dio: 'True'
+            cluster.favorite-child-policy: mtime
+            diagnostics.client-log-level: WARNING
+            diagnostics.brick-log-level: WARNING
diff --git a/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_init_novcp.yml b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_init_novcp.yml
new file mode 100644
index 000000000..1aab6663e
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-common-ha/openstack_init_novcp.yml
@@ -0,0 +1,234 @@
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+parameters:
+  _param:
+
+    openstack_version: pike
+
+    # openstack service addresses
+    openstack_proxy_control_address: ${_param:opnfv_openstack_control_address}
+    openstack_proxy_node01_control_address: ${_param:opnfv_openstack_control_node01_address}
+    openstack_proxy_node02_control_address: ${_param:opnfv_openstack_control_node03_address}
+
+    openstack_proxy_address: ${_param:opnfv_openstack_proxy_address}
+    openstack_proxy_node01_address: ${_param:opnfv_openstack_proxy_node01_address}
+    openstack_proxy_node02_address: ${_param:opnfv_openstack_proxy_node02_address}
+
+    openstack_control_address: ${_param:opnfv_openstack_control_address}
+    openstack_control_node01_address: ${_param:opnfv_openstack_control_node01_address}
+    openstack_control_node02_address: ${_param:opnfv_openstack_control_node02_address}
+    openstack_control_node03_address: ${_param:opnfv_openstack_control_node03_address}
+
+    openstack_database_address: ${_param:openstack_control_address}
+    openstack_database_node01_address: ${_param:openstack_control_node01_address}
+    openstack_database_node02_address: ${_param:openstack_control_node02_address}
+    openstack_database_node03_address: ${_param:openstack_control_node03_address}
+
+    openstack_message_queue_address: ${_param:openstack_control_address}
+    openstack_message_queue_node01_address: ${_param:openstack_control_node01_address}
+    openstack_message_queue_node02_address: ${_param:openstack_control_node02_address}
+    openstack_message_queue_node03_address: ${_param:openstack_control_node03_address}
+
+    openstack_telemetry_address: ${_param:openstack_control_address}
+    openstack_telemetry_node01_address: ${_param:openstack_control_node01_address}
+    openstack_telemetry_node02_address: ${_param:openstack_control_node02_address}
+    openstack_telemetry_node03_address: ${_param:openstack_control_node03_address}
+
+    # OpenStack Compute
+    openstack_compute_node01_single_address: ${_param:opnfv_openstack_compute_node01_single_address}
+    openstack_compute_node02_single_address: ${_param:opnfv_openstack_compute_node02_single_address}
+    openstack_compute_node03_single_address: ${_param:opnfv_openstack_compute_node03_single_address}
+    openstack_compute_node01_control_address: ${_param:opnfv_openstack_compute_node01_control_address}
+    openstack_compute_node02_control_address: ${_param:opnfv_openstack_compute_node02_control_address}
+    openstack_compute_node03_control_address: ${_param:opnfv_openstack_compute_node03_control_address}
+    openstack_compute_node01_tenant_address: ${_param:opnfv_openstack_compute_node01_tenant_address}
+    openstack_compute_node02_tenant_address: ${_param:opnfv_openstack_compute_node02_tenant_address}
+    openstack_compute_node03_tenant_address: ${_param:opnfv_openstack_compute_node03_tenant_address}
+    openstack_compute_node01_external_address: ${_param:opnfv_openstack_compute_node01_external_address}
+    openstack_compute_node02_external_address: ${_param:opnfv_openstack_compute_node02_external_address}
+
+    # openstack service hostnames
+    openstack_proxy_hostname: ${_param:openstack_control_hostname}
+    openstack_proxy_node01_hostname: ${_param:openstack_control_node01_hostname}
+    openstack_proxy_node02_hostname: ${_param:openstack_control_node03_hostname}
+    openstack_control_hostname: kvm
+    openstack_control_node01_hostname: kvm01
+    openstack_control_node02_hostname: kvm02
+    openstack_control_node03_hostname: kvm03
+    openstack_database_hostname: ${_param:openstack_control_hostname}
+    openstack_database_node01_hostname: ${_param:openstack_control_node01_hostname}
+    openstack_database_node02_hostname: ${_param:openstack_control_node02_hostname}
+    openstack_database_node03_hostname: ${_param:openstack_control_node03_hostname}
+    openstack_message_queue_hostname: ${_param:openstack_control_hostname}
+    openstack_message_queue_node01_hostname: ${_param:openstack_control_node01_hostname}
+    openstack_message_queue_node02_hostname: ${_param:openstack_control_node02_hostname}
+    openstack_message_queue_node03_hostname: ${_param:openstack_control_node03_hostname}
+    openstack_telemetry_hostname: ${_param:openstack_control_hostname}
+    openstack_telemetry_node01_hostname: ${_param:openstack_control_node01_hostname}
+    openstack_telemetry_node02_hostname: ${_param:openstack_control_node02_hostname}
+    openstack_telemetry_node03_hostname: ${_param:openstack_control_node03_hostname}
+
+    # openstack compute
+    openstack_compute_node01_hostname: cmp001
+    openstack_compute_node02_hostname: cmp002
+
+    openstack_region: RegionOne
+    admin_email: root@localhost
+    # Neutron osv/nodvr
+    neutron_control_dvr: 'False'
+    neutron_l3_ha: 'True'
+    neutron_global_physnet_mtu: 1500
+    neutron_external_mtu: 1500
+    neutron_gateway_dvr: 'False'
+    neutron_gateway_agent_mode: legacy
+    neutron_compute_dvr: 'False'
+    neutron_compute_agent_mode: legacy
+    neutron_compute_external_access: 'True'
+    galera_server_cluster_name: openstack_cluster
+    galera_server_maintenance_password: opnfv_secret
+    galera_server_admin_password: opnfv_secret
+    rabbitmq_secret_key: opnfv_secret
+    rabbitmq_admin_password: opnfv_secret
+    rabbitmq_openstack_password: opnfv_secret
+    glance_version: ${_param:openstack_version}
+    glance_service_host: ${_param:openstack_control_address}
+    keystone_version: ${_param:openstack_version}
+    keystone_service_host: ${_param:openstack_control_address}
+    heat_version: ${_param:openstack_version}
+    heat_service_host: ${_param:openstack_control_address}
+    heat_domain_admin_password: opnfv_secret
+    cinder_version: ${_param:openstack_version}
+    cinder_service_host: ${_param:openstack_control_address}
+    ceilometer_version: ${_param:openstack_version}
+    ceilometer_service_host: ${_param:openstack_telemetry_address}
+    ceilometer_influxdb_password: opnfv_secret
+    nova_version: ${_param:openstack_version}
+    nova_service_host: ${_param:openstack_control_address}
+    neutron_version: ${_param:openstack_version}
+    neutron_service_host: ${_param:openstack_control_address}
+    glusterfs_service_host: ${_param:openstack_control_address}
+    mysql_admin_user: root
+    mysql_admin_password: opnfv_secret
+    mysql_cinder_password: opnfv_secret
+    mysql_ceilometer_password: opnfv_secret
+    mysql_glance_password: opnfv_secret
+    mysql_grafana_password: opnfv_secret
+    mysql_heat_password: opnfv_secret
+    mysql_keystone_password: opnfv_secret
+    mysql_neutron_password: opnfv_secret
+    mysql_nova_password: opnfv_secret
+    mysql_aodh_password: opnfv_secret
+    mysql_designate_password: opnfv_secret
+    aodh_version: ${_param:openstack_version}
+    keystone_aodh_password: opnfv_secret
+    keystone_service_token: opnfv_secret
+    keystone_admin_password: opnfv_secret
+    keystone_ceilometer_password: opnfv_secret
+    keystone_cinder_password: opnfv_secret
+    keystone_glance_password: opnfv_secret
+    keystone_heat_password: opnfv_secret
+    keystone_keystone_password: opnfv_secret
+    keystone_neutron_password: opnfv_secret
+    keystone_nova_password: opnfv_secret
+    keystone_designate_password: opnfv_secret
+    ceilometer_secret_key: opnfv_secret
+    horizon_version: ${_param:openstack_version}
+    horizon_secret_key: opaesee8Que2yahJoh9fo0eefo1Aeyo6ahyei8zeiboh3aeth5loth7ieNa5xi5e
+    horizon_identity_host: ${_param:openstack_control_address}
+    horizon_identity_encryption: none
+    horizon_identity_version: 3
+    mongodb_server_replica_set: ceilometer
+    mongodb_ceilometer_password: opnfv_secret
+    mongodb_admin_password: opnfv_secret
+    mongodb_shared_key: eoTh1AwahlahqueingeejooLughah4tei9feing0eeVaephooDi2li1TaeV1ooth
+    metadata_password: opnfv_secret
+    openstack_telemetry_keepalived_password: opnfv_secret
+    aodh_service_host: ${_param:openstack_telemetry_address}
+    designate_service_host: ${_param:openstack_control_address}
+    designate_bind9_rndc_key: 4pc+X4PDqb2q+5o72dISm72LM1Ds9X2EYZjqg+nmsS7FhdTwzFFY8l/iEDmHxnyjkA33EQC8H+z0fLLBunoitw==
+    designate_domain_id: 5186883b-91fb-4891-bd49-e6769234a8fc
+    designate_pool_ns_records:
+      - hostname: 'ns1.example.org.'
+        priority: 10
+    designate_pool_nameservers:
+      - host: ${_param:openstack_control_node01_address}
+        port: 53
+      - host: ${_param:openstack_control_node02_address}
+        port: 53
+      - host: ${_param:openstack_control_node03_address}
+        port: 53
+    designate_pool_target_type: bind9
+    designate_pool_target_masters:
+      - host: ${_param:openstack_control_node01_address}
+        port: 5354
+      - host: ${_param:openstack_control_node02_address}
+        port: 5354
+      - host: ${_param:openstack_control_node03_address}
+        port: 5354
+    designate_pool_target_options:
+      host: ${_param:openstack_control_node01_address}
+      port: 53
+      rndc_host: ${_param:openstack_control_node01_address}
+      rndc_port: 953
+      rndc_key_file: /etc/designate/rndc.key
+    designate_version: ${_param:openstack_version}
+    # Billing
+    # keystone_billometer_password: opnfv_secret
+    # keystone_billometer_address: ${_param:billometer_service_host}
+    # billometer_service_host: ${_param:openstack_billing_address}
+    # billometer_version: ${_param:openstack_version}
+    # billometer_secret_key: opnfv_secretpasswordpasswordpassword
+    # billometer_identity_password: ${_param:keystone_billometer_password}
+    # billometer_identity_host: ${_param:openstack_control_address}
+    # billometer_identity_token: ${_param:keystone_service_token}
+  linux:
+    system:
+      repo:
+        uca:
+          source: "deb http://ubuntu-cloud.archive.canonical.com/ubuntu xenial-updates/${_param:openstack_version} main"
+          architectures: amd64
+          key_id: EC4926EA
+          key_server: keyserver.ubuntu.com
+      kernel:
+        sysctl:
+          net.ipv4.tcp_congestion_control: yeah
+          net.ipv4.tcp_slow_start_after_idle: 0
+          net.ipv4.tcp_fin_timeout: 30
+    network:
+      host:
+        kvm:
+          address: ${_param:openstack_control_address}
+          names:
+            - ${_param:openstack_control_hostname}
+            - ${_param:openstack_control_hostname}.${_param:cluster_domain}
+        kvm01:
+          address: ${_param:openstack_control_node01_address}
+          names:
+            - ${_param:openstack_control_node01_hostname}
+            - ${_param:openstack_control_node01_hostname}.${_param:cluster_domain}
+        kvm02:
+          address: ${_param:openstack_control_node02_address}
+          names:
+            - ${_param:openstack_control_node02_hostname}
+            - ${_param:openstack_control_node02_hostname}.${_param:cluster_domain}
+        kvm03:
+          address: ${_param:openstack_control_node03_address}
+          names:
+            - ${_param:openstack_control_node03_hostname}
+            - ${_param:openstack_control_node03_hostname}.${_param:cluster_domain}
+        cmp001:
+          address: ${_param:openstack_compute_node01_control_address}
+          names:
+            - ${_param:openstack_compute_node01_hostname}
+            - ${_param:openstack_compute_node01_hostname}.${_param:cluster_domain}
+        cmp002:
+          address: ${_param:openstack_compute_node02_control_address}
+          names:
+            - ${_param:openstack_compute_node02_hostname}
+            - ${_param:openstack_compute_node02_hostname}.${_param:cluster_domain}
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/config.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/config.yml
new file mode 100644
index 000000000..5681e5a6b
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/config.yml
@@ -0,0 +1,24 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.infra.config
+  - cluster.mcp-pike-ovs-novcp-ha.infra
+parameters:
+  reclass:
+    storage:
+      node:
+        openstack_control_node01:  # openstack_proxy_node01
+          params:
+            external_address: ${_param:openstack_proxy_node01_address}
+        openstack_control_node02:  # no proxy role
+          params:
+            external_address: 0.0.0.0
+        openstack_control_node03:  # openstack_proxy_node02
+          params:
+            external_address: ${_param:openstack_proxy_node02_address}
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/init.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/init.yml
new file mode 100644
index 000000000..23c299cbc
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/init.yml
@@ -0,0 +1,18 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.infra
+  - cluster.mcp-pike-ovs-novcp-ha.openstack
+parameters:
+  _param:
+    cluster_name: mcp-pike-ovs-novcp-ha
+    # For NOVCP, we override kvm addresses to overlap with ctl
+    infra_kvm_node01_address: ${_param:openstack_control_node01_address}
+    infra_kvm_node02_address: ${_param:openstack_control_node02_address}
+    infra_kvm_node03_address: ${_param:openstack_control_node03_address}
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/kvm.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/kvm.yml
new file mode 100644
index 000000000..887af029d
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/kvm.yml
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+# NOTE(armband): we don't want to pull in salt.control for novcp
+# classes:
+#   - cluster.mcp-pike-common-ha.infra.kvm
+#   - cluster.mcp-pike-ovs-novcp-ha.infra
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/maas.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/maas.yml
new file mode 100644
index 000000000..e694679b9
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/infra/maas.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.infra.maas
+  - cluster.mcp-pike-ovs-novcp-ha.infra
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/init.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/init.yml
new file mode 100644
index 000000000..234418d9c
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/init.yml
@@ -0,0 +1,12 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.all-mcp-arch-common
+  - cluster.mcp-pike-ovs-novcp-ha.infra
+  - cluster.mcp-pike-ovs-novcp-ha.openstack
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute.yml
new file mode 100644
index 000000000..afe1cc5a6
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute.yml
@@ -0,0 +1,17 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_compute
+  - cluster.mcp-pike-ovs-novcp-ha.openstack.compute_pdf
+  - cluster.mcp-pike-ovs-novcp-ha.infra
+parameters:
+  nova:
+    compute:
+      libvirt_service: libvirtd
+      libvirt_bin: /etc/default/libvirtd
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute_pdf.yml.j2 b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute_pdf.yml.j2
new file mode 120000
index 000000000..d43451a2a
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/compute_pdf.yml.j2
@@ -0,0 +1 @@
+../../mcp-pike-ovs-ha/openstack/compute_pdf.yml.j2
\ No newline at end of file
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/control.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/control.yml
new file mode 100644
index 000000000..4cd5b9629
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/control.yml
@@ -0,0 +1,19 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - system.neutron.control.openvswitch.cluster
+  - cluster.mcp-pike-common-ha.openstack_control_novcp
+  - cluster.mcp-pike-ovs-novcp-ha
+parameters:
+  apache:
+    server:
+      modules:
+        # NOTE(armband): We first override mods to ~, then to this to drop ssl
+        - rewrite
+        - wsgi
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/database.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/database.yml
new file mode 100644
index 000000000..163769524
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/database.yml
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_database
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/init.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/init.yml
new file mode 100644
index 000000000..7540c0074
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/init.yml
@@ -0,0 +1,24 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_init_novcp
+parameters:
+  _param:
+    neutron_tenant_network_types: "flat,vxlan"
+    nova_cpu_pinning: "1,2,3,4,5,7,8,9,10,11"
+    compute_hugepages_size: 1G
+    compute_hugepages_count: 16
+    compute_hugepages_mount: /mnt/hugepages_1G
+    compute_kernel_isolcpu: ${_param:nova_cpu_pinning}
+  apache:
+    server:
+      # NOTE(armband): override these to ~ first, so we can later drop ssl/443
+      bind:
+        ports: ~
+      modules: ~
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/message_queue.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/message_queue.yml
new file mode 100644
index 000000000..0fd4a8a54
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/message_queue.yml
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_message_queue
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/proxy.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/proxy.yml
new file mode 100644
index 000000000..89b095430
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/proxy.yml
@@ -0,0 +1,59 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_proxy
+  - system.keepalived.cluster.instance.openstack_web_public_vip
+parameters:
+  linux:
+    # Set up routes similar to prx*ovs-ha
+    network:
+      interface:
+        br-ex:
+          route:
+            public:
+              address: 0.0.0.0
+              netmask: 0.0.0.0
+              gateway: ${_param:opnfv_net_public_gw}
+  nginx:
+    server:
+      # NOTE(armband): Define host.address for all proxies for uniformity
+      site:
+        nginx_proxy_novnc: &nginx_openstack_proxy_address
+          host:
+            address: ${_param:openstack_proxy_address}
+        nginx_proxy_openstack_api_aodh:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_ceilometer:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_cinder:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_glance:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_heat:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_heat_cfn:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_heat_cloudwatch:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_keystone:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_keystone_private:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_neutron:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_nova:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_api_nova_ec2:
+          <<: *nginx_openstack_proxy_address
+        nginx_proxy_openstack_web:
+          <<: *nginx_openstack_proxy_address
+        nginx_ssl_redirect_openstack_web:
+          <<: *nginx_openstack_proxy_address
+        nginx_static_reclass_doc:
+          <<: *nginx_openstack_proxy_address
diff --git a/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/telemetry.yml b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/telemetry.yml
new file mode 100644
index 000000000..1cb72ac5c
--- /dev/null
+++ b/mcp/reclass/classes/cluster/mcp-pike-ovs-novcp-ha/openstack/telemetry.yml
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-common-ha.openstack_telemetry
diff --git a/mcp/reclass/nodes/cfg01.mcp-pike-ovs-novcp-ha.local.yml b/mcp/reclass/nodes/cfg01.mcp-pike-ovs-novcp-ha.local.yml
new file mode 100644
index 000000000..eb1f794af
--- /dev/null
+++ b/mcp/reclass/nodes/cfg01.mcp-pike-ovs-novcp-ha.local.yml
@@ -0,0 +1,18 @@
+##############################################################################
+# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+classes:
+  - cluster.mcp-pike-ovs-novcp-ha.infra.config
+parameters:
+  _param:
+    linux_system_codename: xenial
+    reclass_data_revision: master
+  linux:
+    system:
+      name: cfg01
+      domain: mcp-pike-ovs-novcp-ha.local
diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index 5b00c7380..8e9ba97a9 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -449,12 +449,11 @@ function wait_for {
     for attempt in $(seq "${total_attempts}"); do
       echo "[wait_for] Attempt ${attempt}/${total_attempts%.*} for: ${cmdstr}"
       if [ "${total_attempts%.*}" = "${total_attempts}" ]; then
-        # shellcheck disable=SC2015
         eval "${cmdstr}" && echo "[wait_for] OK: ${cmdstr}" && return 0 || true
       else
-        ! (eval "${cmdstr}" || echo __fuel_wf_failure__) |& tee /dev/stderr | \
-          grep -Eq '(Not connected|No response|__fuel_wf_failure__)' && \
-          echo "[wait_for] OK: ${cmdstr}" && return 0 || true
+        ! (eval "${cmdstr}" || echo 'No response') |& tee /dev/stderr | \
+           grep -Eq '(Not connected|No response)' && \
+           echo "[wait_for] OK: ${cmdstr}" && return 0 || true
       fi
       sleep "${sleep_time}"
     done
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
index 15168c0cc..824cada7b 100755
--- a/mcp/scripts/salt.sh
+++ b/mcp/scripts/salt.sh
@@ -106,10 +106,10 @@ ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END
   wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt'
 
   wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage'
-  salt -C "E@^(${NODE_MASK}).*" state.sls linux.network -b 1 || true
+  wait_for 2.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.network' || true
   salt -C "E@^(${NODE_MASK}).*" system.reboot
   wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping'
-  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False'
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False dist_upgrade=True'
 
   wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp'