Enables containerized overcloud deployments

Changes Include:
  - For upstream deployments, Docker local registry will be updated with
    latest current RDO containers, regular deployments will use latest
    stable
  - Upstream container images will then be patched/modified and then
    re-uploaded into local docker registry with 'apex' tag
  - Deployment command modified to deploy with containers
  - Adds a --no-fetch deployment argument to disable pulling latest
    from upstream, and instead using what already exists in cache
  - Moves Undercloud NAT setup to just after undercloud is installed.
    This provides internet during overcloud install which is now
    required for upstream container deployments.
  - Creates loop device for Ceph deployment when no device is
    provided in deploy settings (for container deployment only)
  - Updates NIC J2 template to use the new format in OOO since
    the os-apply-config method is now deprecated in > Queens

JIRA: APEX-566
JIRA: APEX-549

Change-Id: I0652c194c059b915a942ac7401936e8f5c69d1fa
Signed-off-by: Tim Rozet <trozet@redhat.com>

3 files changed, 192 insertions, 181 deletions

diff --git a/build/nics-template.yaml.jinja2 b/build/nics-template.yaml.jinja2
index b1069d2c..16083294 100644
--- a/build/nics-template.yaml.jinja2
+++ b/build/nics-template.yaml.jinja2
@@ -1,4 +1,4 @@
-heat_template_version: 2015-04-30
+heat_template_version: 2016-10-14
 
 description: >
   Software Config to drive os-net-config to configure multiple interfaces
@@ -78,196 +78,200 @@ parameters:
 
 resources:
   OsNetConfigImpl:
-    type: OS::Heat::StructuredConfig
+    type: OS::Heat::SoftwareConfig
     properties:
-      group: os-apply-config
+      group: script
       config:
-        os_net_config:
-          network_config:
-            -
-            {%- if not nets['external'][0]['enabled'] or nets['tenant']['nic_mapping'][role]['vlan'] is number or nets['storage']['nic_mapping'][role]['vlan'] is number or nets['api']['nic_mapping'][role]['vlan'] is number or  nets['external'][0]['nic_mapping'][role]['vlan'] is number %}
-              type: ovs_bridge
-            {%- if nets['external'][0]['enabled'] and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
-              name: br-isolated
-            {%- else %}
-              name: {get_input: bridge_name}
-            {%- endif %}
-              members:
-                -
-                  type: interface
-                  name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
-                  # force the MAC address of the bridge to this interface
-                  primary: true
-                {%- if nets['external'][0]['enabled'] and nets['external'][0]['nic_mapping'][role]['vlan'] is number %}
-                -
-                  type: vlan
-                  vlan_id: {get_param: ExternalNetworkVlanID}
-                  addresses:
+        str_replace:
+          template:
+            get_file: /usr/share/openstack-tripleo-heat-templates/network/scripts/run-os-net-config.sh
+          params:
+            $network_config:
+              network_config:
+              -
+              {%- if not nets['external'][0]['enabled'] or nets['tenant']['nic_mapping'][role]['vlan'] is number or nets['storage']['nic_mapping'][role]['vlan'] is number or nets['api']['nic_mapping'][role]['vlan'] is number or  nets['external'][0]['nic_mapping'][role]['vlan'] is number %}
+                type: ovs_bridge
+              {%- if nets['external'][0]['enabled'] and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
+                name: br-isolated
+              {%- else %}
+                name: br-ex
+              {%- endif %}
+                members:
                   -
-                    ip_netmask: {get_param: ExternalIpSubnet}
-                  routes:
-                    -
-                      default: true
-                      next_hop: {get_param: ExternalInterfaceDefaultRoute}
-                {%- endif %}
-                {%- if nets['tenant']['enabled'] and nets['tenant']['nic_mapping'][role]['vlan'] is number %}
-                -
-                  type: vlan
-                  vlan_id: {get_param: TenantNetworkVlanID}
-                  addresses:
-                    -
-                      ip_netmask: {get_param: TenantIpSubnet}
-                {%- endif %}
-                {%- if nets['storage']['enabled'] and nets['storage']['nic_mapping'][role]['vlan'] is number %}
-                -
-                  type: vlan
-                  vlan_id: {get_param: StorageNetworkVlanID}
-                  addresses:
-                    -
-                      ip_netmask: {get_param: StorageIpSubnet}
-                {%- endif %}
-                {%- if nets['api']['enabled'] and nets['api']['nic_mapping'][role]['vlan'] is number %}
-                -
-                  type: vlan
-                  vlan_id: {get_param: InternalApiNetworkVlanID}
-                  addresses:
+                    type: interface
+                    name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
+                    # force the MAC address of the bridge to this interface
+                    primary: true
+                  {%- if nets['external'][0]['enabled'] and nets['external'][0]['nic_mapping'][role]['vlan'] is number %}
+                  -
+                    type: vlan
+                    vlan_id: {get_param: ExternalNetworkVlanID}
+                    addresses:
                     -
-                      ip_netmask: {get_param: InternalApiIpSubnet}
-                {%- endif %}
-            {%- else %}
-              type: {{ nets['admin']['nic_mapping'][role]['phys_type'] }}
-              {%- if nets['admin']['nic_mapping'][role]['phys_type'] == 'linux_bridge' %}
-              name: br-ctlplane
-              members:
-                -
-                  type: interface
-                  name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
-                  primary: true
+                      ip_netmask: {get_param: ExternalIpSubnet}
+                    routes:
+                      -
+                        default: true
+                        next_hop: {get_param: ExternalInterfaceDefaultRoute}
+                  {%- endif %}
+                  {%- if nets['tenant']['enabled'] and nets['tenant']['nic_mapping'][role]['vlan'] is number %}
+                  -
+                    type: vlan
+                    vlan_id: {get_param: TenantNetworkVlanID}
+                    addresses:
+                      -
+                        ip_netmask: {get_param: TenantIpSubnet}
+                  {%- endif %}
+                  {%- if nets['storage']['enabled'] and nets['storage']['nic_mapping'][role]['vlan'] is number %}
+                  -
+                    type: vlan
+                    vlan_id: {get_param: StorageNetworkVlanID}
+                    addresses:
+                      -
+                        ip_netmask: {get_param: StorageIpSubnet}
+                  {%- endif %}
+                  {%- if nets['api']['enabled'] and nets['api']['nic_mapping'][role]['vlan'] is number %}
+                  -
+                    type: vlan
+                    vlan_id: {get_param: InternalApiNetworkVlanID}
+                    addresses:
+                      -
+                        ip_netmask: {get_param: InternalApiIpSubnet}
+                  {%- endif %}
               {%- else %}
-              name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
-              {%- endif %}
-            {%- endif %}
-              use_dhcp: false
-              dns_servers: {get_param: DnsServers}
-              addresses:
-                -
-                  ip_netmask:
-                    list_join:
-                      - '/'
-                      - - {get_param: ControlPlaneIp}
-                        - {get_param: ControlPlaneSubnetCidr}
-              routes:
-                -
-                  ip_netmask: 169.254.169.254/32
-                  next_hop: {get_param: EC2MetadataIp}
-                {%- if external_net_af == 6 or role == 'compute' or not nets['external'][0]['enabled'] %}
-                -
-                  default: true
-                  next_hop: {get_param: ControlPlaneDefaultRoute}
+                type: {{ nets['admin']['nic_mapping'][role]['phys_type'] }}
+                {%- if nets['admin']['nic_mapping'][role]['phys_type'] == 'linux_bridge' %}
+                name: br-ctlplane
+                members:
+                  -
+                    type: interface
+                    name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
+                    primary: true
+                {%- else %}
+                name: {{ nets['admin']['nic_mapping'][role]['members'][0] }}
                 {%- endif %}
-
-            {%- if nets['tenant']['enabled'] and nets['tenant']['nic_mapping'][role]['vlan'] == 'native' %}
-            {%- if ovs_dpdk_bridge == 'br-phy' and role == 'compute' %}
-            -
-              type: ovs_user_bridge
-              name: {{ ovs_dpdk_bridge }}
-              use_dhcp: false
-              addresses:
-                -
-                  ip_netmask: {get_param: TenantIpSubnet}
-              members:
-                -
-                  type: ovs_dpdk_port
-                  name: dpdk0
-                  driver: {{ nets['tenant']['nic_mapping'][role]['uio_driver'] }}
-                  members:
-                    -
-                      type: interface
-                      name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
-                      # force the MAC address of the bridge to this interface
-                      primary: true
-            {%- else %}
-            -
-              type: {{ nets['tenant']['nic_mapping'][role]['phys_type'] }}
-              name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
-              {%- if 'uio-driver' in nets['tenant']['nic_mapping'][role] %}
-              uio_driver: {{ nets['tenant']['nic_mapping'][role]['uio-driver'] }}
-              {%- endif %}
-              {%- if 'interface-options' in nets['tenant']['nic_mapping'][role] %}
-              options: '{{ nets['tenant']['nic_mapping'][role]['interface-options'] }}'
-              {%- endif %}
-              use_dhcp: false
-              addresses:
-                -
-                  ip_netmask: {get_param: TenantIpSubnet}
-            {%- endif %}
-            {%- endif %}
-            {%- if nets['external'][0]['enabled'] and external_net_type != 'br-ex' and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
-            -
-              type: {{ nets['external'][0]['nic_mapping'][role]['phys_type'] }}
-              name: {{ nets['external'][0]['nic_mapping'][role]['members'][0] }}
-              {%- if 'uio-driver' in nets['external'][0]['nic_mapping'][role] %}
-              uio_driver: {{ nets['external'][0]['nic_mapping'][role]['uio-driver'] }}
-              {%- endif %}
-              {%- if role == 'controller' %}
-              dns_servers: {get_param: DnsServers}
               {%- endif %}
-              use_dhcp: false
-              addresses:
-                -
-                  ip_netmask: {get_param: ExternalIpSubnet}
-              routes:
-                -
-                  {%- if role == 'controller' %}
-                  default: true
+                use_dhcp: false
+                dns_servers: {get_param: DnsServers}
+                addresses:
+                  -
+                    ip_netmask:
+                      list_join:
+                        - '/'
+                        - - {get_param: ControlPlaneIp}
+                          - {get_param: ControlPlaneSubnetCidr}
+                routes:
+                  -
+                    ip_netmask: 169.254.169.254/32
+                    next_hop: {get_param: EC2MetadataIp}
+                  {%- if external_net_af == 6 or role == 'compute' or not nets['external'][0]['enabled'] %}
+                  -
+                    default: true
+                    next_hop: {get_param: ControlPlaneDefaultRoute}
                   {%- endif %}
-                  ip_netmask: 0.0.0.0/0
-                  next_hop: {get_param: ExternalInterfaceDefaultRoute}
-            {%- elif nets['external'][0]['enabled'] and external_net_type == 'br-ex' and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
-            -
+
+              {%- if nets['tenant']['enabled'] and nets['tenant']['nic_mapping'][role]['vlan'] == 'native' %}
               {%- if ovs_dpdk_bridge == 'br-phy' and role == 'compute' %}
-              type: ovs_user_bridge
+              -
+                type: ovs_user_bridge
+                name: {{ ovs_dpdk_bridge }}
+                use_dhcp: false
+                addresses:
+                  -
+                    ip_netmask: {get_param: TenantIpSubnet}
+                members:
+                  -
+                    type: ovs_dpdk_port
+                    name: dpdk0
+                    driver: {{ nets['tenant']['nic_mapping'][role]['uio_driver'] }}
+                    members:
+                      -
+                        type: interface
+                        name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
+                        # force the MAC address of the bridge to this interface
+                        primary: true
               {%- else %}
-              type: ovs_bridge
+              -
+                type: {{ nets['tenant']['nic_mapping'][role]['phys_type'] }}
+                name: {{ nets['tenant']['nic_mapping'][role]['members'][0] }}
+                {%- if 'uio-driver' in nets['tenant']['nic_mapping'][role] %}
+                uio_driver: {{ nets['tenant']['nic_mapping'][role]['uio-driver'] }}
+                {%- endif %}
+                {%- if 'interface-options' in nets['tenant']['nic_mapping'][role] %}
+                options: '{{ nets['tenant']['nic_mapping'][role]['interface-options'] }}'
+                {%- endif %}
+                use_dhcp: false
+                addresses:
+                  -
+                    ip_netmask: {get_param: TenantIpSubnet}
+              {%- endif %}
               {%- endif %}
-              name: {get_input: bridge_name}
-              use_dhcp: false
-              members:
-                -
-                  type: interface
-                  name: {{ nets['external'][0]['nic_mapping'][role]['members'][0] }}
-                  # force the MAC address of the bridge to this interface
-                  primary: true
-              {%- if role == 'controller' %}
-              dns_servers: {get_param: DnsServers}
-              addresses:
-                -
-                  ip_netmask: {get_param: ExternalIpSubnet}
-              routes:
-                -
-                  default: true
-                  ip_netmask: 0.0.0.0/0
-                  next_hop: {get_param: ExternalInterfaceDefaultRoute}
+              {%- if nets['external'][0]['enabled'] and external_net_type != 'br-ex' and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
+              -
+                type: {{ nets['external'][0]['nic_mapping'][role]['phys_type'] }}
+                name: {{ nets['external'][0]['nic_mapping'][role]['members'][0] }}
+                {%- if 'uio-driver' in nets['external'][0]['nic_mapping'][role] %}
+                uio_driver: {{ nets['external'][0]['nic_mapping'][role]['uio-driver'] }}
+                {%- endif %}
+                {%- if role == 'controller' %}
+                dns_servers: {get_param: DnsServers}
+                {%- endif %}
+                use_dhcp: false
+                addresses:
+                  -
+                    ip_netmask: {get_param: ExternalIpSubnet}
+                routes:
+                  -
+                    {%- if role == 'controller' %}
+                    default: true
+                    {%- endif %}
+                    ip_netmask: 0.0.0.0/0
+                    next_hop: {get_param: ExternalInterfaceDefaultRoute}
+              {%- elif nets['external'][0]['enabled'] and external_net_type == 'br-ex' and nets['external'][0]['nic_mapping'][role]['vlan'] == 'native' %}
+              -
+                {%- if ovs_dpdk_bridge == 'br-phy' and role == 'compute' %}
+                type: ovs_user_bridge
+                {%- else %}
+                type: ovs_bridge
+                {%- endif %}
+                name: br-ex
+                use_dhcp: false
+                members:
+                  -
+                    type: interface
+                    name: {{ nets['external'][0]['nic_mapping'][role]['members'][0] }}
+                    # force the MAC address of the bridge to this interface
+                    primary: true
+                {%- if role == 'controller' %}
+                dns_servers: {get_param: DnsServers}
+                addresses:
+                  -
+                    ip_netmask: {get_param: ExternalIpSubnet}
+                routes:
+                  -
+                    default: true
+                    ip_netmask: 0.0.0.0/0
+                    next_hop: {get_param: ExternalInterfaceDefaultRoute}
+                {%- endif %}
+              {%- endif %}
+              {%- if nets['storage']['enabled'] and nets['storage']['nic_mapping'][role]['vlan'] == 'native' %}
+              -
+                type: interface
+                name: {{ nets['storage']['nic_mapping'][role]['members'][0] }}
+                use_dhcp: false
+                addresses:
+                  -
+                    ip_netmask: {get_param: StorageIpSubnet}
+              {%- endif %}
+              {%- if nets['api']['enabled'] and nets['api']['nic_mapping'][role]['vlan'] == 'native' %}
+              -
+                type: interface
+                name: {{ nets['api']['nic_mapping'][role]['members'][0] }}
+                use_dhcp: false
+                addresses:
+                  -
+                    ip_netmask: {get_param: InternalApiIpSubnet}
               {%- endif %}
-            {%- endif %}
-            {%- if nets['storage']['enabled'] and nets['storage']['nic_mapping'][role]['vlan'] == 'native' %}
-            -
-              type: interface
-              name: {{ nets['storage']['nic_mapping'][role]['members'][0] }}
-              use_dhcp: false
-              addresses:
-                -
-                  ip_netmask: {get_param: StorageIpSubnet}
-            {%- endif %}
-            {%- if nets['api']['enabled'] and nets['api']['nic_mapping'][role]['vlan'] == 'native' %}
-            -
-              type: interface
-              name: {{ nets['api']['nic_mapping'][role]['members'][0] }}
-              use_dhcp: false
-              addresses:
-                -
-                  ip_netmask: {get_param: InternalApiIpSubnet}
-            {%- endif %}
 
 outputs:
   OS::stack_id:
diff --git a/build/rpm_specs/opnfv-apex-common.spec b/build/rpm_specs/opnfv-apex-common.spec
index 124f2527..5b82b728 100644
--- a/build/rpm_specs/opnfv-apex-common.spec
+++ b/build/rpm_specs/opnfv-apex-common.spec
@@ -76,8 +76,12 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %{_sysconfdir}/opnfv-apex/os-nosdn-ovs_dpdk-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-performance-ha.yaml
-%{_sysconfdir}/opnfv-apex/os-nosdn-pike-noha.yaml
-%{_sysconfdir}/opnfv-apex/os-odl-pike-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-nosdn-pike_upstream-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-nosdn-queens_upstream-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-nosdn-master_upstream-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-odl-pike_upstream-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-odl-queens_upstream-noha.yaml
+%{_sysconfdir}/opnfv-apex/os-odl-master_upstream-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-ovs_dpdk-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-kvm-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-kvm-noha.yaml
@@ -120,6 +124,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %doc %{_docdir}/opnfv/inventory.yaml.example
 
 %changelog
+* Fri Mar 09 2018 Tim Rozet <trozet@redhat.com> - 6.0-2
+  Add upstream deploy files with containers
 * Wed Feb 14 2018 Tim Rozet <trozet@redhat.com> - 6.0-1
   Fix docutils requirement and add python34-distro
 * Wed Nov 29 2017 Tim Rozet <trozet@redhat.com> - 6.0-0
diff --git a/build/upstream-environment.yaml b/build/upstream-environment.yaml
index ef6cdb62..debe6f3a 100644
--- a/build/upstream-environment.yaml
+++ b/build/upstream-environment.yaml
@@ -6,6 +6,7 @@ parameters:
   CloudDomain: opnfvlf.org
 
 parameter_defaults:
+  DockerPuppetProcessCount: 10
   NeutronNetworkVLANRanges: 'datacentre:500:525'
   SshServerOptions:
     HostKey: