Containerized Services for Composable Roles

This change modifies the template interface to support containers and
converts the compute services to composable roles.

Co-Authored-By: Dan Prince <dprince@redhat.com>
Co-Authored-By: Flavio Percoco <flavio@redhat.com>
Co-Authored-By: Martin André <m.andre@redhat.com>
Co-Authored-By: Steve Baker <sbaker@redhat.com>
Change-Id: I82fa58e19de94ec78ca242154bc6ecc592112d1b

9 files changed, 729 insertions, 349 deletions

diff --git a/docker/README-containers.md b/docker/README-containers.md
index ff062a93..5a9f6f3c 100644
--- a/docker/README-containers.md
+++ b/docker/README-containers.md
@@ -20,6 +20,9 @@ glance image-create --name atomic-image --file Fedora-Cloud-Atomic-22-20150521.x
 You can use the tripleo.sh script up until the point of running the Overcloud.
 https://github.com/openstack/tripleo-common/blob/master/scripts/tripleo.sh
 
+You will want to set up the runtime puppet script delivery system described here:
+http://hardysteven.blogspot.ca/2016/08/tripleo-deploy-artifacts-and-puppet.html
+
 Create the Overcloud:
 ```
 $ openstack overcloud deploy --templates=tripleo-heat-templates -e tripleo-heat-templates/environments/docker.yaml -e tripleo-heat-templates/environments/docker-network.yaml --libvirt-type=qemu
diff --git a/docker/compute-post.yaml b/docker/compute-post.yaml
deleted file mode 100644
index 60b831be..00000000
--- a/docker/compute-post.yaml
+++ /dev/null
@@ -1,349 +0,0 @@
-heat_template_version: 2015-10-15
-description: >
-  OpenStack compute node post deployment for Docker.
-
-parameters:
-  servers:
-    type: json
-  DeployIdentifier:
-     type: string
-     description: Value which changes if the node configuration may need to be re-applied
-  DockerNamespace:
-    type: string
-    default: tripleoupstream
-  DockerComputeImage:
-    type: string
-  DockerComputeDataImage:
-    type: string
-  DockerLibvirtImage:
-    type: string
-  DockerOpenvswitchImage:
-    type: string
-  DockerOvsVswitchdImage:
-    type: string
-  DockerOpenvswitchDBImage:
-    type: string
-  LibvirtConfig:
-    type: string
-    default: "/etc/libvirt/libvirtd.conf"
-  NovaConfig:
-    type: string
-    default: "/etc/nova/nova.conf,/etc/nova/rootwrap.conf"
-  NeutronOpenvswitchAgentConfig:
-    type: string
-    default: "/etc/neutron/neutron.conf,/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini"
-  NeutronOpenvswitchAgentPluginVolume:
-    type: string
-    default: "/var/lib/etc-data/neutron/plugins/ml2/openvswitch_agent.ini:/var/lib/kolla/config_files/ovs_neutron_plugin.ini:ro"
-  NeutronOpenvswitchAgentOvsVolume:
-    type: string
-    default: " "
-  StepConfig:
-    type: string
-    description: Config manifests that will be used to step through the deployment.
-    default: ''
-  RoleData:
-    type: json
-    default: {}
-
-
-resources:
-
-  ComputePuppetConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: puppet
-      options:
-        enable_hiera: True
-        enable_facter: False
-        tags: package,file,concat,file_line,nova_config,neutron_config,neutron_agent_ovs,neutron_plugin_ml2
-      inputs:
-      - name: tripleo::packages::enable_install
-        type: Boolean
-        default: True
-      outputs:
-      - name: result
-      config:
-        list_join:
-        - ''
-        - - get_file: ../puppet/manifests/overcloud_compute.pp
-          - {get_param: StepConfig}
-
-
-  ComputePuppetDeployment:
-    type: OS::Heat::SoftwareDeployments
-    properties:
-      name: ComputePuppetDeployment
-      servers:  {get_param: servers}
-      config: {get_resource: ComputePuppetConfig}
-      input_values:
-        update_identifier: {get_param: DeployIdentifier}
-        tripleo::packages::enable_install: True
-
-  CopyEtcConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      outputs:
-      - name: result
-      config: {get_file: ./copy-etc.sh}
-
-  CopyEtcDeployment:
-    type: OS::Heat::SoftwareDeployments
-    depends_on: ComputePuppetDeployment
-    properties:
-      name: CopyEtcDeployment
-      config: {get_resource: CopyEtcConfig}
-      servers:  {get_param: servers}
-
-  CopyJsonConfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      inputs:
-      - name: libvirt_config
-      - name: nova_config
-      - name: neutron_openvswitch_agent_config
-      config: |
-        #!/bin/python
-        import json
-        import os
-
-        data = {}
-        file_perms = '600'
-        libvirt_perms = '644'
-
-        libvirt_config = os.getenv('libvirt_config').split(',')
-        nova_config = os.getenv('nova_config').split(',')
-        neutron_openvswitch_agent_config = os.getenv('neutron_openvswitch_agent_config').split(',')
-
-        # Command, Config_files, Owner, Perms
-        services = {'nova-libvirt': ['/usr/sbin/libvirtd', libvirt_config, 'root', libvirt_perms],
-                    'nova-compute': ['/usr/bin/nova-compute', nova_config, 'nova', file_perms],
-                    'neutron-openvswitch-agent': ['/usr/bin/neutron-openvswitch-agent', neutron_openvswitch_agent_config, 'neutron', file_perms],
-                    'ovs-vswitchd': ['/usr/sbin/ovs-vswitchd unix:/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --log-file=/var/log/openvswitch/ovs-vswitchd.log'],
-                    'ovsdb-server': ['/usr/sbin/ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/run/openvswitch/db.sock --log-file=/var/log/openvswitch/ovsdb-server.log']
-                   }
-
-
-        def build_config_files(config, owner, perms):
-            config_source = '/var/lib/kolla/config_files/'
-            config_files_dict = {}
-            source = os.path.basename(config)
-            dest = config
-            config_files_dict.update({'source': config_source + source,
-                                      'dest': dest,
-                                      'owner': owner,
-                                      'perm': perms})
-            return config_files_dict
-
-
-        for service in services:
-            if service != 'ovs-vswitchd' and service != 'ovsdb-server':
-                command = services.get(service)[0]
-                config_files = services.get(service)[1]
-                owner = services.get(service)[2]
-                perms = services.get(service)[3]
-                config_files_list = []
-                for config_file in config_files:
-                    if service == 'nova-libvirt':
-                        command = command + ' --config ' + config_file
-                    else:
-                        command = command + ' --config-file ' + config_file
-                    data['command'] = command
-                    config_files_dict = build_config_files(config_file, owner, perms)
-                    config_files_list.append(config_files_dict)
-                data['config_files'] = config_files_list
-            else:
-                data['command'] = services.get(service)[0]
-                data['config_files'] = []
-
-            json_config_dir = '/var/lib/etc-data/json-config/'
-            with open(json_config_dir + service + '.json', 'w') as json_file:
-                json.dump(data, json_file, sort_keys=True, indent=4, separators=(',', ': '))
-
-  CopyJsonDeployment:
-    type: OS::Heat::SoftwareDeployments
-    depends_on: CopyEtcDeployment
-    properties:
-      name: CopyJsonDeployment
-      config: {get_resource: CopyJsonConfig}
-      servers:  {get_param: servers}
-      input_values:
-        libvirt_config: {get_param: LibvirtConfig}
-        nova_config: {get_param: NovaConfig}
-        neutron_openvswitch_agent_config: {get_param: NeutronOpenvswitchAgentConfig}
-
-  NovaComputeContainersDeploymentOVS:
-    type: OS::Heat::StructuredDeployments
-    depends_on: CopyJsonDeployment
-    properties:
-      name: NovaComputeContainersDeploymentOVS
-      config: {get_resource: NovaComputeContainersConfigOVS}
-      servers: {get_param: servers}
-
-  NovaComputeContainersConfigOVS:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: docker-cmd
-      config:
-        ovsvswitchd:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerOvsVswitchdImage} ]
-          net: host
-          privileged: true
-          restart: always
-          volumes:
-           - /run:/run
-           - /lib/modules:/lib/modules:ro
-           - /var/lib/etc-data/json-config/ovs-vswitchd.json:/var/lib/kolla/config_files/config.json
-          environment:
-           - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-
-        openvswitchdb:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerOpenvswitchDBImage} ]
-          net: host
-          restart: always
-          volumes:
-           - /run:/run
-           - /var/lib/etc-data/json-config/ovsdb-server.json:/var/lib/kolla/config_files/config.json
-           - /etc/openvswitchd:/etc/openvswitchd
-          environment:
-           - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-
-  NovaComputeContainersDeploymentNetconfig:
-    type: OS::Heat::SoftwareDeployments
-    depends_on: NovaComputeContainersDeploymentOVS
-    properties:
-      name: NovaComputeContainersDeploymentNetconfig
-      config: {get_resource: NovaComputeContainersConfigNetconfig}
-      servers: {get_param: servers}
-
-  # We run os-net-config here because we depend on the ovs containers to be up
-  # and running before we configure the network.  This allows explicit timing
-  # of the network configuration.
-  NovaComputeContainersConfigNetconfig:
-    type: OS::Heat::SoftwareConfig
-    properties:
-      group: script
-      outputs:
-      - name: result
-      config: |
-        #!/bin/bash
-        /usr/local/bin/run-os-net-config
-
-  LibvirtContainersDeployment:
-    type: OS::Heat::StructuredDeployments
-    depends_on: [CopyJsonDeployment, CopyEtcDeployment, ComputePuppetDeployment, NovaComputeContainersDeploymentNetconfig]
-    properties:
-      name: LibvirtContainersDeployment
-      config: {get_resource: LibvirtContainersConfig}
-      servers: {get_param: servers}
-
-  LibvirtContainersConfig:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: docker-cmd
-      config:
-        computedata:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerComputeDataImage} ]
-          container_name: computedata
-          volumes:
-           - /var/lib/nova/instances
-           - /var/lib/libvirt
-
-        libvirt:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerLibvirtImage} ]
-          net: host
-          pid: host
-          privileged: true
-          restart: always
-          volumes:
-           - /run:/run
-           - /lib/modules:/lib/modules:ro
-           - /dev:/dev
-           - /lib/udev:/lib/udev
-           - /sys/fs/cgroup:/sys/fs/cgroup
-           - /var/lib/etc-data/json-config/nova-libvirt.json:/var/lib/kolla/config_files/config.json
-           - /var/lib/etc-data/libvirt/libvirtd.conf:/var/lib/kolla/config_files/libvirtd.conf
-          environment:
-           - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-          volumes_from:
-           - computedata
-
-  NovaComputeContainersDeployment:
-    type: OS::Heat::StructuredDeployments
-    depends_on: [CopyJsonDeployment, CopyEtcDeployment, ComputePuppetDeployment, NovaComputeContainersDeploymentNetconfig, LibvirtContainersDeployment]
-    properties:
-      name: NovaComputeContainersDeployment
-      config: {get_resource: NovaComputeContainersConfig}
-      servers: {get_param: servers}
-
-  NovaComputeContainersConfig:
-    type: OS::Heat::StructuredConfig
-    properties:
-      group: docker-cmd
-      config:
-        neutronovsagent:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerOpenvswitchImage} ]
-          net: host
-          pid: host
-          privileged: true
-          restart: always
-          volumes:
-            str_split:
-              - ","
-              - list_join:
-                 - ","
-                 - [ "/run:/run", "/lib/modules:/lib/modules:ro",
-                     "/var/lib/etc-data/json-config/neutron-openvswitch-agent.json:/var/lib/kolla/config_files/config.json",
-                     "/var/lib/etc-data/neutron/neutron.conf:/var/lib/kolla/config_files/neutron.conf:ro",
-                     "/var/lib/etc-data/neutron/plugins/ml2/ml2_conf.ini:/var/lib/kolla/config_files/ml2_conf.ini:ro",
-                     {get_param: NeutronOpenvswitchAgentPluginVolume},
-                     {get_param: NeutronOpenvswitchAgentOvsVolume} ]
-          environment:
-           - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-          volumes_from:
-           - computedata
-
-        novacompute:
-          image:
-            list_join:
-            - '/'
-            - [ {get_param: DockerNamespace}, {get_param: DockerComputeImage} ]
-          net: host
-          privileged: true
-          restart: always
-          volumes:
-           - /run:/run
-           - /lib/modules:/lib/modules:ro
-           - /dev:/dev
-           - /lib/udev:/lib/udev
-           - /etc/iscsi:/etc/iscsi
-           - /var/lib/etc-data/json-config/nova-compute.json:/var/lib/kolla/config_files/config.json
-           - /var/lib/etc-data/nova/nova.conf:/var/lib/kolla/config_files/nova.conf:ro
-           - /var/lib/etc-data/nova/rootwrap.conf:/var/lib/kolla/config_files/rootwrap.conf:ro
-          environment:
-           - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-          volumes_from:
-           - computedata
-
-  ExtraConfig:
-    depends_on: NovaComputeContainersDeployment
-    type: OS::TripleO::NodeExtraConfigPost
-    properties:
-        servers: {get_param: servers}
diff --git a/docker/copy-json.py b/docker/copy-json.py
new file mode 100644
index 00000000..e85ff11e
--- /dev/null
+++ b/docker/copy-json.py
@@ -0,0 +1,72 @@
+#!/bin/python
+import json
+import os
+
+data = {}
+file_perms = '0600'
+libvirt_perms = '0644'
+
+libvirt_config = os.getenv('libvirt_config').split(',')
+nova_config = os.getenv('nova_config').split(',')
+neutron_openvswitch_agent_config = os.getenv('neutron_openvswitch_agent_config').split(',')
+
+# Command, Config_files, Owner, Perms
+services = {
+    'nova-libvirt': [
+        '/usr/sbin/libvirtd',
+        libvirt_config,
+        'root',
+        libvirt_perms],
+    'nova-compute': [
+        '/usr/bin/nova-compute',
+        nova_config,
+        'nova',
+        file_perms],
+    'neutron-openvswitch-agent': [
+        '/usr/bin/neutron-openvswitch-agent',
+        neutron_openvswitch_agent_config,
+        'neutron',
+        file_perms],
+    'ovs-vswitchd': [
+        '/usr/sbin/ovs-vswitchd unix:/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --log-file=/var/log/kolla/openvswitch/ovs-vswitchd.log'],
+    'ovsdb-server': [
+        '/usr/sbin/ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/run/openvswitch/db.sock --remote=ptcp:6640:127.0.0.1 --log-file=/var/log/kolla/openvswitch/ovsdb-server.log']
+}
+
+
+def build_config_files(config, owner, perms):
+    config_source = '/var/lib/kolla/config_files/'
+    config_files_dict = {}
+    source = os.path.basename(config)
+    dest = config
+    config_files_dict.update({'source': config_source + source,
+                              'dest': dest,
+                              'owner': owner,
+                              'perm': perms})
+    return config_files_dict
+
+
+for service in services:
+    if service != 'ovs-vswitchd' and service != 'ovsdb-server':
+        command = services.get(service)[0]
+        config_files = services.get(service)[1]
+        owner = services.get(service)[2]
+        perms = services.get(service)[3]
+        config_files_list = []
+        for config_file in config_files:
+            if service == 'nova-libvirt':
+                command = command + ' --config ' + config_file
+            else:
+                command = command + ' --config-file ' + config_file
+            data['command'] = command
+            config_files_dict = build_config_files(config_file, owner, perms)
+            config_files_list.append(config_files_dict)
+        data['config_files'] = config_files_list
+    else:
+        data['command'] = services.get(service)[0]
+        data['config_files'] = []
+
+    json_config_dir = '/var/lib/etc-data/json-config/'
+    with open(json_config_dir + service + '.json', 'w') as json_file:
+        json.dump(data, json_file, sort_keys=True, indent=4,
+                  separators=(',', ': '))
diff --git a/docker/post.j2.yaml b/docker/post.j2.yaml
new file mode 100644
index 00000000..de17cffe
--- /dev/null
+++ b/docker/post.j2.yaml
@@ -0,0 +1,308 @@
+heat_template_version: 2016-10-14
+
+description: >
+  Post-deploy configuration steps via puppet for all roles,
+  as defined in ../roles_data.yaml
+
+parameters:
+  servers:
+    type: json
+    description: Mapping of Role name e.g Controller to a list of servers
+
+  role_data:
+    type: json
+    description: Mapping of Role name e.g Controller to the per-role data
+
+  DeployIdentifier:
+    default: ''
+    type: string
+    description: >
+      Setting this to a unique value will re-run any deployment tasks which
+      perform configuration on a Heat stack-update.
+
+  DockerNamespace:
+    description: namespace
+    default: 'tripleoupstream'
+    type: string
+
+  DockerOpenvswitchDBImage:
+    description: image
+    default: 'centos-binary-openvswitch-db-server'
+    type: string
+
+  DockerOvsVswitchdImage:
+    description: image
+    default: 'centos-binary-openvswitch-vswitchd'
+    type: string
+
+  LibvirtConfig:
+    type: string
+    default: "/etc/libvirt/libvirtd.conf"
+
+  NovaConfig:
+    type: string
+    default: "/etc/nova/nova.conf,/etc/nova/rootwrap.conf"
+
+  NeutronOpenvswitchAgentConfig:
+    type: string
+    default: "/etc/neutron/neutron.conf,/etc/neutron/plugins/openvswitch/ovs_neutron_plugin.ini"
+
+resources:
+
+{% for role in roles %}
+  # Post deployment steps for all roles
+  # A single config is re-applied with an incrementing step number
+  # {{role.name}} Role steps
+  {{role.name}}ArtifactsConfig:
+    type: ../puppet/deploy-artifacts.yaml
+
+  {{role.name}}ArtifactsDeploy:
+    type: OS::Heat::StructuredDeploymentGroup
+    properties:
+      servers:  {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}ArtifactsConfig}
+
+  {{role.name}}PreConfig:
+    type: OS::TripleO::Tasks::{{role.name}}PreConfig
+    properties:
+      servers: {get_param: [servers, {{role.name}}]}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}Config:
+    type: OS::TripleO::{{role.name}}Config
+    properties:
+      StepConfig: {get_param: [role_data, {{role.name}}, step_config]}
+  {% if role.name.lower() == 'compute' %}
+      PuppetTags: {get_param: [role_data, {{role.name}}, puppet_tags]}
+  {% endif %}
+
+  # Step through a series of configuration steps
+  {{role.name}}Deployment_Step1:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy]
+    properties:
+      name: {{role.name}}Deployment_Step1
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: 1
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}Deployment_Step2:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step1
+  {% endfor %}
+    properties:
+      name: {{role.name}}Deployment_Step2
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: 2
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}Deployment_Step3:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step2
+  {% endfor %}
+    properties:
+      name: {{role.name}}Deployment_Step3
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: 3
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}Deployment_Step4:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step3
+  {% endfor %}
+    properties:
+      name: {{role.name}}Deployment_Step4
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: 4
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}Deployment_Step5:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step4
+  {% endfor %}
+    properties:
+      name: {{role.name}}Deployment_Step5
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}Config}
+      input_values:
+        step: 5
+        update_identifier: {get_param: DeployIdentifier}
+
+  {{role.name}}PostConfig:
+    type: OS::TripleO::Tasks::{{role.name}}PostConfig
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}Deployment_Step5
+  {% endfor %}
+    properties:
+      servers:  {get_param: servers}
+      input_values:
+        update_identifier: {get_param: DeployIdentifier}
+
+  # Note, this should come last, so use depends_on to ensure
+  # this is created after any other resources.
+  {{role.name}}ExtraConfigPost:
+    depends_on:
+  {% for dep in roles %}
+      - {{dep.name}}PostConfig
+  {% endfor %}
+    type: OS::TripleO::NodeExtraConfigPost
+    properties:
+        servers: {get_param: [servers, {{role.name}}]}
+
+  {% if role.name.lower() == 'compute' %}
+  CopyEtcConfig:
+    type: OS::Heat::SoftwareConfig
+    depends_on: {{role.name}}PostConfig
+    properties:
+      group: script
+      outputs:
+        - name: result
+      config: {get_file: ../docker/copy-etc.sh}
+
+  CopyEtcDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      name: CopyEtcDeployment
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: CopyEtcConfig}
+
+  CopyJsonConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      inputs:
+      - name: libvirt_config
+      - name: nova_config
+      - name: neutron_openvswitch_agent_config
+      config: {get_file: ../docker/copy-json.py}
+
+  CopyJsonDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: CopyEtcDeployment
+    properties:
+      name: CopyJsonDeployment
+      config: {get_resource: CopyJsonConfig}
+      servers: {get_param: [servers, {{role.name}}]}
+      input_values:
+        libvirt_config: {get_param: LibvirtConfig}
+        nova_config: {get_param: NovaConfig}
+        neutron_openvswitch_agent_config: {get_param: NeutronOpenvswitchAgentConfig}
+
+  NovaComputeContainersDeploymentOVS:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on: CopyJsonDeployment
+    properties:
+      name: NovaComputeContainersDeploymentOVS
+      config: {get_resource: NovaComputeContainersConfigOVS}
+      servers: {get_param: [servers, {{role.name}}]}
+
+  NovaComputeContainersConfigOVS:
+    type: OS::Heat::StructuredConfig
+    properties:
+      group: docker-cmd
+      config:
+        openvswitchdb:
+          image:
+            list_join:
+              - '/'
+              - [ {get_param: DockerNamespace}, {get_param: DockerOpenvswitchDBImage} ]
+          net: host
+          restart: always
+          volumes:
+            - /var/lib/etc-data/json-config/ovsdb-server.json:/var/lib/kolla/config_files/config.json
+            - /etc/localtime:/etc/localtime:ro
+            - /run:/run
+            - logs:/var/log/kolla/
+            - openvswitch_db:/var/lib/openvswitch/
+          environment:
+            - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+
+        ovsvswitchd:
+          image:
+            list_join:
+              - '/'
+              - [ {get_param: DockerNamespace}, {get_param: DockerOvsVswitchdImage} ]
+          net: host
+          privileged: true
+          restart: always
+          volumes:
+            - /var/lib/etc-data/json-config/ovs-vswitchd.json:/var/lib/kolla/config_files/config.json
+            - /etc/localtime:/etc/localtime:ro
+            - /lib/modules:/lib/modules:ro
+            - /run:/run
+            - logs:/var/log/kolla/
+          environment:
+            - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+
+  NovaComputeContainersDeploymentNetconfig:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: NovaComputeContainersDeploymentOVS
+    properties:
+      name: NovaComputeContainersDeploymentNetconfig
+      config: {get_resource: NovaComputeContainersConfigNetconfig}
+      servers: {get_param: [servers, {{role.name}}]}
+
+  # We run os-net-config here because we depend on the ovs containers to be up
+  # and running before we configure the network.  This allows explicit timing
+  # of the network configuration.
+  NovaComputeContainersConfigNetconfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      outputs:
+      - name: result
+      config: |
+        #!/bin/bash
+        /usr/local/bin/run-os-net-config
+
+  {{role.name}}ContainersConfig_Step1:
+    type: OS::Heat::StructuredConfig
+    depends_on: CopyJsonDeployment
+    properties:
+      group: docker-cmd
+      config:
+        {get_param: [role_data, {{role.name}}, docker_config, step_1]}
+
+  {{role.name}}ContainersConfig_Step2:
+    type: OS::Heat::StructuredConfig
+    depends_on: CopyJsonDeployment
+    properties:
+      group: docker-cmd
+      config:
+        {get_param: [role_data, {{role.name}}, docker_config, step_2]}
+
+  {{role.name}}ContainersDeployment_Step1:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy, NovaComputeContainersDeploymentNetconfig]
+    properties:
+      name: {{role.name}}ContainersDeployment_Step1
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}ContainersConfig_Step1}
+
+  {{role.name}}ContainersDeployment_Step2:
+    type: OS::Heat::StructuredDeploymentGroup
+    depends_on: {{role.name}}ContainersDeployment_Step1
+    properties:
+      name: {{role.name}}ContainersDeployment_Step2
+      servers: {get_param: [servers, {{role.name}}]}
+      config: {get_resource: {{role.name}}ContainersConfig_Step2}
+  {% endif %}
+{% endfor %}
diff --git a/docker/services/README.rst b/docker/services/README.rst
new file mode 100644
index 00000000..8d1f9e86
--- /dev/null
+++ b/docker/services/README.rst
@@ -0,0 +1,60 @@
+========
+services
+========
+
+A TripleO nested stack Heat template that encapsulates generic configuration
+data to configure a specific service. This generally includes everything
+needed to configure the service excluding the local bind ports which
+are still managed in the per-node role templates directly (controller.yaml,
+compute.yaml, etc.). All other (global) service settings go into
+the puppet/service templates.
+
+Input Parameters
+----------------
+
+Each service may define its own input parameters and defaults.
+Operators will use the parameter_defaults section of any Heat
+environment to set per service parameters.
+
+Config Settings
+---------------
+
+Each service may define a config_settings output variable which returns
+Hiera settings to be configured.
+
+Steps
+-----
+
+Each service may define an output variable which returns a puppet manifest
+snippet that will run at each of the following steps. Earlier manifests
+are re-asserted when applying latter ones.
+
+ * config_settings: Custom hiera settings for this service. These are
+   used to generate configs.
+
+ * step_config: A puppet manifest that is used to step through the deployment
+   sequence. Each sequence is given a "step" (via hiera('step') that provides
+   information for when puppet classes should activate themselves.
+
+ * docker_compose:
+
+ * container_name:
+
+ * volumes:
+
+Steps correlate to the following:
+
+   1) Service configuration generation with puppet.
+
+   2) Early Openstack Service setup (database init?)
+
+   3) Early containerized networking services startup (OVS)
+
+   4) Network configuration
+
+   5) General OpenStack Services
+
+   6) Service activation (Pacemaker)
+
+   7) Fencing (Pacemaker)
+
diff --git a/docker/services/neutron-ovs-agent.yaml b/docker/services/neutron-ovs-agent.yaml
new file mode 100644
index 00000000..8d092a34
--- /dev/null
+++ b/docker/services/neutron-ovs-agent.yaml
@@ -0,0 +1,75 @@
+heat_template_version: 2015-04-30
+
+description: >
+  OpenStack Neutron openvswitch service
+
+parameters:
+  DockerNamespace:
+    description: namespace
+    default: 'tripleoupstream'
+    type: string
+  DockerOpenvswitchImage:
+    description: image
+    default: 'centos-binary-neutron-openvswitch-agent'
+    type: string
+  NeutronOpenvswitchAgentPluginVolume:
+    type: string
+    default: "/var/lib/etc-data/neutron/plugins/ml2/openvswitch_agent.ini:/var/lib/kolla/config_files/ovs_neutron_plugin.ini:ro"
+  NeutronOpenvswitchAgentOvsVolume:
+    type: string
+    default: " "
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  NeutronOvsAgentBase:
+    type: ../../puppet/services/neutron-ovs-agent.yaml
+    properties:
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for Neutron openvswitch service
+    value:
+      config_settings: {get_attr: [NeutronOvsAgentBase, role_data, config_settings]}
+      step_config: {get_attr: [NeutronOvsAgentBase, role_data, step_config]}
+      puppet_tags: neutron_config,neutron_agent_ovs,neutron_plugin_ml2
+      docker_config:
+        step_1:
+          neutronovsagent:
+            image:
+              list_join:
+              - '/'
+              - [ {get_param: DockerNamespace}, {get_param: DockerOpenvswitchImage} ]
+            net: host
+            pid: host
+            privileged: true
+            restart: always
+            volumes:
+              - /var/lib/etc-data/json-config/neutron-openvswitch-agent.json:/var/lib/kolla/config_files/config.json
+              - /var/lib/etc-data/neutron/neutron.conf:/var/lib/kolla/config_files/neutron.conf:ro
+              - /var/lib/etc-data/neutron/plugins/ml2/ml2_conf.ini:/var/lib/kolla/config_files/ml2_conf.ini:ro
+              - {get_param: NeutronOpenvswitchAgentPluginVolume}
+              - {get_param: NeutronOpenvswitchAgentOvsVolume}
+              - /etc/localtime:/etc/localtime:ro
+              - /lib/modules:/lib/modules:ro
+              - /run:/run
+              - logs:/var/log/kolla/
+            environment:
+              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+        step_2: {}
diff --git a/docker/services/nova-compute.yaml b/docker/services/nova-compute.yaml
new file mode 100644
index 00000000..5c56aeee
--- /dev/null
+++ b/docker/services/nova-compute.yaml
@@ -0,0 +1,69 @@
+heat_template_version: 2015-04-30
+
+description: >
+  OpenStack containerized Nova Compute service
+
+parameters:
+  DockerNamespace:
+    description: namespace
+    default: 'tripleoupstream'
+    type: string
+  DockerNovaComputeImage:
+    description: image
+    default: 'centos-binary-nova-compute'
+    type: string
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  NovaComputeBase:
+    type: ../../puppet/services/nova-compute.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Nova Compute service.
+    value:
+      config_settings: {get_attr: [NovaComputeBase, role_data, config_settings]}
+      step_config: {get_attr: [NovaComputeBase, role_data, step_config]}
+      puppet_tags: nova_config,nova_paste_api_ini
+      docker_config:
+        step_1:
+          novacompute:
+            image:
+              list_join:
+              - '/'
+              - [ {get_param: DockerNamespace}, {get_param: DockerNovaComputeImage} ]
+            net: host
+            privileged: true
+            user: root
+            restart: always
+            volumes:
+              - /var/lib/etc-data/json-config/nova-compute.json:/var/lib/kolla/config_files/config.json
+              - /var/lib/etc-data/nova/nova.conf:/var/lib/kolla/config_files/nova.conf:ro
+              - /var/lib/etc-data/nova/rootwrap.conf:/var/lib/kolla/config_files/rootwrap.conf:ro
+              - /etc/localtime:/etc/localtime:ro
+              - /lib/modules:/lib/modules:ro
+              - /run:/run
+              - /dev:/dev
+              - logs:/var/log/kolla/
+              - /etc/iscsi:/etc/iscsi
+              - libvirtd:/var/lib/libvirt
+              - nova_compute:/var/lib/nova/
+            environment:
+             - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+        step_2: {}
diff --git a/docker/services/nova-libvirt.yaml b/docker/services/nova-libvirt.yaml
new file mode 100644
index 00000000..36511557
--- /dev/null
+++ b/docker/services/nova-libvirt.yaml
@@ -0,0 +1,69 @@
+heat_template_version: 2015-04-30
+
+description: >
+  OpenStack Libvirt Service
+
+parameters:
+  DockerNamespace:
+    description: namespace
+    default: 'tripleoupstream'
+    type: string
+  DockerLibvirtImage:
+    description: image
+    default: 'centos-binary-libvirt'
+    type: string
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  NovaLibvirtBase:
+    type: ../../puppet/services/nova-libvirt.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Libvirt service.
+    value:
+      config_settings: {get_attr: [NovaLibvirtBase, role_data, config_settings]}
+      step_config: {get_attr: [NovaLibvirtBase, role_data, step_config]}
+      puppet_tags: nova_config
+      docker_config:
+        step_1:
+          nova_libvirt:
+            image:
+              list_join:
+                - '/'
+                - [ {get_param: DockerNamespace}, {get_param: DockerLibvirtImage} ]
+            net: host
+            pid: host
+            privileged: true
+            restart: always
+            volumes:
+              - /var/lib/etc-data/json-config/nova-libvirt.json:/var/lib/kolla/config_files/config.json
+              - /var/lib/etc-data/libvirt/libvirtd.conf:/var/lib/kolla/config_files/libvirtd.conf
+              - /etc/localtime:/etc/localtime:ro
+              - /lib/modules:/lib/modules:ro
+              - /run:/run
+              - /dev:/dev
+              - /sys/fs/cgroup:/sys/fs/cgroup
+              - logs:/var/log/kolla/
+              - libvirtd:/var/lib/libvirt
+              - nova_compute:/var/lib/nova/
+              - nova_libvirt_qemu:/etc/libvirt/qemu
+            environment:
+              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+        step_2: {}
diff --git a/docker/services/services.yaml b/docker/services/services.yaml
new file mode 100644
index 00000000..37e7b655
--- /dev/null
+++ b/docker/services/services.yaml
@@ -0,0 +1,73 @@
+heat_template_version: 2016-10-14
+
+description: >
+  Utility stack to convert an array of services into a set of combined
+  role configs.
+
+parameters:
+  Services:
+    default: []
+    description: |
+        List nested stack service templates.
+    type: comma_delimited_list
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  DefaultPasswords:
+    default: {}
+    description: Mapping of service -> default password. Used to help
+                 pass top level passwords managed by Heat into services.
+    type: json
+
+resources:
+
+  PuppetServices:
+    type: ../../puppet/services/services.yaml
+    properties:
+      Services: {get_param: Services}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      EndpointMap: {get_param: EndpointMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+
+  ServiceChain:
+    type: OS::Heat::ResourceChain
+    properties:
+      resources: {get_param: Services}
+      concurrent: true
+      resource_properties:
+        ServiceNetMap: {get_param: ServiceNetMap}
+        EndpointMap: {get_param: EndpointMap}
+        DefaultPasswords: {get_param: DefaultPasswords}
+
+outputs:
+  role_data:
+    description: Combined Role data for this set of services.
+    value:
+      service_names:
+        {get_attr: [PuppetServices, role_data, service_names]}
+      monitoring_subscriptions:
+        {get_attr: [PuppetServices, role_data, monitoring_subscriptions]}
+      logging_sources:
+        {get_attr: [PuppetServices, role_data, logging_sources]}
+      logging_groups:
+        {get_attr: [PuppetServices, role_data, logging_groups]}
+      service_config_settings:
+        {get_attr: [PuppetServices, role_data, service_config_settings]}
+      config_settings:
+        {get_attr: [PuppetServices, role_data, config_settings]}
+      global_config_settings:
+        {get_attr: [PuppetServices, role_data, global_config_settings]}
+      step_config:
+        {get_attr: [PuppetServices, role_data, step_config]}
+      puppet_tags: {list_join: [",", {get_attr: [ServiceChain, role_data, puppet_tags]}]}
+      docker_config:
+        step_1: {map_merge: {get_attr: [ServiceChain, role_data, docker_config, step_1]}}
+        step_2: {map_merge: {get_attr: [ServiceChain, role_data, docker_config, step_2]}}