docker: eliminate copy-json.py in favor of json-file

This patch rewires how we configure the Kolla external config files
via Heat templates and uses a more simple json-file heat hook to
directly write out Kolla config files to disk.

By using a heat hook instead of a shell script we can avoid
Json conversion issues.  Additionally, This generic json file hook will
be useful for other ad-hoc Json file configuration within the TripleO
docker architecture.

Co-Authored-By: Martin André <m.andre@redhat.com>
Change-Id: I8c72a4a9a7022f722bfe1cef3e18517605720cce
Depends-On: I2b372ac2e291339e436202c9fe58a681ed6a743f
Depends-On: Id3f779b11e23fd3122ef29b7ccbae116667d4520

7 files changed, 54 insertions, 107 deletions

diff --git a/docker/copy-json.py b/docker/copy-json.py
deleted file mode 100644
index e85ff11e..00000000
--- a/docker/copy-json.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/python
-import json
-import os
-
-data = {}
-file_perms = '0600'
-libvirt_perms = '0644'
-
-libvirt_config = os.getenv('libvirt_config').split(',')
-nova_config = os.getenv('nova_config').split(',')
-neutron_openvswitch_agent_config = os.getenv('neutron_openvswitch_agent_config').split(',')
-
-# Command, Config_files, Owner, Perms
-services = {
-    'nova-libvirt': [
-        '/usr/sbin/libvirtd',
-        libvirt_config,
-        'root',
-        libvirt_perms],
-    'nova-compute': [
-        '/usr/bin/nova-compute',
-        nova_config,
-        'nova',
-        file_perms],
-    'neutron-openvswitch-agent': [
-        '/usr/bin/neutron-openvswitch-agent',
-        neutron_openvswitch_agent_config,
-        'neutron',
-        file_perms],
-    'ovs-vswitchd': [
-        '/usr/sbin/ovs-vswitchd unix:/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --log-file=/var/log/kolla/openvswitch/ovs-vswitchd.log'],
-    'ovsdb-server': [
-        '/usr/sbin/ovsdb-server /etc/openvswitch/conf.db -vconsole:emer -vsyslog:err -vfile:info --remote=punix:/run/openvswitch/db.sock --remote=ptcp:6640:127.0.0.1 --log-file=/var/log/kolla/openvswitch/ovsdb-server.log']
-}
-
-
-def build_config_files(config, owner, perms):
-    config_source = '/var/lib/kolla/config_files/'
-    config_files_dict = {}
-    source = os.path.basename(config)
-    dest = config
-    config_files_dict.update({'source': config_source + source,
-                              'dest': dest,
-                              'owner': owner,
-                              'perm': perms})
-    return config_files_dict
-
-
-for service in services:
-    if service != 'ovs-vswitchd' and service != 'ovsdb-server':
-        command = services.get(service)[0]
-        config_files = services.get(service)[1]
-        owner = services.get(service)[2]
-        perms = services.get(service)[3]
-        config_files_list = []
-        for config_file in config_files:
-            if service == 'nova-libvirt':
-                command = command + ' --config ' + config_file
-            else:
-                command = command + ' --config-file ' + config_file
-            data['command'] = command
-            config_files_dict = build_config_files(config_file, owner, perms)
-            config_files_list.append(config_files_dict)
-        data['config_files'] = config_files_list
-    else:
-        data['command'] = services.get(service)[0]
-        data['config_files'] = []
-
-    json_config_dir = '/var/lib/etc-data/json-config/'
-    with open(json_config_dir + service + '.json', 'w') as json_file:
-        json.dump(data, json_file, sort_keys=True, indent=4,
-                  separators=(',', ': '))
diff --git a/docker/post.j2.yaml b/docker/post.j2.yaml
index 1ba96e27..865c74e5 100644
--- a/docker/post.j2.yaml
+++ b/docker/post.j2.yaml
@@ -20,23 +20,6 @@ parameters:
       Setting this to a unique value will re-run any deployment tasks which
       perform configuration on a Heat stack-update.
 
-  DockerNamespace:
-    description: namespace
-    default: 'tripleoupstream'
-    type: string
-
-  LibvirtConfig:
-    type: string
-    default: "/etc/libvirt/libvirtd.conf"
-
-  NovaConfig:
-    type: string
-    default: "/etc/nova/nova.conf,/etc/nova/rootwrap.conf"
-
-  NeutronOpenvswitchAgentConfig:
-    type: string
-    default: "/etc/neutron/neutron.conf,/etc/neutron/plugins/ml2/openvswitch_agent.ini"
-
 resources:
 
 {% for role in roles %}
@@ -174,31 +157,24 @@ resources:
       servers: {get_param: [servers, {{role.name}}]}
       config: {get_resource: CopyEtcConfig}
 
-  CopyJsonConfig:
-    type: OS::Heat::SoftwareConfig
+  {{role.name}}KollaJsonConfig:
+    type: OS::Heat::StructuredConfig
+    depends_on: CopyEtcDeployment
     properties:
-      group: script
-      inputs:
-      - name: libvirt_config
-      - name: nova_config
-      - name: neutron_openvswitch_agent_config
-      config: {get_file: ../docker/copy-json.py}
+      group: json-file
+      config:
+        {get_param: [role_data, {{role.name}}, kolla_config]}
 
-  CopyJsonDeployment:
+  {{role.name}}KollaJsonDeployment:
     type: OS::Heat::SoftwareDeploymentGroup
-    depends_on: CopyEtcDeployment
     properties:
-      name: CopyJsonDeployment
-      config: {get_resource: CopyJsonConfig}
+      name: {{role.name}}KollaJsonDeployment
+      config: {get_resource: {{role.name}}KollaJsonConfig}
       servers: {get_param: [servers, {{role.name}}]}
-      input_values:
-        libvirt_config: {get_param: LibvirtConfig}
-        nova_config: {get_param: NovaConfig}
-        neutron_openvswitch_agent_config: {get_param: NeutronOpenvswitchAgentConfig}
 
   {{role.name}}ContainersConfig_Step1:
     type: OS::Heat::StructuredConfig
-    depends_on: CopyJsonDeployment
+    depends_on: {{role.name}}KollaJsonDeployment
     properties:
       group: docker-cmd
       config:
@@ -206,7 +182,7 @@ resources:
 
   {{role.name}}ContainersConfig_Step2:
     type: OS::Heat::StructuredConfig
-    depends_on: CopyJsonDeployment
+    depends_on: {{role.name}}KollaJsonDeployment
     properties:
       group: docker-cmd
       config:
diff --git a/docker/services/README.rst b/docker/services/README.rst
index 8d1f9e86..60719bfc 100644
--- a/docker/services/README.rst
+++ b/docker/services/README.rst
@@ -32,6 +32,11 @@ are re-asserted when applying latter ones.
  * config_settings: Custom hiera settings for this service. These are
    used to generate configs.
 
+ * kolla_config: Contains YAML that represents how to map config files
+   into the kolla container. This config file is typically mapped into
+   the container itself at the /var/lib/kolla/config_files/config.json
+   location and drives how kolla's external config mechanisms work.
+
  * step_config: A puppet manifest that is used to step through the deployment
    sequence. Each sequence is given a "step" (via hiera('step') that provides
    information for when puppet classes should activate themselves.
diff --git a/docker/services/neutron-ovs-agent.yaml b/docker/services/neutron-ovs-agent.yaml
index 1c9e60db..0a061f6c 100644
--- a/docker/services/neutron-ovs-agent.yaml
+++ b/docker/services/neutron-ovs-agent.yaml
@@ -43,6 +43,22 @@ outputs:
       config_settings: {get_attr: [NeutronOvsAgentBase, role_data, config_settings]}
       step_config: {get_attr: [NeutronOvsAgentBase, role_data, step_config]}
       puppet_tags: neutron_config,neutron_agent_ovs,neutron_plugin_ml2
+      kolla_config:
+        /var/lib/etc-data/json-config/neutron-openvswitch-agent.json:
+           command: /usr/bin/neutron-openvswitch-agent --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/openvswitch_agent.ini --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
+           config_files:
+           - dest: /etc/neutron/neutron.conf
+             owner: neutron
+             perm: '0600'
+             source: /var/lib/kolla/config_files/neutron.conf
+           - dest: /etc/neutron/plugins/ml2/openvswitch_agent.ini
+             owner: neutron
+             perm: '0600'
+             source: /var/lib/kolla/config_files/openvswitch_agent.ini
+           - dest: /etc/neutron/plugins/ml2/ml2_conf.ini
+             owner: neutron
+             perm: '0600'
+             source: /var/lib/kolla/config_files/ml2_conf.ini
       docker_config:
         step_1:
           neutronovsagent:
diff --git a/docker/services/nova-compute.yaml b/docker/services/nova-compute.yaml
index c695c94d..e765609e 100644
--- a/docker/services/nova-compute.yaml
+++ b/docker/services/nova-compute.yaml
@@ -41,6 +41,18 @@ outputs:
       config_settings: {get_attr: [NovaComputeBase, role_data, config_settings]}
       step_config: {get_attr: [NovaComputeBase, role_data, step_config]}
       puppet_tags: nova_config,nova_paste_api_ini
+      kolla_config:
+        /var/lib/etc-data/json-config/nova-compute.json:
+           command: /usr/bin/nova-compute --config-file /etc/nova/nova.conf --config-file /etc/nova/rootwrap.conf
+           config_files:
+           - dest: /etc/nova/nova.conf
+             owner: nova
+             perm: '0600'
+             source: /var/lib/kolla/config_files/nova.conf
+           - dest: /etc/nova/rootwrap.conf
+             owner: nova
+             perm: '0600'
+             source: /var/lib/kolla/config_files/rootwrap.conf
       docker_config:
         step_1:
           novacompute:
diff --git a/docker/services/nova-libvirt.yaml b/docker/services/nova-libvirt.yaml
index a40a21fd..004d624a 100644
--- a/docker/services/nova-libvirt.yaml
+++ b/docker/services/nova-libvirt.yaml
@@ -41,6 +41,14 @@ outputs:
       config_settings: {get_attr: [NovaLibvirtBase, role_data, config_settings]}
       step_config: {get_attr: [NovaLibvirtBase, role_data, step_config]}
       puppet_tags: nova_config
+      kolla_config:
+        /var/lib/etc-data/json-config/nova-libvirt.json:
+           command: /usr/sbin/libvirtd --config /etc/libvirt/libvirtd.conf
+           config_files:
+           - dest: /etc/libvirt/libvirtd.conf
+             owner: root
+             perm: '0644'
+             source: /var/lib/kolla/config_files/libvirtd.conf
       docker_config:
         step_1:
           nova_libvirt:
diff --git a/docker/services/services.yaml b/docker/services/services.yaml
index 3d51eb19..8c31107f 100644
--- a/docker/services/services.yaml
+++ b/docker/services/services.yaml
@@ -68,6 +68,8 @@ outputs:
       step_config:
         {get_attr: [PuppetServices, role_data, step_config]}
       puppet_tags: {list_join: [",", {get_attr: [ServiceChain, role_data, puppet_tags]}]}
+      kolla_config:
+        map_merge: {get_attr: [ServiceChain, role_data, kolla_config]}
       docker_config:
         step_1: {map_merge: {get_attr: [ServiceChain, role_data, docker_config, step_1]}}
         step_2: {map_merge: {get_attr: [ServiceChain, role_data, docker_config, step_2]}}