Remove hardcoded tempos in heat_ims

It adds loops to wait for services.
It sets dns forwarders to allow sending Heat signal at last
instructions.

Change-Id: I3ae29ddbaee5f8b2086078bd5245e140348f0dcf
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit ce305aa6d78ca62649d43b73b1d8f3b851584d81)

1 files changed, 126 insertions, 103 deletions

diff --git a/docker/vnf/clearwater-heat-singlenet-deps.patch b/docker/vnf/clearwater-heat-singlenet-deps.patch
index 15ff3e23f..a1bc3db7b 100644
--- a/docker/vnf/clearwater-heat-singlenet-deps.patch
+++ b/docker/vnf/clearwater-heat-singlenet-deps.patch
@@ -1,5 +1,5 @@
 diff --git a/bono.yaml b/bono.yaml
-index f0189cd..0a39057 100644
+index f0189cd..7566338 100644
 --- a/bono.yaml
 +++ b/bono.yaml
 @@ -23,26 +23,6 @@ parameters:
@@ -84,7 +84,7 @@ index f0189cd..0a39057 100644
        user_data_format: RAW
        user_data:
          str_replace:
-@@ -143,48 +112,18 @@ resources:
+@@ -143,15 +112,11 @@ resources:
              __zone__: { get_param: zone }
              __public_mgmt_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
              __private_mgmt_ip__: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -101,10 +101,10 @@ index f0189cd..0a39057 100644
            template: |
              #!/bin/bash
  
-             # Log all output to file.
+@@ -159,33 +124,6 @@ resources:
              exec > >(tee -a /var/log/clearwater-heat-bono.log) 2>&1
              set -x
--
+ 
 -            # Set up the signaling network namespace on each boot by creating an init file and
 -            # linking to it from runlevel 2 and 3
 -            cat >/etc/init.d/signaling_namespace <<EOF
@@ -131,11 +131,11 @@ index f0189cd..0a39057 100644
 -
 -            # Also set up the signaling namespace now.
 -            /etc/init.d/signaling_namespace
-+            sleep 180
- 
+-
              # Configure the APT software source.
              echo 'deb __repo_url__ binary/' > /etc/apt/sources.list.d/clearwater.list
-@@ -196,11 +135,8 @@ resources:
+             curl -L http://repo.cw-ngv.com/repo_key | apt-key add -
+@@ -196,11 +134,8 @@ resources:
              etcd_ip=__etcd_ip__
              [ -n "$etcd_ip" ] || etcd_ip=__private_mgmt_ip__
              cat > /etc/clearwater/local_config << EOF
@@ -149,7 +149,7 @@ index f0189cd..0a39057 100644
              public_hostname=__index__.bono.__zone__
              etcd_cluster=$etcd_ip
              EOF
-@@ -223,8 +159,8 @@ resources:
+@@ -223,8 +158,8 @@ resources:
              while ! { nsupdate -y "__zone__:__dnssec_key__" -v << EOF
              server __dns_mgmt_ip__
              update add bono-__index__.__zone__. 30 $(ip2rr __public_mgmt_ip__)
@@ -160,12 +160,7 @@ index f0189cd..0a39057 100644
              update add __zone__. 30 NAPTR 0 0 "s" "SIP+D2T" "" _sip._tcp.__zone__.
              update add __zone__. 30 NAPTR 0 0 "s" "SIP+D2U" "" _sip._udp.__zone__.
              update add _sip._tcp.__zone__. 30 SRV 0 0 5060 __index__.bono.__zone__.
-@@ -238,13 +174,20 @@ resources:
-               sleep 5
-             done
- 
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
+@@ -241,10 +176,19 @@ resources:
              # Use the DNS server.
              echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
@@ -173,17 +168,21 @@ index f0189cd..0a39057 100644
 -            echo 'nameserver __dns_sig_ip__' > /etc/netns/signaling/resolv.conf
              service dnsmasq force-reload
  
-+            sleep 60
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_cluster_state
 +            cw-check_config_sync
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public management network
-@@ -252,9 +195,3 @@ outputs:
+@@ -252,9 +196,3 @@ outputs:
    private_mgmt_ip:
      description: IP address in private management network
      value: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -194,7 +193,7 @@ index f0189cd..0a39057 100644
 -    description: IP address in private signaling network
 -    value: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] }
 diff --git a/clearwater.yaml b/clearwater.yaml
-index a155c60..1bfe257 100644
+index a155c60..dd90cb7 100644
 --- a/clearwater.yaml
 +++ b/clearwater.yaml
 @@ -37,41 +37,6 @@ parameters:
@@ -258,7 +257,7 @@ index a155c60..1bfe257 100644
    security_groups:
      type: ./security-groups.yaml
      properties:
-@@ -173,18 +126,16 @@ resources:
+@@ -173,18 +126,17 @@ resources:
      properties:
        public_mgmt_net_id: { get_param: public_mgmt_net_id }
        private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -272,6 +271,7 @@ index a155c60..1bfe257 100644
 +      base_mgmt_security_group: { get_attr: [ security_groups, base_mgmt ] }
        zone: { get_param: zone }
        dnssec_key: { get_param: dnssec_key }
++      dns_ip: { get_param: external_mgmt_dns_ip }
  
    ellis:
      type: ./ellis.yaml
@@ -279,7 +279,7 @@ index a155c60..1bfe257 100644
      properties:
        public_mgmt_net_id: { get_param: public_mgmt_net_id }
        private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
-@@ -192,7 +143,6 @@ resources:
+@@ -192,7 +144,6 @@ resources:
        image: { get_param: image }
        key_name: { get_param: key_name }
        base_mgmt_security_group: { get_attr: [ security_groups, base_mgmt ] }
@@ -287,7 +287,7 @@ index a155c60..1bfe257 100644
        repo_url: { get_param: repo_url }
        zone: { get_param: zone }
        dn_range_start: { get_param: dn_range_start }
-@@ -202,6 +152,7 @@ resources:
+@@ -202,6 +153,7 @@ resources:
  
    bono:
      type: OS::Heat::ResourceGroup
@@ -295,7 +295,7 @@ index a155c60..1bfe257 100644
      properties:
        count: { get_param: bono_cluster_size }
        index_var: "__index__"
-@@ -210,25 +161,20 @@ resources:
+@@ -210,25 +162,20 @@ resources:
          properties:
            public_mgmt_net_id: { get_param: public_mgmt_net_id }
            private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -322,7 +322,7 @@ index a155c60..1bfe257 100644
      properties:
        count: { get_param: sprout_cluster_size }
        index_var: __index__
-@@ -237,26 +183,20 @@ resources:
+@@ -237,26 +184,20 @@ resources:
          properties:
            public_mgmt_net_id: { get_param: public_mgmt_net_id }
            private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -350,7 +350,7 @@ index a155c60..1bfe257 100644
      properties:
        count: { get_param: homer_cluster_size }
        index_var: __index__
-@@ -265,26 +205,20 @@ resources:
+@@ -265,26 +206,20 @@ resources:
          properties:
            public_mgmt_net_id: { get_param: public_mgmt_net_id }
            private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -378,7 +378,7 @@ index a155c60..1bfe257 100644
      properties:
        count: { get_param: dime_cluster_size }
        index_var: __index__
-@@ -293,26 +227,20 @@ resources:
+@@ -293,26 +228,20 @@ resources:
          properties:
            public_mgmt_net_id: { get_param: public_mgmt_net_id }
            private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -406,7 +406,7 @@ index a155c60..1bfe257 100644
      properties:
        count: { get_param: vellum_cluster_size }
        index_var: __index__
-@@ -321,20 +249,13 @@ resources:
+@@ -321,20 +250,13 @@ resources:
          properties:
            public_mgmt_net_id: { get_param: public_mgmt_net_id }
            private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] }
@@ -428,7 +428,7 @@ index a155c60..1bfe257 100644
            etcd_ip: { get_attr: [ ellis, private_mgmt_ip ] }
            index: __index__
 diff --git a/dime.yaml b/dime.yaml
-index 642f19d..214a985 100644
+index 642f19d..6ae2676 100644
 --- a/dime.yaml
 +++ b/dime.yaml
 @@ -23,26 +23,6 @@ parameters:
@@ -519,7 +519,7 @@ index 642f19d..214a985 100644
        user_data_format: RAW
        user_data:
          str_replace:
-@@ -141,47 +112,18 @@ resources:
+@@ -141,14 +112,11 @@ resources:
              __zone__: { get_param: zone }
              __public_mgmt_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
              __private_mgmt_ip__: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -535,10 +535,10 @@ index 642f19d..214a985 100644
            template: |
              #!/bin/bash
  
-             # Log all output to file.
+@@ -156,33 +124,6 @@ resources:
              exec > >(tee -a /var/log/clearwater-heat-dime.log) 2>&1
              set -x
--
+ 
 -            # Set up the signaling network namespace on each boot by creating an init file and
 -            # linking to it from runlevel 2 and 3
 -            cat >/etc/init.d/signaling_namespace <<EOF
@@ -565,11 +565,11 @@ index 642f19d..214a985 100644
 -
 -            # Also set up the signaling namespace now.
 -            /etc/init.d/signaling_namespace
-+            sleep 180
- 
+-
              # Configure the APT software source.
              echo 'deb __repo_url__ binary/' > /etc/apt/sources.list.d/clearwater.list
-@@ -193,11 +135,8 @@ resources:
+             curl -L http://repo.cw-ngv.com/repo_key | apt-key add -
+@@ -193,11 +134,8 @@ resources:
              etcd_ip=__etcd_ip__
              [ -n "$etcd_ip" ] || etcd_ip=__private_mgmt_ip__
              cat > /etc/clearwater/local_config << EOF
@@ -583,7 +583,7 @@ index 642f19d..214a985 100644
              public_hostname=dime-__index__.__zone__
              etcd_cluster=$etcd_ip
              EOF
-@@ -220,9 +159,9 @@ resources:
+@@ -220,9 +158,9 @@ resources:
              while ! { nsupdate -y "__zone__:__dnssec_key__" -v << EOF
              server __dns_mgmt_ip__
              update add dime-__index__.__zone__. 30 $(ip2rr __public_mgmt_ip__)
@@ -596,12 +596,7 @@ index 642f19d..214a985 100644
              send
              EOF
              } && [ $retries -lt 10 ]
-@@ -232,20 +171,23 @@ resources:
-               sleep 5
-             done
- 
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
+@@ -235,17 +173,22 @@ resources:
              # Use the DNS server.
              echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
@@ -609,12 +604,16 @@ index 642f19d..214a985 100644
 -            echo 'nameserver __dns_sig_ip__' > /etc/netns/signaling/resolv.conf
              service dnsmasq force-reload
  
-+            sleep 180
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_config_sync
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public (management) network
@@ -627,7 +626,7 @@ index 642f19d..214a985 100644
 -    description: IP address in private signaling network
 -    value: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] }
 diff --git a/dns.yaml b/dns.yaml
-index 825ede1..11a833b 100644
+index 825ede1..f98ffa3 100644
 --- a/dns.yaml
 +++ b/dns.yaml
 @@ -23,22 +23,6 @@ parameters:
@@ -662,7 +661,18 @@ index 825ede1..11a833b 100644
      type: string
      description: ID of security group for DNS nodes
    zone:
-@@ -75,7 +59,7 @@ resources:
+@@ -67,6 +51,10 @@ parameters:
+   dnssec_key:
+     type: string
+     description: DNSSEC private key (Base64-encoded)
++  dns_ip:
++    type: string
++    description: IP address of DNS server
++    default: 8.8.8.8
+ 
+ resources:
+   mgmt_port:
+@@ -75,7 +63,7 @@ resources:
        # Specify the network ID by string to work around OpenStack issues - see https://github.com/Metaswitch/clearwater-heat/issues/18.
        network_id: { str_replace: { params: { x: { get_param: private_mgmt_net_id } }, template: x } }
        security_groups:
@@ -671,7 +681,7 @@ index 825ede1..11a833b 100644
  
    mgmt_floating_ip:
      type: OS::Neutron::FloatingIP
-@@ -83,13 +67,15 @@ resources:
+@@ -83,13 +71,15 @@ resources:
        floating_network_id: { get_param: public_mgmt_net_id }
        port_id: { get_resource: mgmt_port }
  
@@ -693,7 +703,7 @@ index 825ede1..11a833b 100644
  
    server:
      type: OS::Nova::Server
-@@ -100,7 +86,6 @@ resources:
+@@ -100,16 +90,15 @@ resources:
        key_name: { get_param: key_name }
        networks:
          - port: { get_resource: mgmt_port }
@@ -701,9 +711,10 @@ index 825ede1..11a833b 100644
        user_data_format: RAW
        user_data:
          str_replace:
-@@ -108,8 +93,7 @@ resources:
+           params:
              __zone__: { get_param: zone }
              __dnssec_key__: { get_param: dnssec_key }
++            __dns_ip__: { get_param: dns_ip }
              __public_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
 -            __private_sig_ip__: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] }
 -            __private_sig_cidr__: { get_param: private_sig_net_cidr }
@@ -711,7 +722,7 @@ index 825ede1..11a833b 100644
            template: |
              #!/bin/bash
  
-@@ -117,10 +101,6 @@ resources:
+@@ -117,14 +106,22 @@ resources:
              exec > >(tee -a /var/log/clearwater-heat-dns.log) 2>&1
              set -x
  
@@ -722,7 +733,23 @@ index 825ede1..11a833b 100644
              # Install BIND.
              apt-get update
              DEBIAN_FRONTEND=noninteractive apt-get install bind9 --yes
-@@ -163,6 +143,8 @@ resources:
+ 
++            cat > /etc/bind/named.conf.options << EOF
++            options {
++              directory "/var/cache/bind";
++              forwarders {
++                __dns_ip__;
++              };
++              dnssec-validation auto;
++              auth-nxdomain no;
++              listen-on-v6 { any; };
++            };
++            EOF
++
+             # Update BIND configuration with the specified zone and key.
+             cat >> /etc/bind/named.conf.local << EOF
+             key __zone__. {
+@@ -163,6 +160,8 @@ resources:
              # Now that BIND configuration is correct, kick it to reload.
              service bind9 reload
  
@@ -731,7 +758,7 @@ index 825ede1..11a833b 100644
  outputs:
    public_mgmt_ip:
      description: IP address in public management network
-@@ -170,9 +152,6 @@ outputs:
+@@ -170,9 +169,6 @@ outputs:
    private_mgmt_ip:
      description: IP address in private signaling network
      value: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -742,7 +769,7 @@ index 825ede1..11a833b 100644
      description: DNS zone
      value: { get_param: zone }
 diff --git a/ellis.yaml b/ellis.yaml
-index 963352d..c9fbb46 100644
+index 963352d..ff59895 100644
 --- a/ellis.yaml
 +++ b/ellis.yaml
 @@ -44,9 +44,6 @@ parameters:
@@ -814,30 +841,27 @@ index 963352d..c9fbb46 100644
  
              # Function to give DNS record type and IP address for specified IP address
              ip2rr() {
-@@ -223,11 +231,21 @@ resources:
-               sleep 5
-             done
- 
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
-             # Use the DNS server.
-             echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
+@@ -228,6 +236,18 @@ resources:
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
              service dnsmasq force-reload
  
-+            sleep 180
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_config_sync
 +
 +            /usr/share/clearwater/ellis/env/bin/python /usr/share/clearwater/ellis/src/metaswitch/ellis/tools/create_numbers.py --start __dn_range_start__ --count __dn_range_length__ --realm __zone__
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public management network
 diff --git a/homer.yaml b/homer.yaml
-index 4337984..0640d4b 100644
+index 4337984..9a93cfb 100644
 --- a/homer.yaml
 +++ b/homer.yaml
 @@ -23,26 +23,6 @@ parameters:
@@ -933,7 +957,7 @@ index 4337984..0640d4b 100644
        user_data_format: RAW
        user_data:
          str_replace:
-@@ -147,48 +112,18 @@ resources:
+@@ -147,15 +112,11 @@ resources:
              __zone__: { get_param: zone }
              __public_mgmt_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
              __private_mgmt_ip__: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -950,10 +974,10 @@ index 4337984..0640d4b 100644
            template: |
              #!/bin/bash
  
-             # Log all output to file.
+@@ -163,33 +124,6 @@ resources:
              exec > >(tee -a /var/log/clearwater-heat-homer.log) 2>&1
              set -x
--
+ 
 -            # Set up the signaling network namespace on each boot by creating an init file and
 -            # linking to it from runlevel 2 and 3
 -            cat >/etc/init.d/signaling_namespace <<EOF
@@ -980,11 +1004,11 @@ index 4337984..0640d4b 100644
 -
 -            # Also set up the signaling namespace now.
 -            /etc/init.d/signaling_namespace
-+            sleep 180
- 
+-
              # Configure the APT software source.
              echo 'deb __repo_url__ binary/' > /etc/apt/sources.list.d/clearwater.list
-@@ -200,11 +135,8 @@ resources:
+             curl -L http://repo.cw-ngv.com/repo_key | apt-key add -
+@@ -200,11 +134,8 @@ resources:
              etcd_ip=__etcd_ip__
              [ -n "$etcd_ip" ] || etcd_ip=__private_mgmt_ip__
              cat > /etc/clearwater/local_config << EOF
@@ -998,7 +1022,7 @@ index 4337984..0640d4b 100644
              public_hostname=homer-__index__.__zone__
              etcd_cluster=$etcd_ip
              EOF
-@@ -227,7 +159,7 @@ resources:
+@@ -227,7 +158,7 @@ resources:
              while ! { nsupdate -y "__zone__:__dnssec_key__" -v << EOF
              server __dns_mgmt_ip__
              update add homer-__index__.__zone__. 30 $(ip2rr __public_mgmt_ip__)
@@ -1007,31 +1031,32 @@ index 4337984..0640d4b 100644
              send
              EOF
              } && [ $retries -lt 10 ]
-@@ -237,14 +169,20 @@ resources:
-               sleep 5
+@@ -238,13 +169,21 @@ resources:
              done
  
--            # Use the DNS server.
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
              # Use the DNS server.
+-            # Use the DNS server.
              echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
 -            mkdir -p /etc/netns/signaling
 -            echo 'nameserver __dns_sig_ip__' > /etc/netns/signaling/resolv.conf
              service dnsmasq force-reload
  
-+            sleep 180
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_cluster_state
 +            cw-check_config_sync
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public management network
-@@ -252,9 +190,3 @@ outputs:
+@@ -252,9 +191,3 @@ outputs:
    private_mgmt_ip:
      description: IP address in private management network
      value: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -1391,7 +1416,7 @@ index 5921d32..c73fe2b 100644
 -    value: { get_resource: vellum_sig_inbound }
 +    value: { get_resource: base_mgmt }
 diff --git a/sprout.yaml b/sprout.yaml
-index 9c533b7..4a5d4b6 100644
+index 9c533b7..2649e11 100644
 --- a/sprout.yaml
 +++ b/sprout.yaml
 @@ -23,26 +23,6 @@ parameters:
@@ -1475,7 +1500,7 @@ index 9c533b7..4a5d4b6 100644
        user_data_format: RAW
        user_data:
          str_replace:
-@@ -141,47 +112,18 @@ resources:
+@@ -141,14 +112,11 @@ resources:
              __zone__: { get_param: zone }
              __public_mgmt_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] }
              __private_mgmt_ip__: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }
@@ -1491,10 +1516,10 @@ index 9c533b7..4a5d4b6 100644
            template: |
              #!/bin/bash
  
-             # Log all output to file.
+@@ -156,33 +124,6 @@ resources:
              exec > >(tee -a /var/log/clearwater-heat-sprout.log) 2>&1
              set -x
--
+ 
 -            # Set up the signaling network namespace on each boot by creating an init file and
 -            # linking to it from runlevel 2 and 3
 -            cat >/etc/init.d/signaling_namespace <<EOF
@@ -1521,11 +1546,11 @@ index 9c533b7..4a5d4b6 100644
 -
 -            # Also set up the signaling namespace now.
 -            /etc/init.d/signaling_namespace
-+            sleep 300
- 
+-
              # Configure the APT software source.
              echo 'deb __repo_url__ binary/' > /etc/apt/sources.list.d/clearwater.list
-@@ -193,11 +135,8 @@ resources:
+             curl -L http://repo.cw-ngv.com/repo_key | apt-key add -
+@@ -193,11 +134,8 @@ resources:
              etcd_ip=__etcd_ip__
              [ -n "$etcd_ip" ] || etcd_ip=__private_mgmt_ip__
              cat > /etc/clearwater/local_config << EOF
@@ -1539,7 +1564,7 @@ index 9c533b7..4a5d4b6 100644
              public_hostname=__index__.sprout.__zone__
              etcd_cluster=$etcd_ip
              EOF
-@@ -220,10 +159,10 @@ resources:
+@@ -220,10 +158,10 @@ resources:
              while ! { nsupdate -y "__zone__:__dnssec_key__" -v << EOF
              server __dns_mgmt_ip__
              update add sprout-__index__.__zone__. 30 $(ip2rr __public_mgmt_ip__)
@@ -1554,12 +1579,7 @@ index 9c533b7..4a5d4b6 100644
              update add sprout.__zone__. 30 NAPTR 0 0 "s" "SIP+D2T" "" _sip._tcp.sprout.__zone__.
              update add _sip._tcp.sprout.__zone__. 30 SRV 0 0 5054 __index__.sprout.__zone__.
              update add icscf.sprout.__zone__. 30 NAPTR 0 0 "s" "SIP+D2T" "" _sip._tcp.icscf.sprout.__zone__.
-@@ -239,20 +178,24 @@ resources:
-               sleep 5
-             done
- 
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
+@@ -242,17 +180,23 @@ resources:
              # Use the DNS server.
              echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
@@ -1567,13 +1587,17 @@ index 9c533b7..4a5d4b6 100644
 -            echo 'nameserver __dns_sig_ip__' > /etc/netns/signaling/resolv.conf
              service dnsmasq force-reload
  
-+            sleep 180
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_cluster_state
 +            cw-check_config_sync
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public (management) network
@@ -1586,7 +1610,7 @@ index 9c533b7..4a5d4b6 100644
 -    description: IP address in private signaling network
 -    value: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] }
 diff --git a/vellum.yaml b/vellum.yaml
-index aab71f9..c2ca279 100644
+index aab71f9..375b3d8 100644
 --- a/vellum.yaml
 +++ b/vellum.yaml
 @@ -23,26 +23,6 @@ parameters:
@@ -1762,12 +1786,7 @@ index aab71f9..c2ca279 100644
              send
              EOF
              } && [ $retries -lt 10 ]
-@@ -249,13 +189,20 @@ resources:
-               sleep 5
-             done
- 
-+            wc_notify --data-binary '{"status": "SUCCESS"}'
-+
+@@ -252,10 +192,19 @@ resources:
              # Use the DNS server.
              echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf
              echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq
@@ -1775,17 +1794,21 @@ index aab71f9..c2ca279 100644
 -            echo 'nameserver __dns_sig_ip__' > /etc/netns/signaling/resolv.conf
              service dnsmasq force-reload
  
-+            sleep 300
-+            monit summary
++            for ((i=1;i<=10;i++)); do
++                monit summary
++                test -z "$(monit summary 2>&1 |sed '1,2d' |grep -v Running |grep -v Status\ ok )" && break || sleep 60
++            done
 +            clearwater-etcdctl cluster-health
 +            clearwater-etcdctl member list
 +            cw-check_cluster_state
 +            cw-check_config_sync
 +
++            wc_notify --data-binary '{"status": "SUCCESS"}'
++
  outputs:
    public_mgmt_ip:
      description: IP address in public management network
-@@ -263,6 +210,3 @@ outputs:
+@@ -263,6 +212,3 @@ outputs:
    private_mgmt_ip:
      description: IP address in private management network
      value: { get_attr: [ mgmt_port, fixed_ips, 0, ip_address ] }