Adds Undercloud retry and log fetch

Retry to install the undercloud if it fails due to sometimes failing.
Also copy undercloud log over to host temp dir.

Change-Id: I50f98c4f0d52fa6bb56d9dcd0766bbe745160452
Signed-off-by: Tim Rozet <trozet@redhat.com>

2 files changed, 31 insertions, 7 deletions

diff --git a/apex/undercloud/undercloud.py b/apex/undercloud/undercloud.py
index a1af4d00..da695f5b 100644
--- a/apex/undercloud/undercloud.py
+++ b/apex/undercloud/undercloud.py
@@ -11,6 +11,7 @@ import libvirt
 import logging
 import os
 import shutil
+import subprocess
 import time
 
 from apex.virtual import virtual_utils as virt_utils
@@ -107,14 +108,25 @@ class Undercloud:
     def configure(self, net_settings, playbook, apex_temp_dir):
         """
         Configures undercloud VM
-        :return:
+        :param net_setings: Network settings for deployment
+        :param playbook: playbook to use to configure undercloud
+        :param apex_temp_dir: temporary apex directory to hold configs/logs
+        :return: None
         """
-        # TODO(trozet): If undercloud install fails we can add a retry
+
         logging.info("Configuring Undercloud...")
         # run ansible
         ansible_vars = Undercloud.generate_config(net_settings)
         ansible_vars['apex_temp_dir'] = apex_temp_dir
-        utils.run_ansible(ansible_vars, playbook, host=self.ip, user='stack')
+        try:
+            utils.run_ansible(ansible_vars, playbook, host=self.ip,
+                              user='stack')
+        except subprocess.CalledProcessError:
+            logging.error(
+                "Failed to install undercloud..."
+                "please check log: {}".format(os.path.join(
+                    apex_temp_dir, 'apex-undercloud-install.log')))
+            raise ApexUndercloudException('Failed to install undercloud')
         logging.info("Undercloud installed!")
 
     def setup_volumes(self):
diff --git a/lib/ansible/playbooks/configure_undercloud.yml b/lib/ansible/playbooks/configure_undercloud.yml
index aa6c960a..f19218c3 100644
--- a/lib/ansible/playbooks/configure_undercloud.yml
+++ b/lib/ansible/playbooks/configure_undercloud.yml
@@ -47,10 +47,22 @@
         regexp: '_link_ip_address_pxe_configs'
         line: '_link_mac_pxe_configs(task)'
       when: aarch64
-    - name: undercloud install
-      shell: openstack undercloud install &> apex-undercloud-install.log
-      become: yes
-      become_user: stack
+    - block:
+        - name: undercloud install
+          shell: openstack undercloud install &> apex-undercloud-install.log
+          become: yes
+          become_user: stack
+      rescue:
+        - name: undercloud install retry
+          shell: openstack undercloud install >> apex-undercloud-install.log 2>&1
+          become: yes
+          become_user: stack
+      always:
+        - name: fetch undercloud log
+          fetch:
+            src: /home/stack/apex-undercloud-install.log
+            dest: "{{ apex_temp_dir }}/"
+            flat: yes
     - name: openstack-configs nova
       shell: openstack-config --set /etc/nova/nova.conf DEFAULT {{ item }}
       become: yes