Remove downloading undercloud.qcow2

OOO team is removing the undercloud disk image as it is no longer needed
for containerized undercloud deployments. Instead, we can just use the
overcloud image as the undercloud image.

Additionally, OOO team has recommended we use current-tripleo instead of
current-tripleo-rdo. current-tripleo-rdo was previously thought to be
more stable with more promotion checks, but now it seems that it is
older and current-tripleo now has the same stability/checks.

This patch also bumps the undercloud RAM from 8GB to 10GB. With the new
containerized undercloud there is more RAM consumption during
deployment.

Change-Id: I9e6bb2260dbe9f8796ee54d20527c0aad96476ec
Signed-off-by: Tim Rozet <trozet@redhat.com>

7 files changed, 111 insertions, 12 deletions

diff --git a/apex/builders/undercloud_builder.py b/apex/builders/undercloud_builder.py
index 4efd00d5..ec75f3dc 100644
--- a/apex/builders/undercloud_builder.py
+++ b/apex/builders/undercloud_builder.py
@@ -9,7 +9,9 @@
 
 # Used to modify undercloud qcow2 image
 import logging
+import json
 import os
+import subprocess
 
 from apex.common import constants as con
 from apex.common import utils
@@ -34,7 +36,8 @@ def add_upstream_packages(image):
         'openstack-tripleo-validations',
         'libguestfs-tools',
         'ceph-ansible',
-        'python-tripleoclient'
+        'python-tripleoclient',
+        'openstack-tripleo-heat-templates'
     ]
 
     for pkg in pkgs:
@@ -59,3 +62,45 @@ def inject_calipso_installer(tmp_dir, image):
 
 # TODO(trozet): add unit testing for calipso injector
 # TODO(trozet): add rest of build for undercloud here as well
+
+
+def update_repos(image, branch):
+    virt_ops = [
+        {con.VIRT_RUN_CMD: "rm -f /etc/yum.repos.d/delorean*"},
+        {con.VIRT_RUN_CMD: "yum-config-manager --add-repo "
+                           "https://trunk.rdoproject.org/centos7/{}"
+                           "/delorean.repo".format(con.RDO_TAG)},
+        {con.VIRT_INSTALL: "python2-tripleo-repos"},
+        {con.VIRT_RUN_CMD: "tripleo-repos -b {} {} ceph".format(branch,
+                                                                con.RDO_TAG)}
+    ]
+    virt_utils.virt_customize(virt_ops, image)
+
+
+def expand_disk(image, desired_size=50):
+    """
+    Expands a disk image to desired_size in GigaBytes
+    :param image: image to resize
+    :param desired_size: desired size in GB
+    :return: None
+    """
+    # there is a lib called vminspect which has some dependencies and is
+    # not yet available in pip. Consider switching to this lib later.
+    try:
+        img_out = json.loads(subprocess.check_output(
+            ['qemu-img', 'info', '--output=json', image],
+            stderr=subprocess.STDOUT).decode())
+        disk_gb_size = int(img_out['virtual-size'] / 1000000000)
+        if disk_gb_size < desired_size:
+            logging.info("Expanding disk image: {}. Current size: {} is less"
+                         "than require size: {}".format(image, disk_gb_size,
+                                                        desired_size))
+            diff_size = desired_size - disk_gb_size
+            subprocess.check_call(['qemu-img', 'resize', image,
+                                   "+{}G".format(diff_size)],
+                                  stderr=subprocess.STDOUT)
+
+    except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError) \
+            as e:
+        logging.warning("Unable to resize disk, disk may not be large "
+                        "enough: {}".format(e))
diff --git a/apex/common/constants.py b/apex/common/constants.py
index e0afbc0a..e89e7e75 100644
--- a/apex/common/constants.py
+++ b/apex/common/constants.py
@@ -49,11 +49,12 @@ PUPPET_ODL_URL = 'https://git.opendaylight.org/gerrit/integration/packaging' \
 DEBUG_OVERCLOUD_PW = 'opnfvapex'
 NET_ENV_FILE = 'network-environment.yaml'
 DEPLOY_TIMEOUT = 120
-UPSTREAM_RDO = 'https://images.rdoproject.org/master/rdo_trunk/current' \
-               '-tripleo-rdo/'
+RDO_TAG = 'current-tripleo'
+UPSTREAM_RDO = "https://images.rdoproject.org/master/rdo_trunk/{}/".format(
+    RDO_TAG)
 OPENSTACK_GERRIT = 'https://review.openstack.org'
 
-DOCKER_TAG = 'current-tripleo-rdo'
+DOCKER_TAG = RDO_TAG
 # Maps regular service files to docker versions
 # None value means mapping is same as key
 VALID_DOCKER_SERVICES = {
diff --git a/apex/deploy.py b/apex/deploy.py
index 8330dc80..b74d5292 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -44,6 +44,12 @@ from apex.overcloud import deploy as oc_deploy
 
 APEX_TEMP_DIR = tempfile.mkdtemp(prefix='apex_tmp')
 SDN_IMAGE = 'overcloud-full-opendaylight.qcow2'
+UC_DISK_FILES = [
+    'overcloud-full.vmlinuz',
+    'overcloud-full.initrd',
+    'ironic-python-agent.initramfs',
+    'ironic-python-agent.kernel'
+]
 
 
 def validate_cross_settings(deploy_settings, net_settings, inventory):
@@ -377,16 +383,26 @@ def main():
         args.image_dir = os.path.join(args.image_dir, os_version)
         upstream_url = constants.UPSTREAM_RDO.replace(
             constants.DEFAULT_OS_VERSION, os_version)
-        upstream_targets = ['overcloud-full.tar', 'undercloud.qcow2']
+        upstream_targets = ['overcloud-full.tar', 'ironic-python-agent.tar']
         utils.fetch_upstream_and_unpack(args.image_dir, upstream_url,
                                         upstream_targets,
                                         fetch=not args.no_fetch)
+        # Copy ironic files and overcloud ramdisk and kernel into temp dir
+        # to be copied by ansible into undercloud /home/stack
+        # Note the overcloud disk does not need to be copied here as it will
+        # be modified and copied later
+        for tmp_file in UC_DISK_FILES:
+            shutil.copyfile(os.path.join(args.image_dir, tmp_file),
+                            os.path.join(APEX_TEMP_DIR, tmp_file))
         sdn_image = os.path.join(args.image_dir, 'overcloud-full.qcow2')
         # copy undercloud so we don't taint upstream fetch
         uc_image = os.path.join(args.image_dir, 'undercloud_mod.qcow2')
-        uc_fetch_img = os.path.join(args.image_dir, 'undercloud.qcow2')
+        uc_fetch_img = sdn_image
         shutil.copyfile(uc_fetch_img, uc_image)
         # prep undercloud with required packages
+        if platform.machine() != 'aarch64':
+            uc_builder.update_repos(image=uc_image,
+                                    branch=branch.replace('stable/', ''))
         uc_builder.add_upstream_packages(uc_image)
         uc_builder.inject_calipso_installer(APEX_TEMP_DIR, uc_image)
         # add patches from upstream to undercloud and overcloud
@@ -490,6 +506,8 @@ def main():
             except Exception:
                 logging.error("Unable to complete container prep on "
                               "Undercloud")
+                for tmp_file in UC_DISK_FILES:
+                    os.remove(os.path.join(APEX_TEMP_DIR, tmp_file))
                 os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
                 raise
 
@@ -537,6 +555,8 @@ def main():
             raise
         finally:
             os.remove(os.path.join(APEX_TEMP_DIR, 'overcloud-full.qcow2'))
+            for tmp_file in UC_DISK_FILES:
+                os.remove(os.path.join(APEX_TEMP_DIR, tmp_file))
 
         # Post install
         logging.info("Executing post deploy configuration")
diff --git a/apex/tests/test_apex_undercloud.py b/apex/tests/test_apex_undercloud.py
index 2d0dffcb..14586528 100644
--- a/apex/tests/test_apex_undercloud.py
+++ b/apex/tests/test_apex_undercloud.py
@@ -10,6 +10,7 @@
 import ipaddress
 import libvirt
 import os
+import platform
 import subprocess
 import unittest
 
@@ -239,13 +240,16 @@ class TestUndercloud(unittest.TestCase):
         assert_raises(ApexUndercloudException,
                       uc.configure, ns, ds, 'playbook', '/tmp/dir')
 
+    @patch('apex.undercloud.undercloud.virt_utils')
+    @patch('apex.undercloud.undercloud.uc_builder')
     @patch('apex.undercloud.undercloud.os.remove')
     @patch('apex.undercloud.undercloud.os.path')
     @patch('apex.undercloud.undercloud.shutil')
     @patch.object(Undercloud, '_get_vm', return_value=None)
     @patch.object(Undercloud, 'create')
     def test_setup_vols(self, mock_get_vm, mock_create,
-                        mock_shutil, mock_os_path, mock_os_remove):
+                        mock_shutil, mock_os_path, mock_os_remove,
+                        mock_uc_builder, mock_virt_utils):
         uc = Undercloud('img_path', 'tplt_path', external_network=True)
         mock_os_path.isfile.return_value = True
         mock_os_path.exists.return_value = True
@@ -255,6 +259,9 @@ class TestUndercloud(unittest.TestCase):
             src_img = os.path.join(uc.image_path, img_file)
             dest_img = os.path.join(constants.LIBVIRT_VOLUME_PATH, img_file)
             mock_shutil.copyfile.assert_called_with(src_img, dest_img)
+        if platform.machine() != 'aarch64':
+            mock_uc_builder.expand_disk.assert_called()
+            mock_virt_utils.virt_customize.assert_called()
 
     @patch('apex.undercloud.undercloud.os.path')
     @patch.object(Undercloud, '_get_vm', return_value=None)
@@ -278,12 +285,19 @@ class TestUndercloud(unittest.TestCase):
                     {'--run-command': 'chmod 600 /root/.ssh/authorized_keys'},
                     {'--run-command': 'restorecon '
                                       '-R -v /root/.ssh'},
+                    {'--run-command': 'id -u stack || useradd -m stack'},
+                    {'--run-command': 'mkdir -p /home/stack/.ssh'},
+                    {'--run-command': 'chown stack:stack /home/stack/.ssh'},
                     {'--run-command':
                      'cp /root/.ssh/authorized_keys /home/stack/.ssh/'},
                     {'--run-command':
                      'chown stack:stack /home/stack/.ssh/authorized_keys'},
                     {'--run-command':
-                     'chmod 600 /home/stack/.ssh/authorized_keys'}]
+                     'chmod 600 /home/stack/.ssh/authorized_keys'},
+                    {'--run-command':
+                     'echo "stack       ALL = (ALL) NOPASSWD: ALL" >> '
+                     '/etc/sudoers'},
+                    {'--run-command': 'touch /etc/cloud/cloud-init.disabled'}]
         mock_vutils.virt_customize.assert_called_with(test_ops, uc.volume)
 
     @patch.object(Undercloud, '_get_vm', return_value=None)
diff --git a/apex/undercloud/undercloud.py b/apex/undercloud/undercloud.py
index 8b6b9d4c..feae43c3 100644
--- a/apex/undercloud/undercloud.py
+++ b/apex/undercloud/undercloud.py
@@ -15,6 +15,7 @@ import shutil
 import subprocess
 import time
 
+from apex.builders import undercloud_builder as uc_builder
 from apex.virtual import utils as virt_utils
 from apex.virtual import configure_vm as vm_lib
 from apex.common import constants
@@ -72,7 +73,8 @@ class Undercloud:
                                    kernel_args=['console={}'.format(console),
                                                 'root=/dev/{}'.format(root)],
                                    default_network=True,
-                                   template_dir=self.template_path)
+                                   template_dir=self.template_path,
+                                   memory=10240)
         self.setup_volumes()
         self.inject_auth()
 
@@ -180,11 +182,19 @@ class Undercloud:
             if os.path.exists(dest_img):
                 os.remove(dest_img)
             shutil.copyfile(src_img, dest_img)
+            if img_file == self.image_name and platform.machine() != 'aarch64':
+                uc_builder.expand_disk(dest_img)
+                self.expand_root_fs()
+
             shutil.chown(dest_img, user='qemu', group='qemu')
             os.chmod(dest_img, 0o0744)
-        # TODO(trozet):check if resize needed right now size is 50gb
+
+    def expand_root_fs(self):
         # there is a lib called vminspect which has some dependencies and is
         # not yet available in pip.  Consider switching to this lib later.
+        logging.debug("Expanding root filesystem on /dev/sda partition")
+        virt_ops = [{constants.VIRT_RUN_CMD: 'xfs_growfs /dev/sda'}]
+        virt_utils.virt_customize(virt_ops, self.volume)
 
     def inject_auth(self):
         virt_ops = list()
@@ -200,9 +210,14 @@ class Undercloud:
         run_cmds = [
             'chmod 600 /root/.ssh/authorized_keys',
             'restorecon -R -v /root/.ssh',
+            'id -u stack || useradd -m stack',
+            'mkdir -p /home/stack/.ssh',
+            'chown stack:stack /home/stack/.ssh',
             'cp /root/.ssh/authorized_keys /home/stack/.ssh/',
             'chown stack:stack /home/stack/.ssh/authorized_keys',
-            'chmod 600 /home/stack/.ssh/authorized_keys'
+            'chmod 600 /home/stack/.ssh/authorized_keys',
+            'echo "stack       ALL = (ALL) NOPASSWD: ALL" >> /etc/sudoers',
+            'touch /etc/cloud/cloud-init.disabled'
         ]
         for cmd in run_cmds:
             virt_ops.append({constants.VIRT_RUN_CMD: cmd})
diff --git a/build/containers-prepare-parameter.yaml b/build/containers-prepare-parameter.yaml
index d6cfd10a..5089c335 100644
--- a/build/containers-prepare-parameter.yaml
+++ b/build/containers-prepare-parameter.yaml
@@ -10,7 +10,7 @@ parameter_defaults:
         name_suffix: ''
         namespace: docker.io/tripleomaster
         neutron_driver: null
-        tag: current-tripleo-rdo
+        tag: current-tripleo
       excludes:
         - sensu
         - manila
diff --git a/lib/ansible/playbooks/deploy_overcloud.yml b/lib/ansible/playbooks/deploy_overcloud.yml
index 39fbf52e..b8fb4938 100644
--- a/lib/ansible/playbooks/deploy_overcloud.yml
+++ b/lib/ansible/playbooks/deploy_overcloud.yml
@@ -12,6 +12,10 @@
         - network-environment.yaml
         - instackenv.json
         - overcloud-full.qcow2
+        - overcloud-full.vmlinuz
+        - overcloud-full.initrd
+        - ironic-python-agent.initramfs
+        - ironic-python-agent.kernel
         - deploy_command
         - virtual-environment.yaml
         - baremetal-environment.yaml