Leverage cloudify_ims on Cloudify

It also defines 2 new methods to avoid duplicating code:
  - TenantNetwork1.get_public_auth_url()
  - VmReady1.publish_image_alt()

Shaker and Tempest are updated as well.

Change-Id: Ie9095c57c580b95f067d5a8275ad71818b4df335
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

2 files changed, 94 insertions, 286 deletions

diff --git a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
index add99468b..34d3c628d 100644
--- a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
+++ b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
@@ -20,7 +20,6 @@ import time
 import pkg_resources
 import requests
 
-import functest.core.vnf as vnf
 from functest.utils import config
 import functest.utils.functest_utils as ft_utils
 
@@ -28,17 +27,16 @@ __author__ = ("Valentin Boucher <valentin.boucher@orange.com>, "
               "Helen Yao <helanyao@gmail.com>")
 
 
-class ClearwaterOnBoardingBase(vnf.VnfOnBoarding):
+class ClearwaterOnBoardingBase(object):
     """vIMS clearwater base usable by several orchestrators"""
 
-    def __init__(self, **kwargs):
+    def __init__(self, case_name):
         self.logger = logging.getLogger(__name__)
-        super(ClearwaterOnBoardingBase, self).__init__(**kwargs)
         self.case_dir = pkg_resources.resource_filename(
             'functest', 'opnfv_tests/vnf/ims')
         self.data_dir = getattr(config.CONF, 'dir_ims_data')
-        self.result_dir = os.path.join(getattr(config.CONF, 'dir_results'),
-                                       self.case_name)
+        self.result_dir = os.path.join(
+            getattr(config.CONF, 'dir_results'), case_name)
         self.test_dir = getattr(config.CONF, 'dir_repo_vims_test')
 
         if not os.path.exists(self.data_dir):
diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
index 786c535ed..91de92405 100644
--- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py
+++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
@@ -14,46 +14,36 @@ from __future__ import division
 import logging
 import os
 import time
-import uuid
+import yaml
 
-from cloudify_rest_client import CloudifyClient
 from cloudify_rest_client.executions import Execution
-from scp import SCPClient
+import pkg_resources
+import scp
 import six
-from snaps.config.flavor import FlavorConfig
-from snaps.config.image import ImageConfig
-from snaps.config.keypair import KeypairConfig
-from snaps.config.network import NetworkConfig, PortConfig, SubnetConfig
-from snaps.config.router import RouterConfig
-from snaps.config.security_group import (
-    Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
-from snaps.config.user import UserConfig
-from snaps.config.vm_inst import FloatingIpConfig, VmInstanceConfig
-from snaps.openstack.create_flavor import OpenStackFlavor
-from snaps.openstack.create_image import OpenStackImage
-from snaps.openstack.create_instance import OpenStackVmInstance
-from snaps.openstack.create_keypairs import OpenStackKeypair
-from snaps.openstack.create_network import OpenStackNetwork
-from snaps.openstack.create_router import OpenStackRouter
-from snaps.openstack.create_security_group import OpenStackSecurityGroup
-from snaps.openstack.create_user import OpenStackUser
-from snaps.openstack.utils import keystone_utils
-from xtesting.energy import energy
-import yaml
 
-from functest.opnfv_tests.openstack.snaps import snaps_utils
+from functest.core import cloudify
 from functest.opnfv_tests.vnf.ims import clearwater_ims_base
 from functest.utils import config
-from functest.utils import env
 
 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
 
 
-class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
+class CloudifyIms(cloudify.Cloudify):
     """Clearwater vIMS deployed with Cloudify Orchestrator Case."""
 
     __logger = logging.getLogger(__name__)
 
+    filename_alt = ('/home/opnfv/functest/images/'
+                    'ubuntu-14.04-server-cloudimg-amd64-disk1.img')
+
+    flavor_alt_ram = 2048
+    flavor_alt_vcpus = 2
+    flavor_alt_disk = 25
+
+    quota_security_group = 20
+    quota_security_group_rule = 100
+    quota_port = 50
+
     def __init__(self, **kwargs):
         """Initialize CloudifyIms testcase object."""
         if "case_name" not in kwargs:
@@ -67,6 +57,8 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         except Exception:
             raise Exception("VNF config file not found")
 
+        self.case_dir = pkg_resources.resource_filename(
+            'functest', 'opnfv_tests/vnf/ims')
         config_file = os.path.join(self.case_dir, self.config)
         self.orchestrator = dict(
             requirements=get_config("orchestrator.requirements", config_file),
@@ -94,30 +86,12 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             name=get_config("vnf_test_suite.name", config_file),
             version=get_config("vnf_test_suite.version", config_file)
         )
-        self.images = get_config("tenant_images", config_file)
-        self.__logger.info("Images needed for vIMS: %s", self.images)
-
-    def prepare(self):
-        """Prepare testscase (Additional pre-configuration steps)."""
-        super(CloudifyIms, self).prepare()
 
-        self.__logger.info("Additional pre-configuration steps")
+        self.image_alt = None
+        self.flavor_alt = None
 
-        compute_quotas = self.os_project.get_compute_quotas()
-        network_quotas = self.os_project.get_network_quotas()
-
-        for key, value in (
-                self.vnf['requirements']['compute_quotas'].items()):
-            setattr(compute_quotas, key, value)
-
-        for key, value in (
-                self.vnf['requirements']['network_quotas'].items()):
-            setattr(network_quotas, key, value)
-
-        compute_quotas = self.os_project.update_compute_quotas(compute_quotas)
-        network_quotas = self.os_project.update_network_quotas(network_quotas)
-
-    def deploy_orchestrator(self):
+    def execute(self):
+        assert super(CloudifyIms, self).execute() == 0
         # pylint: disable=too-many-locals,too-many-statements
         """
         Deploy Cloudify Manager.
@@ -125,241 +99,92 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         network, security group, fip, VM creation
         """
         start_time = time.time()
-
-        # orchestrator VM flavor
-        self.__logger.info("Get or create flavor for cloudify manager vm ...")
-        flavor_settings = FlavorConfig(
-            name="{}-{}".format(
-                self.orchestrator['requirements']['flavor']['name'],
-                self.uuid),
-            ram=self.orchestrator['requirements']['flavor']['ram_min'],
-            disk=50,
-            vcpus=2)
-        flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
-        flavor_creator.create()
-        self.created_object.append(flavor_creator)
-
-        self.__logger.info("Creating a second user to bypass issues ...")
-        user_creator = OpenStackUser(
-            self.snaps_creds,
-            UserConfig(
-                name='cloudify_network_bug-{}'.format(self.uuid),
-                password=str(uuid.uuid4()),
-                project_name=self.tenant_name,
-                domain_name=self.snaps_creds.user_domain_name,
-                roles={'_member_': self.tenant_name}))
-        user_creator.create()
-        self.created_object.append(user_creator)
-
-        snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
-        self.__logger.debug("snaps creds: %s", snaps_creds)
-
-        self.__logger.info("Creating keypair ...")
-        kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
-        keypair_settings = KeypairConfig(
-            name='cloudify_ims_kp-{}'.format(self.uuid),
-            private_filepath=kp_file)
-        keypair_creator = OpenStackKeypair(snaps_creds, keypair_settings)
-        keypair_creator.create()
-        self.created_object.append(keypair_creator)
-
-        # needs some images
-        self.__logger.info("Upload some OS images if it doesn't exist")
-        for image_name, image_file in six.iteritems(self.images):
-            self.__logger.info("image: %s, file: %s", image_name, image_file)
-            if image_file and image_name:
-                image_creator = OpenStackImage(
-                    snaps_creds,
-                    ImageConfig(
-                        name=image_name, image_user='cloud',
-                        img_format='qcow2', image_file=image_file))
-                image_creator.create()
-                self.created_object.append(image_creator)
-
-        # network creation
-        self.__logger.info("Creating full network ...")
-        subnet_settings = SubnetConfig(
-            name='cloudify_ims_subnet-{}'.format(self.uuid),
-            cidr='10.67.79.0/24',
-            dns_nameservers=[env.get('NAMESERVER')])
-        network_settings = NetworkConfig(
-            name='cloudify_ims_network-{}'.format(self.uuid),
-            subnet_settings=[subnet_settings])
-        network_creator = OpenStackNetwork(snaps_creds, network_settings)
-        network_creator.create()
-        self.created_object.append(network_creator)
-        ext_net_name = snaps_utils.get_ext_net_name(snaps_creds)
-        router_creator = OpenStackRouter(
-            snaps_creds,
-            RouterConfig(
-                name='cloudify_ims_router-{}'.format(self.uuid),
-                external_gateway=ext_net_name,
-                internal_subnets=[subnet_settings.name]))
-        router_creator.create()
-        self.created_object.append(router_creator)
-
-        # security group creation
-        self.__logger.info("Creating security group for cloudify manager vm")
-        sg_rules = list()
-        sg_rules.append(
-            SecurityGroupRuleConfig(
-                sec_grp_name="sg-cloudify-manager-{}".format(self.uuid),
-                direction=Direction.ingress, protocol=Protocol.tcp,
-                port_range_min=1, port_range_max=65535))
-        sg_rules.append(
-            SecurityGroupRuleConfig(
-                sec_grp_name="sg-cloudify-manager-{}".format(self.uuid),
-                direction=Direction.ingress, protocol=Protocol.udp,
-                port_range_min=1, port_range_max=65535))
-        security_group_creator = OpenStackSecurityGroup(
-            snaps_creds,
-            SecurityGroupConfig(
-                name="sg-cloudify-manager-{}".format(self.uuid),
-                rule_settings=sg_rules))
-        security_group_creator.create()
-        self.created_object.append(security_group_creator)
-
-        image_settings = ImageConfig(
-            name=self.orchestrator['requirements']['os_image'],
-            image_user='centos',
-            exists=True)
-        port_settings = PortConfig(
-            name='cloudify_manager_port-{}'.format(self.uuid),
-            network_name=network_settings.name)
-        manager_settings = VmInstanceConfig(
-            name='cloudify_manager-{}'.format(self.uuid),
-            flavor=flavor_settings.name,
-            port_settings=[port_settings],
-            security_group_names=[
-                security_group_creator.sec_grp_settings.name],
-            floating_ip_settings=[FloatingIpConfig(
-                name='cloudify_manager_fip-{}'.format(self.uuid),
-                port_name=port_settings.name,
-                router_name=router_creator.router_settings.name)])
-        manager_creator = OpenStackVmInstance(
-            snaps_creds, manager_settings, image_settings,
-            keypair_settings)
-        self.__logger.info("Creating cloudify manager VM")
-        manager_creator.create()
-        self.created_object.append(manager_creator)
-
-        public_auth_url = keystone_utils.get_endpoint(snaps_creds, 'identity')
-
+        self.orig_cloud.set_network_quotas(
+            self.project.project.name,
+            security_group=self.quota_security_group,
+            security_group_rule=self.quota_security_group_rule,
+            port=self.quota_port)
+        self.__logger.info("Put OpenStack creds in manager")
         cfy_creds = dict(
-            keystone_username=snaps_creds.username,
-            keystone_password=snaps_creds.password,
-            keystone_tenant_name=snaps_creds.project_name,
-            keystone_url=public_auth_url,
-            region=snaps_creds.region_name if snaps_creds.region_name else (
-                'RegionOne'),
-            user_domain_name=snaps_creds.user_domain_name,
-            project_domain_name=snaps_creds.project_domain_name)
+            keystone_username=self.project.user.name,
+            keystone_password=self.project.password,
+            keystone_tenant_name=self.project.project.name,
+            keystone_url=self.get_public_auth_url(self.orig_cloud),
+            region=os.environ.get('OS_REGION_NAME', 'RegionOne'),
+            user_domain_name=os.environ.get(
+                'OS_USER_DOMAIN_NAME', 'Default'),
+            project_domain_name=os.environ.get(
+                'OS_PROJECT_DOMAIN_NAME', 'Default'))
         self.__logger.info("Set creds for cloudify manager %s", cfy_creds)
-
-        cfy_client = CloudifyClient(
-            host=manager_creator.get_floating_ip().ip,
-            username='admin', password='admin', tenant='default_tenant',
-            api_version='v3')
-
-        self.orchestrator['object'] = cfy_client
-
-        self.__logger.info("Attemps running status of the Manager")
-        for loop in range(10):
-            try:
-                self.__logger.debug(
-                    "status %s", cfy_client.manager.get_status())
-                cfy_status = cfy_client.manager.get_status()['status']
-                self.__logger.info(
-                    "The current manager status is %s", cfy_status)
-                if str(cfy_status) != 'running':
-                    raise Exception("Cloudify Manager isn't up and running")
-                self.__logger.info("Put OpenStack creds in manager")
-                secrets_list = cfy_client.secrets.list()
-                for k, val in six.iteritems(cfy_creds):
-                    if not any(d.get('key', None) == k for d in secrets_list):
-                        cfy_client.secrets.create(k, val)
-                    else:
-                        cfy_client.secrets.update(k, val)
-                break
-            except Exception:  # pylint: disable=broad-except
-                self.logger.info(
-                    "try %s: Cloudify Manager isn't up and running", loop + 1)
-                time.sleep(30)
-        else:
-            self.logger.error("Cloudify Manager isn't up and running")
-            return False
+        secrets_list = self.cfy_client.secrets.list()
+        for k, val in six.iteritems(cfy_creds):
+            if not any(d.get('key', None) == k for d in secrets_list):
+                self.cfy_client.secrets.create(k, val)
+            else:
+                self.cfy_client.secrets.update(k, val)
 
         duration = time.time() - start_time
 
-        if manager_creator.vm_ssh_active(block=True):
-            self.__logger.info("Put private keypair in manager")
-            ssh = manager_creator.ssh_client()
-            scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
-            scp.put(kp_file, '~/')
-            cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
-            self.run_blocking_ssh_command(ssh, cmd)
-            cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
-            self.run_blocking_ssh_command(ssh, cmd)
-            # cmd2 is badly unpinned by Cloudify
-            cmd = "sudo yum install -y gcc python-devel python-cmd2"
-            self.run_blocking_ssh_command(
-                ssh, cmd, "Unable to install packages on manager")
-            self.run_blocking_ssh_command(ssh, 'cfy status')
-        else:
-            self.__logger.error("Cannot connect to manager")
-            return False
+        self.__logger.info("Put private keypair in manager")
+        scpc = scp.SCPClient(self.ssh.get_transport())
+        scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem')
+        (_, stdout, stderr) = self.ssh.exec_command(
+            "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && "
+            "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && "
+            "sudo yum install -y gcc python-devel python-cmd2 && "
+            "cfy status")
+        self.__logger.info("output:\n%s", stdout.read())
+        self.__logger.info("error:\n%s", stderr.read())
 
         self.details['orchestrator'].update(status='PASS', duration=duration)
 
         self.vnf['inputs'].update(dict(
-            external_network_name=ext_net_name,
-            network_name=network_settings.name,
-            key_pair_name=keypair_settings.name
+            external_network_name=self.ext_net.name,
+            network_name=self.network.name,
+            key_pair_name=self.keypair.name
         ))
+        if (self.deploy_vnf() or self.test_vnf()):
+            self.result = 100
+            return 0
         self.result = 1/3 * 100
-        return True
+        return 1
 
     def deploy_vnf(self):
         """Deploy Clearwater IMS."""
         start_time = time.time()
 
+        self.cloud.create_security_group_rule(
+            'default', port_range_min=22, port_range_max=22,
+            protocol='tcp', direction='ingress')
+
         self.__logger.info("Upload VNFD")
-        cfy_client = self.orchestrator['object']
         descriptor = self.vnf['descriptor']
-        cfy_client.blueprints.upload(
+        self.cfy_client.blueprints.upload(
             descriptor.get('file_name'), descriptor.get('name'))
-        self.__logger.info("Get or create flavor for all clearwater vm")
-        flavor_settings = FlavorConfig(
-            name="{}-{}".format(
-                self.vnf['requirements']['flavor']['name'],
-                self.uuid),
-            ram=self.vnf['requirements']['flavor']['ram_min'],
-            disk=25,
-            vcpus=2)
-        flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
-        flavor_creator.create()
-        self.created_object.append(flavor_creator)
 
+        self.image_alt = self.publish_image_alt()
+        self.flavor_alt = self.create_flavor_alt()
         self.vnf['inputs'].update(dict(
-            flavor_id=flavor_settings.name,
+            image_id=self.image_alt.id,
+            flavor_id=self.flavor_alt.id,
         ))
 
         self.__logger.info("Create VNF Instance")
-        cfy_client.deployments.create(descriptor.get('name'),
-                                      descriptor.get('name'),
-                                      self.vnf.get('inputs'))
+        self.cfy_client.deployments.create(
+            descriptor.get('name'), descriptor.get('name'),
+            self.vnf.get('inputs'))
 
         wait_for_execution(
-            cfy_client,
-            get_execution_id(cfy_client, descriptor.get('name')),
+            self.cfy_client,
+            get_execution_id(self.cfy_client, descriptor.get('name')),
             self.__logger, timeout=300)
 
         self.__logger.info("Start the VNF Instance deployment")
-        execution = cfy_client.executions.start(descriptor.get('name'),
-                                                'install')
+        execution = self.cfy_client.executions.start(
+            descriptor.get('name'), 'install')
         # Show execution log
         execution = wait_for_execution(
-            cfy_client, execution, self.__logger, timeout=3600)
+            self.cfy_client, execution, self.__logger, timeout=3600)
 
         duration = time.time() - start_time
 
@@ -377,18 +202,17 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         """Run test on clearwater ims instance."""
         start_time = time.time()
 
-        cfy_client = self.orchestrator['object']
-
-        outputs = cfy_client.deployments.outputs.get(
+        testing = clearwater_ims_base.ClearwaterOnBoardingBase(self.case_name)
+        outputs = self.cfy_client.deployments.outputs.get(
             self.vnf['descriptor'].get('name'))['outputs']
         dns_ip = outputs['dns_ip']
         ellis_ip = outputs['ellis_ip']
-        self.config_ellis(ellis_ip)
+        testing.config_ellis(ellis_ip)
 
         if not dns_ip:
             return False
 
-        short_result = self.run_clearwater_live_test(
+        short_result = testing.run_clearwater_live_test(
             dns_ip=dns_ip,
             public_domain=self.vnf['inputs']["public_domain"])
         duration = time.time() - start_time
@@ -413,48 +237,34 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
     def clean(self):
         """Clean created objects/functions."""
         try:
-            cfy_client = self.orchestrator['object']
             dep_name = self.vnf['descriptor'].get('name')
             # kill existing execution
             self.__logger.info('Deleting the current deployment')
-            exec_list = cfy_client.executions.list(dep_name)
+            exec_list = self.cfy_client.executions.list(dep_name)
             for execution in exec_list:
                 if execution['status'] == "started":
                     try:
-                        cfy_client.executions.cancel(execution['id'],
-                                                     force=True)
+                        self.cfy_client.executions.cancel(
+                            execution['id'], force=True)
                     except Exception:  # pylint: disable=broad-except
                         self.__logger.warn("Can't cancel the current exec")
 
-            execution = cfy_client.executions.start(
+            execution = self.cfy_client.executions.start(
                 dep_name,
                 'uninstall',
                 parameters=dict(ignore_failure=True),
                 force=True)
 
-            wait_for_execution(cfy_client, execution, self.__logger)
-            cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
-            cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
+            wait_for_execution(self.cfy_client, execution, self.__logger)
+            self.cfy_client.deployments.delete(
+                self.vnf['descriptor'].get('name'))
+            self.cfy_client.blueprints.delete(
+                self.vnf['descriptor'].get('name'))
         except Exception:  # pylint: disable=broad-except
             self.__logger.exception("Some issue during the undeployment ..")
 
         super(CloudifyIms, self).clean()
 
-    @staticmethod
-    def run_blocking_ssh_command(ssh, cmd,
-                                 error_msg="Unable to run this command"):
-        """Command to run ssh command with the exit status."""
-        _, stdout, stderr = ssh.exec_command(cmd)
-        CloudifyIms.__logger.debug("SSH %s stdout: %s", cmd, stdout.read())
-        if stdout.channel.recv_exit_status() != 0:
-            CloudifyIms.__logger.error("SSH %s stderr: %s", cmd, stderr.read())
-            raise Exception(error_msg)
-
-    @energy.enable_recording
-    def run(self, **kwargs):
-        """Execute CloudifyIms test case."""
-        return super(CloudifyIms, self).run(**kwargs)
-
 
 # ----------------------------------------------------------
 #