[WIP] update cloudify_ims case

bug fixes + adaptation to abstraction class

JIRA: FUNCTEST-710

Change-Id: I687876637bd9eeeba9256e28b98a1e3a4f5c3e22
Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>

4 files changed, 205 insertions, 115 deletions

diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml
index f6cb14cb1..f10ff4ae1 100755
--- a/functest/ci/config_functest.yaml
+++ b/functest/ci/config_functest.yaml
@@ -5,7 +5,6 @@ general:
         dir_odl:           functest/opnfv_tests/sdn/odl
         rally:             functest/opnfv_tests/openstack/rally
         tempest_cases:     functest/opnfv_tests/openstack/tempest/custom_tests
-        dir_vIMS:          functest/opnfv_tests/vnf/ims
         dir_onos:          functest/opnfv_tests/sdn/onos/teston
         dir_onos_sfc:      functest/opnfv_tests/sdn/onos/sfc
 
@@ -33,7 +32,7 @@ general:
         functest_logging_cfg:   /home/opnfv/repos/functest/functest/ci/logging.json
         functest_conf:      /home/opnfv/functest/conf
         functest_data:      /home/opnfv/functest/data
-        dir_vIMS_data:      /home/opnfv/functest/data/vIMS/
+        ims_data:           /home/opnfv/functest/data/ims/
         rally_inst:         /home/opnfv/.rally
 
     openstack:
@@ -125,45 +124,7 @@ vnf:
     cloudify_ims:
         tenant_name: cloudify_ims
         tenant_description: vIMS
-        tenant_images:
-            ubuntu_14.04: http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
-            centos_7: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1510.qcow2
-        cloudify:
-            blueprint:
-                url: https://github.com/boucherv-orange/cloudify-manager-blueprints.git
-                branch: "3.3.1-build"
-            requierments:
-                ram_min: 3000
-                os_image: centos_7
-            inputs:
-                keystone_username: ""
-                keystone_password: ""
-                keystone_tenant_name: ""
-                keystone_url: ""
-                manager_public_key_name: 'manager-kp'
-                agent_public_key_name: 'agent-kp'
-                image_id: ""
-                flavor_id: "3"
-                external_network_name: ""
-                ssh_user: centos
-                agents_user: ubuntu
-        clearwater:
-            blueprint:
-                file_name: 'openstack-blueprint.yaml'
-                name: "clearwater-opnfv"
-                destination_folder: "opnfv-cloudify-clearwater"
-                url: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater.git
-                branch: "stable"
-            deployment_name: 'clearwater-opnfv'
-            requirements:
-                ram_min: 1700
-                os_image: ubuntu_14.04
-            inputs:
-                image_id: ''
-                flavor_id: ''
-                agent_user: 'ubuntu'
-                external_network_name: ''
-                public_domain: clearwater.opnfv
+        config: cloudify_ims.yaml
     orchestra_ims:
         tenant_name: orchestra_ims
         tenant_description: ims deployed with openbaton
diff --git a/functest/core/vnf_base.py b/functest/core/vnf_base.py
index 4d019858a..44b4ae04c 100644
--- a/functest/core/vnf_base.py
+++ b/functest/core/vnf_base.py
@@ -7,12 +7,10 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 
-import os
 import time
 
 import inspect
 
-
 import functest.utils.functest_logger as ft_logger
 import functest.utils.openstack_utils as os_utils
 import functest.utils.functest_utils as ft_utils
@@ -161,30 +159,6 @@ class VnfOnBoardingBase(base.TestcaseBase):
             "username": self.tenant_name,
             "password": self.tenant_name,
         })
-        self.glance_client = os_utils.get_glance_client(self.creds)
-        self.logger.info("Upload some OS images if it doesn't exist")
-
-        temp_dir = os.path.join(self.data_dir, "tmp/")
-        for image_name, image_url in self.images.iteritems():
-            image_id = os_utils.get_image_id(self.glance_client, image_name)
-
-            if image_id == '':
-                self.logger.info("""%s image doesn't exist on glance repository. Try
-                downloading this image and upload on glance !""" % image_name)
-                image_id = os_utils.download_and_add_image_on_glance(
-                    self.glance_client, image_name, image_url, temp_dir)
-
-            if image_id == '':
-                self.step_failure(
-                    "Failed to find or upload required OS "
-                    "image for this deployment")
-
-        self.logger.info("Update security group quota for this tenant")
-
-        if not os_utils.update_sg_quota(self.neutron_client,
-                                        tenant_id, 50, 100):
-            self.step_failure("Failed to update security group quota" +
-                              " for tenant " + self.tenant_name)
 
     # orchestrator is not mandatory to dpeloy and test VNF
     def deploy_orchestrator(self, **kwargs):
diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
index 13a5af4fd..e2508c223 100644
--- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py
+++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
@@ -11,7 +11,9 @@ import json
 import os
 import requests
 import subprocess
+import sys
 import time
+import yaml
 
 import functest.core.vnf_base as vnf_base
 import functest.utils.functest_logger as ft_logger
@@ -29,57 +31,127 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
                  repo='', cmd=''):
         super(ImsVnf, self).__init__(project, case, repo, cmd)
         self.logger = ft_logger.Logger("vIMS").getLogger()
-        self.case_dir = os.path.join(CONST.functest_test, 'vnf/ims/')
-        self.data_dir = CONST.dir_vIMS_data
+        self.case_dir = os.path.join(CONST.dir_functest_test, 'vnf/ims/')
+        self.data_dir = CONST.dir_ims_data
         self.test_dir = CONST.dir_repo_vims_test
 
+        # Retrieve the configuration
+        try:
+            self.config = CONST.__getattribute__(
+                'vnf_{}_config'.format(self.case_name))
+        except:
+            raise Exception("VNF config file not found")
+
+        config_file = self.case_dir + self.config
         self.orchestrator = dict(
-            requirements=CONST.cloudify_requirements,
-            blueprint=CONST.cloudify_blueprint,
-            inputs=CONST.cloudify_inputs
+            requirements=get_config("cloudify.requirements", config_file),
+            blueprint=get_config("cloudify.blueprint", config_file),
+            inputs=get_config("cloudify.inputs", config_file)
         )
-
+        self.logger.debug("Orchestrator configuration: %s" % self.orchestrator)
         self.vnf = dict(
-            blueprint=CONST.clearwater_blueprint,
-            deployment_name=CONST.clearwater_deployment_name,
-            inputs=CONST.clearwater_inputs,
-            requirements=CONST.clearwater_requirements
+            blueprint=get_config("clearwater.blueprint", config_file),
+            deployment_name=get_config("clearwater.deployment_name",
+                                       config_file),
+            inputs=get_config("clearwater.inputs", config_file),
+            requirements=get_config("clearwater.requirements", config_file)
         )
+        self.logger.debug("VNF configuration: %s" % self.vnf)
+
+        self.images = get_config("tenant_images", config_file)
+        self.logger.info("Images needed for vIMS: %s" % self.images)
 
         # vIMS Data directory creation
         if not os.path.exists(self.data_dir):
             os.makedirs(self.data_dir)
 
     def deploy_orchestrator(self, **kwargs):
+
+        self.logger.info("Additional pre-configuration steps")
+        self.neutron_client = os_utils.get_neutron_client(self.creds)
+        self.glance_client = os_utils.get_glance_client(self.creds)
+        self.keystone_client = os_utils.get_keystone_client(self.creds)
+        self.nova_client = os_utils.get_nova_client(self.creds)
+
+        # needs some images
+        self.logger.info("Upload some OS images if it doesn't exist")
+        temp_dir = os.path.join(self.data_dir, "tmp/")
+        for image_name, image_url in self.images.iteritems():
+            self.logger.info("image: %s, url: %s" % (image_name, image_url))
+            try:
+                image_id = os_utils.get_image_id(self.glance_client,
+                                                 image_name)
+                self.logger.debug("image_id: %s" % image_id)
+            except:
+                self.logger.error("Unexpected error: %s" % sys.exc_info()[0])
+
+            if image_id == '':
+                self.logger.info("""%s image doesn't exist on glance repository. Try
+                downloading this image and upload on glance !""" % image_name)
+                image_id = download_and_add_image_on_glance(self.glance_client,
+                                                            image_name,
+                                                            image_url,
+                                                            temp_dir)
+            if image_id == '':
+                self.step_failure(
+                    "Failed to find or upload required OS "
+                    "image for this deployment")
+        # Need to extend quota
+        self.logger.info("Update security group quota for this tenant")
+        tenant_id = os_utils.get_tenant_id(self.keystone_client,
+                                           self.tenant_name)
+        self.logger.debug("Tenant id found %s" % tenant_id)
+        if not os_utils.update_sg_quota(self.neutron_client,
+                                        tenant_id, 50, 100):
+            self.step_failure("Failed to update security group quota" +
+                              " for tenant " + self.tenant_name)
+        self.logger.debug("group quota extended")
+
+        # start the deployment of cloudify
         public_auth_url = os_utils.get_endpoint('identity')
 
-        cfy = Orchestrator(self.data_dir, self.orchestrator.inputs)
-        self.orchestrator.object = cfy
+        self.logger.debug("CFY inputs: %s" % self.orchestrator['inputs'])
+        cfy = Orchestrator(self.data_dir, self.orchestrator['inputs'])
+        self.orchestrator['object'] = cfy
+        self.logger.debug("Orchestrator object created")
 
-        if 'tenant_name' in self.creds.keys():
-            tenant_name = self.creds['tenant_name']
-        elif 'project_name' in self.creds.keys():
-            tenant_name = self.creds['project_name']
+        self.logger.debug("Tenant name: %s" % self.tenant_name)
 
-        cfy.set_credentials(username=self.creds['username'],
-                            password=self.creds['password'],
-                            tenant_name=tenant_name,
+        cfy.set_credentials(username=self.tenant_name,
+                            password=self.tenant_name,
+                            tenant_name=self.tenant_name,
                             auth_url=public_auth_url)
+        self.logger.info("Credentials set in CFY")
 
         # orchestrator VM flavor
-        flavor_id = self.get_flavor("m1.large", self.orchestrator.requirements)
+        self.logger.info("Check Flavor is available, if not create one")
+        self.logger.debug("Flavor details %s " %
+                          self.orchestrator['requirements']['ram_min'])
+        flavor_exist, flavor_id = os_utils.get_or_create_flavor(
+            "m1.large",
+            self.orchestrator['requirements']['ram_min'],
+            '1',
+            '1',
+            public=True)
+        self.logger.debug("Flavor id: %s" % flavor_id)
+
         if not flavor_id:
             self.logger.info("Available flavors are: ")
-            self.pMsg(self.nova_client.flavor.list())
+            self.logger.info(self.nova_client.flavor.list())
             self.step_failure("Failed to find required flavor"
                               "for this deployment")
         cfy.set_flavor_id(flavor_id)
+        self.logger.debug("Flavor OK")
 
         # orchestrator VM image
-        if 'os_image' in self.orchestrator.requirements.keys():
+        self.logger.debug("Orchestrator image")
+        if 'os_image' in self.orchestrator['requirements'].keys():
             image_id = os_utils.get_image_id(
-                self.glance_client, self.orchestrator.requirements['os_image'])
+                self.glance_client,
+                self.orchestrator['requirements']['os_image'])
+            self.logger.debug("Orchestrator image id: %s" % image_id)
             if image_id == '':
+                self.logger.error("CFY image not found")
                 self.step_failure("Failed to find required OS image"
                                   " for cloudify manager")
         else:
@@ -87,16 +159,22 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
                               " for cloudify manager")
 
         cfy.set_image_id(image_id)
+        self.logger.debug("Orchestrator image set")
 
+        self.logger.debug("Get External network")
         ext_net = os_utils.get_external_net(self.neutron_client)
+        self.logger.debug("External network: %s" % ext_net)
         if not ext_net:
             self.step_failure("Failed to get external network")
 
         cfy.set_external_network_name(ext_net)
+        self.logger.debug("CFY External network set")
 
+        self.logger.debug("get resolvconf")
         ns = ft_utils.get_resolvconf_ns()
         if ns:
             cfy.set_nameservers(ns)
+            self.logger.debug("Resolvconf set")
 
         if 'compute' in self.nova_client.client.services_url:
             cfy.set_nova_url(self.nova_client.client.services_url['compute'])
@@ -110,8 +188,9 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
         cmd = self.case_dir + "create_venv.sh " + self.data_dir
         ft_utils.execute_command(cmd)
 
-        cfy.download_manager_blueprint(self.orchestrator.blueprint['url'],
-                                       self.orchestrator.blueprint['branch'])
+        cfy.download_manager_blueprint(
+            self.orchestrator['blueprint']['url'],
+            self.orchestrator['blueprint']['branch'])
 
         cfy.deploy_manager()
         return {'status': 'PASS', 'result': ''}
@@ -121,10 +200,16 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
         self.vnf.object = cw
 
         self.logger.info("Collect flavor id for all clearwater vm")
-        flavor_id = self.get_flavor("m1.small", self.vnf.requirements)
+        flavor_exist, flavor_id = os_utils.get_or_create_flavor(
+            "m1.small",
+            self.vnf['requirements']['ram_min'],
+            '1',
+            '1',
+            public=True)
+        self.logger.debug("Flavor id: %s" % flavor_id)
         if not flavor_id:
             self.logger.info("Available flavors are: ")
-            self.pMsg(self.nova_client.flavor.list())
+            self.logger.info(self.nova_client.flavor.list())
             self.step_failure("Failed to find required flavor"
                               " for this deployment")
 
@@ -133,7 +218,7 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
         # VMs image
         if 'os_image' in self.vnf.requirements.keys():
             image_id = os_utils.get_image_id(
-                self.glance_client, self.vnf.requirements['os_image'])
+                self.glance_client, self.vnf['requirements']['os_image'])
             if image_id == '':
                 self.step_failure("Failed to find required OS image"
                                   " for clearwater VMs")
@@ -256,23 +341,54 @@ class ImsVnf(vnf_base.VnfOnBoardingBase):
         self.orchestrator.object.undeploy_manager()
         super(ImsVnf, self).clean()
 
-    def get_flavor(self, flavor_name, requirements):
-        try:
-            flavor_id = os_utils.get_flavor_id(self.nova_client, flavor_name)
-            if 'ram_min' in requirements.keys():
-                flavor_id = os_utils.get_flavor_id_by_ram_range(
-                    self.nova_client, requirements['ram_min'], 7500)
-
-            if flavor_id == '':
-                self.logger.error(
-                    "Failed to find %s flavor. "
-                    "Try with ram range default requirement !" % flavor_name)
-                flavor_id = os_utils.get_flavor_id_by_ram_range(
-                                    self.nova_client,
-                                    4000, 10000)
-            return flavor_id
-        except:
-            self.logger.error("Flavor '%s' not found." % self.flavor_name)
-            self.logger.info("Available flavors are: ")
-            self.pMsg(self.nova_client.flavor.list())
-            return None
+    def main(self, **kwargs):
+        self.logger.info("Cloudify IMS VNF onboarding test starting")
+        self.execute()
+        self.logger.info("Cloudify IMS VNF onboarding test executed")
+        if self.criteria is "PASS":
+            return self.EX_OK
+        else:
+            return self.EX_RUN_ERROR
+
+    def run(self):
+        kwargs = {}
+        return self.main(**kwargs)
+
+
+# ----------------------------------------------------------
+#
+#               YAML UTILS
+#
+# -----------------------------------------------------------
+def get_config(parameter, file):
+    """
+    Returns the value of a given parameter in file.yaml
+    parameter must be given in string format with dots
+    Example: general.openstack.image_name
+    """
+    with open(file) as f:
+        file_yaml = yaml.safe_load(f)
+    f.close()
+    value = file_yaml
+    for element in parameter.split("."):
+        value = value.get(element)
+        if value is None:
+            raise ValueError("The parameter %s is not defined in"
+                             " reporting.yaml" % parameter)
+    return value
+
+
+def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
+    dest_path = data_dir
+    if not os.path.exists(dest_path):
+        os.makedirs(dest_path)
+    file_name = image_url.rsplit('/')[-1]
+    if not ft_utils.download_url(image_url, dest_path):
+        return False
+
+    image = os_utils.create_glance_image(
+        glance, image_name, dest_path + file_name)
+    if not image:
+        return False
+
+    return image
diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.yaml b/functest/opnfv_tests/vnf/ims/cloudify_ims.yaml
new file mode 100644
index 000000000..c59180876
--- /dev/null
+++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.yaml
@@ -0,0 +1,39 @@
+tenant_images:
+    ubuntu_14.04: http://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
+    centos_7: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1510.qcow2
+cloudify:
+    blueprint:
+        url: https://github.com/boucherv-orange/cloudify-manager-blueprints.git
+        branch: '3.3.1-build'
+    requirements:
+        ram_min: 3000
+        os_image: centos_7
+    inputs:
+      keystone_username: ""
+      keystone_password: ""
+      keystone_tenant_name: ""
+      keystone_url: ""
+      manager_public_key_name: 'manager-kp'
+      agent_public_key_name: 'agent-kp'
+      image_id: ""
+      flavor_id: "3"
+      external_network_name: ""
+      ssh_user: centos
+      agents_user: ubuntu
+clearwater:
+    blueprint:
+        file_name: openstack-blueprint.yaml
+        name: clearwater-opnfv
+        destination_folder: opnfv-cloudify-clearwater
+        url: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater.git
+        branch: stable
+    deployment_name: clearwater-opnfv
+    requirements:
+        ram_min: 1700
+        os_image: ubuntu_14.04
+    inputs:
+        image_id: ''
+        flavor_id: ''
+        agent_user: ubuntu
+        external_network_name: ''
+        public_domain: clearwaterfv