Authenticate clients with keystoneauth1.session

JIRA: FUNCTEST-529

1. use keystoneauth1.session to initialize the client for each service
The keystoneauth1.session.Session class was introduced into keystoneauth1 as an attempt
to bring a unified interface to the various OpenStack clients that share common authentication
 and request parameters between a variety of services.

2. update ODL case to leverage session to get the endpoint info

Change-Id: Ic8c01b9b7ed86d3bdd9f5125504bc47f46a37700
Signed-off-by: helenyao <yaohelan@huawei.com>

6 files changed, 126 insertions, 102 deletions

diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index 3f02c8725..d6991f66b 100644
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -124,6 +124,7 @@ def run_test(test, tier_name):
     logger.info("Running test case '%s'..." % test_name)
     print_separator("=")
     logger.debug("\n%s" % test)
+    source_rc_file()
 
     if GlobalVariables.CLEAN_FLAG:
         generate_os_snapshot()
diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py
index 95440746c..bc9f1bb7f 100755
--- a/functest/opnfv_tests/sdn/odl/odl.py
+++ b/functest/opnfv_tests/sdn/odl/odl.py
@@ -146,11 +146,8 @@ class ODLTests(testcase_base.TestcaseBase):
 
     def run(self):
         try:
-            kclient = op_utils.get_keystone_client()
-            keystone_url = kclient.service_catalog.url_for(
-                service_type='identity', endpoint_type='publicURL')
-            neutron_url = kclient.service_catalog.url_for(
-                service_type='network', endpoint_type='publicURL')
+            keystone_url = op_utils.get_endpoint(service_type='identity')
+            neutron_url = op_utils.get_endpoint(service_type='network')
             kwargs = {'keystoneip': urlparse.urlparse(keystone_url).hostname}
             kwargs['neutronip'] = urlparse.urlparse(neutron_url).hostname
             kwargs['odlip'] = kwargs['neutronip']
diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py
index ef18016bf..c29bfd7f3 100644..100755
--- a/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@ -59,14 +59,6 @@ class ODLTesting(unittest.TestCase):
         else:
             return None
 
-    @classmethod
-    def _get_fake_keystone_client(cls):
-        kclient = mock.Mock()
-        kclient.service_catalog = mock.Mock()
-        kclient.service_catalog.url_for = mock.Mock(
-            side_effect=cls._fake_url_for)
-        return kclient
-
     def _get_main_kwargs(self, key=None):
         kwargs = {'odlusername': self._odl_username,
                   'odlpassword': self._odl_password,
@@ -247,8 +239,9 @@ class ODLTesting(unittest.TestCase):
 
     def _test_run(self, status=testcase_base.TestcaseBase.EX_OK,
                   exception=None, odlip="127.0.0.3", odlwebport="8080"):
-        with mock.patch('functest.utils.openstack_utils.get_keystone_client',
-                        return_value=self._get_fake_keystone_client()):
+        with mock.patch('functest.utils.openstack_utils.get_endpoint',
+                        side_effect=[self._fake_url_for('identity'),
+                                     self._fake_url_for('network')]):
             if exception:
                 self.test.main = mock.Mock(side_effect=exception)
             else:
@@ -288,12 +281,10 @@ class ODLTesting(unittest.TestCase):
                            odlwebport=self._odl_webport)
 
     def test_run_missing_sdn_controller_ip(self):
-        with mock.patch('functest.utils.openstack_utils.get_keystone_client',
-                        return_value=self._get_fake_keystone_client()):
-            ft_constants.CI_INSTALLER_TYPE = None
-            ft_constants.SDN_CONTROLLER_IP = None
-            self.assertEqual(self.test.run(),
-                             testcase_base.TestcaseBase.EX_RUN_ERROR)
+        ft_constants.CI_INSTALLER_TYPE = None
+        ft_constants.SDN_CONTROLLER_IP = None
+        self.assertEqual(self.test.run(),
+                         testcase_base.TestcaseBase.EX_RUN_ERROR)
 
     def test_run_without_installer_type(self):
         ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
@@ -308,12 +299,10 @@ class ODLTesting(unittest.TestCase):
                        odlip=self._neutron_ip, odlwebport='8282')
 
     def test_run_apex_missing_sdn_controller_ip(self):
-        with mock.patch('functest.utils.openstack_utils.get_keystone_client',
-                        return_value=self._get_fake_keystone_client()):
-            ft_constants.CI_INSTALLER_TYPE = "apex"
-            ft_constants.SDN_CONTROLLER_IP = None
-            self.assertEqual(self.test.run(),
-                             testcase_base.TestcaseBase.EX_RUN_ERROR)
+        ft_constants.CI_INSTALLER_TYPE = "apex"
+        ft_constants.SDN_CONTROLLER_IP = None
+        self.assertEqual(self.test.run(),
+                         testcase_base.TestcaseBase.EX_RUN_ERROR)
 
     def test_run_apex(self):
         ft_constants.SDN_CONTROLLER_IP = self._sdn_controller_ip
@@ -322,12 +311,10 @@ class ODLTesting(unittest.TestCase):
                        odlip=self._sdn_controller_ip, odlwebport='8181')
 
     def test_run_joid_missing_sdn_controller(self):
-        with mock.patch('functest.utils.openstack_utils.get_keystone_client',
-                        return_value=self._get_fake_keystone_client()):
-            ft_constants.CI_INSTALLER_TYPE = "joid"
-            ft_constants.SDN_CONTROLLER = None
-            self.assertEqual(self.test.run(),
-                             testcase_base.TestcaseBase.EX_RUN_ERROR)
+        ft_constants.CI_INSTALLER_TYPE = "joid"
+        ft_constants.SDN_CONTROLLER = None
+        self.assertEqual(self.test.run(),
+                         testcase_base.TestcaseBase.EX_RUN_ERROR)
 
     def test_run_joid(self):
         ft_constants.SDN_CONTROLLER = self._sdn_controller_ip
diff --git a/functest/utils/openstack_clean.py b/functest/utils/openstack_clean.py
index 949eee90f..c08568bde 100755
--- a/functest/utils/openstack_clean.py
+++ b/functest/utils/openstack_clean.py
@@ -9,6 +9,8 @@
 #       - Neutron networks, subnets and ports
 #       - Routers
 #       - Users and tenants
+#       - Tacker VNFDs and VNFs
+#       - Tacker SFCs and SFC classifiers
 #
 # Author:
 #    jose.lausuch@ericsson.com
@@ -105,7 +107,7 @@ def remove_volumes(cinder_client, default_volumes):
 
     for volume in volumes:
         volume_id = getattr(volume, 'id')
-        volume_name = getattr(volume, 'display_name')
+        volume_name = getattr(volume, 'name')
         logger.debug("'%s', ID=%s " % (volume_name, volume_id))
         if (volume_id not in default_volumes and
                 volume_name not in default_volumes.values()):
@@ -393,7 +395,7 @@ def main():
     default_security_groups = snapshot_yaml.get('secgroups')
     default_floatingips = snapshot_yaml.get('floatingips')
     default_users = snapshot_yaml.get('users')
-    default_tenants = snapshot_yaml.get('tenants')
+    # default_tenants = snapshot_yaml.get('tenants')
 
     if not os_utils.check_credentials():
         logger.error("Please source the openrc credentials and run "
@@ -414,8 +416,10 @@ def main():
     separator()
     remove_users(keystone_client, default_users)
     separator()
-    remove_tenants(keystone_client, default_tenants)
-    separator()
+    # TODO (Helen) tenant does not exist in V3
+    # need to figure our anohter general verification point
+    # remove_tenants(keystone_client, default_tenants)
+    # separator()
 
 
 if __name__ == '__main__':
diff --git a/functest/utils/openstack_snapshot.py b/functest/utils/openstack_snapshot.py
index 4be1af443..5b50ffa50 100755
--- a/functest/utils/openstack_snapshot.py
+++ b/functest/utils/openstack_snapshot.py
@@ -62,7 +62,7 @@ def get_volumes(cinder_client):
     volumes = os_utils.get_volumes(cinder_client)
     if volumes is not None:
         for volume in volumes:
-            dic_volumes.update({volume.id: volume.display_name})
+            dic_volumes.update({volume.id: volume.name})
     return {'volumes': dic_volumes}
 
 
@@ -149,7 +149,7 @@ def main():
     snapshot.update(get_security_groups(neutron_client))
     snapshot.update(get_floatinips(nova_client))
     snapshot.update(get_users(keystone_client))
-    snapshot.update(get_tenants(keystone_client))
+    # snapshot.update(get_tenants(keystone_client))
 
     with open(OS_SNAPSHOT_FILE, 'w+') as yaml_file:
         yaml_file.write(yaml.safe_dump(snapshot, default_flow_style=False))
diff --git a/functest/utils/openstack_utils.py b/functest/utils/openstack_utils.py
index 8f698d73d..711a62f61 100755
--- a/functest/utils/openstack_utils.py
+++ b/functest/utils/openstack_utils.py
@@ -14,16 +14,21 @@ import subprocess
 import sys
 import time
 
+from keystoneauth1 import loading
+from keystoneauth1 import session
 from cinderclient import client as cinderclient
-import functest.utils.functest_logger as ft_logger
-import functest.utils.functest_utils as ft_utils
 from glanceclient import client as glanceclient
-from keystoneclient.v2_0 import client as keystoneclient
-from neutronclient.v2_0 import client as neutronclient
 from novaclient import client as novaclient
+from keystoneclient import client as keystoneclient
+from neutronclient.neutron import client as neutronclient
+
+import functest.utils.functest_logger as ft_logger
+import functest.utils.functest_utils as ft_utils
 
 logger = ft_logger.Logger("openstack_utils").getLogger()
 
+DEFAULT_API_VERSION = '2'
+
 
 # *********************************************
 #   CREDENTIALS
@@ -45,14 +50,8 @@ def check_credentials():
     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
 
 
-def get_credentials(service):
-    """Returns a creds dictionary filled with the following keys:
-    * username
-    * password/api_key (depending on the service)
-    * tenant_name/project_id (depending on the service)
-    * auth_url
-    :param service: a string indicating the name of the service
-                    requesting the credentials.
+def get_credentials():
+    """Returns a creds dictionary filled with parsed from env
     """
     creds = {}
 
@@ -73,19 +72,11 @@ def get_credentials(service):
         if os.getenv(envvar) is None:
             raise MissingEnvVar(envvar)
 
-    # Unfortunately, each of the OpenStack client will request slightly
-    # different entries in their credentials dict.
-    if service.lower() in ("nova", "cinder"):
-        password = "api_key"
-        tenant = "project_id"
-    else:
-        password = "password"
-
     # The most common way to pass these info to the script is to do it through
     # environment variables.
     creds.update({
         "username": os.environ.get("OS_USERNAME"),
-        password: os.environ.get("OS_PASSWORD"),
+        "password": os.environ.get("OS_PASSWORD"),
         "auth_url": os.environ.get("OS_AUTH_URL"),
         tenant: os.environ.get(tenant_env)
     })
@@ -132,7 +123,7 @@ def source_credentials(rc_file):
 
 
 def get_credentials_for_rally():
-    creds = get_credentials("keystone")
+    creds = get_credentials()
     keystone_api_version = os.getenv('OS_IDENTITY_API_VERSION')
     if (keystone_api_version is None or
             keystone_api_version == '2'):
@@ -156,42 +147,95 @@ def get_credentials_for_rally():
     return rally_conf
 
 
+def get_session_auth():
+    loader = loading.get_plugin_loader('password')
+    creds = get_credentials()
+    auth = loader.load_from_options(**creds)
+    return auth
+
+
+def get_endpoint(service_type, endpoint_type='publicURL'):
+    auth = get_session_auth()
+    return get_session().get_endpoint(auth=auth,
+                                      service_type=service_type,
+                                      endpoint_type=endpoint_type)
+
+
+def get_session():
+    auth = get_session_auth()
+    return session.Session(auth=auth)
+
+
 # *********************************************
 #   CLIENTS
 # *********************************************
+def get_keystone_client_version():
+    api_version = os.getenv('OS_IDENTITY_API_VERSION')
+    if api_version is not None:
+        logger.info("OS_IDENTITY_API_VERSION is set in env as '%s'",
+                    api_version)
+        return api_version
+    return DEFAULT_API_VERSION
+
+
 def get_keystone_client():
-    creds_keystone = get_credentials("keystone")
-    return keystoneclient.Client(**creds_keystone)
+    sess = get_session()
+    return keystoneclient.Client(get_keystone_client_version(), session=sess)
+
+
+def get_nova_client_version():
+    api_version = os.getenv('OS_COMPUTE_API_VERSION')
+    if api_version is not None:
+        logger.info("OS_COMPUTE_API_VERSION is set in env as '%s'",
+                    api_version)
+        return api_version
+    return DEFAULT_API_VERSION
 
 
 def get_nova_client():
-    creds_nova = get_credentials("nova")
-    return novaclient.Client('2', **creds_nova)
+    sess = get_session()
+    return novaclient.Client(get_nova_client_version(), session=sess)
+
+
+def get_cinder_client_version():
+    api_version = os.getenv('OS_VOLUME_API_VERSION')
+    if api_version is not None:
+        logger.info("OS_VOLUME_API_VERSION is set in env as '%s'",
+                    api_version)
+        return api_version
+    return DEFAULT_API_VERSION
 
 
 def get_cinder_client():
-    creds_cinder = get_credentials("cinder")
-    creds_cinder.update({
-        "service_type": "volume"
-    })
-    return cinderclient.Client('2', **creds_cinder)
+    sess = get_session()
+    return cinderclient.Client(get_cinder_client_version(), session=sess)
+
+
+def get_neutron_client_version():
+    api_version = os.getenv('OS_NETWORK_API_VERSION')
+    if api_version is not None:
+        logger.info("OS_NETWORK_API_VERSION is set in env as '%s'",
+                    api_version)
+        return api_version
+    return DEFAULT_API_VERSION
 
 
 def get_neutron_client():
-    creds_neutron = get_credentials("neutron")
-    return neutronclient.Client(**creds_neutron)
+    sess = get_session()
+    return neutronclient.Client(get_neutron_client_version(), session=sess)
+
+
+def get_glance_client_version():
+    api_version = os.getenv('OS_IMAGE_API_VERSION')
+    if api_version is not None:
+        logger.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
+        return api_version
+    return DEFAULT_API_VERSION
 
 
 def get_glance_client():
-    keystone_client = get_keystone_client()
-    glance_endpoint_type = 'publicURL'
-    os_endpoint_type = os.getenv('OS_ENDPOINT_TYPE')
-    if os_endpoint_type is not None:
-        glance_endpoint_type = os_endpoint_type
-    glance_endpoint = keystone_client.service_catalog.url_for(
-        service_type='image', endpoint_type=glance_endpoint_type)
-    return glanceclient.Client(1, glance_endpoint,
-                               token=keystone_client.auth_token)
+    sess = get_session()
+    return glanceclient.Client(get_glance_client_version(), session=sess)
 
 
 # *********************************************
@@ -1070,38 +1114,29 @@ def get_image_id(glance_client, image_name):
 
 
 def create_glance_image(glance_client, image_name, file_path, disk="qcow2",
-                        container="bare", public=True):
+                        container="bare", public="public"):
     if not os.path.isfile(file_path):
         logger.error("Error: file %s does not exist." % file_path)
         return None
     try:
         image_id = get_image_id(glance_client, image_name)
         if image_id != '':
-            if logger:
-                logger.info("Image %s already exists." % image_name)
+            logger.info("Image %s already exists." % image_name)
         else:
-            if logger:
-                logger.info("Creating image '%s' from '%s'..." % (image_name,
-                                                                  file_path))
-            try:
-                properties = ft_utils.get_functest_config(
-                    'general.image_properties')
-            except ValueError:
-                # image properties are not configured
-                # therefore don't add any properties
-                properties = {}
-            with open(file_path) as fimage:
-                image = glance_client.images.create(name=image_name,
-                                                    is_public=public,
-                                                    disk_format=disk,
-                                                    container_format=container,
-                                                    properties=properties,
-                                                    data=fimage)
+            logger.info("Creating image '%s' from '%s'..." % (image_name,
+                                                              file_path))
+
+            image = glance_client.images.create(name=image_name,
+                                                visibility=public,
+                                                disk_format=disk,
+                                                container_format=container)
             image_id = image.id
+            with open(file_path) as image_data:
+                glance_client.images.upload(image_id, image_data)
         return image_id
     except Exception, e:
         logger.error("Error [create_glance_image(glance_client, '%s', '%s', "
-                     "'%s')]: %s" % (image_name, file_path, str(public), e))
+                     "'%s')]: %s" % (image_name, file_path, public, e))
         return None