Add common method of openstack in openstack_utils

JIRA: YARDSTICK-569

We have much redundancy code when using nova/neutron/glance... python client.
So I write this code in openstack_utils for calling.

Change-Id: I322b7577de4933246a15e6742ae5a28bea16eb02
Signed-off-by: chenjiankun <chenjiankun1@huawei.com>

4 files changed, 361 insertions, 35 deletions

diff --git a/api/resources/env_action.py b/api/resources/env_action.py
index 2d3496cbc..ea94bc123 100644
--- a/api/resources/env_action.py
+++ b/api/resources/env_action.py
@@ -27,6 +27,7 @@ from api.utils.common import result_handler
 from docker import Client
 from yardstick.common import constants as config
 from yardstick.common import utils as yardstick_utils
+from yardstick.common import openstack_utils
 from yardstick.common.httpClient import HttpClient
 
 logger = logging.getLogger(__name__)
@@ -263,7 +264,7 @@ def _get_remote_rc_file(rc_file, installer_ip, installer_type):
 
 
 def _append_external_network(rc_file):
-    neutron_client = yardstick_utils.get_neutron_client()
+    neutron_client = openstack_utils.get_neutron_client()
     networks = neutron_client.list_networks()['networks']
     try:
         ext_network = next(n['name'] for n in networks if n['router:external'])
diff --git a/tests/unit/common/test_openstack_utils.py b/tests/unit/common/test_openstack_utils.py
index d610e181c..b3dc2d9c4 100644
--- a/tests/unit/common/test_openstack_utils.py
+++ b/tests/unit/common/test_openstack_utils.py
@@ -22,21 +22,25 @@ class GetCredentialsTestCase(unittest.TestCase):
 
     @mock.patch('yardstick.common.openstack_utils.os')
     def test_get_credentials(self, mock_os):
-        mock_os.getenv.return_value = ('2')
-        openstack_utils.get_credentials()
+        with mock.patch.dict('os.environ', {'OS_IDENTITY_API_VERSION': '2'},
+                             clear=True):
+            openstack_utils.get_credentials()
 
 
 class GetHeatApiVersionTestCase(unittest.TestCase):
 
-    @mock.patch('yardstick.common.openstack_utils.os')
-    def test_get_heat_api_version(self, mock_os):
+    def test_get_heat_api_version_check_result(self):
         API = 'HEAT_API_VERSION'
-        openstack_utils.get_heat_api_version()
-        mock_os.getenv.assert_called_with(API)
-
-    @mock.patch('yardstick.common.openstack_utils.os')
-    def test_get_heat_api_version(self, mock_os):
-        mock_os.getenv.return_value = ('2')
         expected_result = '2'
-        api_version = openstack_utils.get_heat_api_version()
-        self.assertEqual(api_version, expected_result)
+
+        with mock.patch.dict('os.environ', {API: '2'}, clear=True):
+            api_version = openstack_utils.get_heat_api_version()
+            self.assertEqual(api_version, expected_result)
+
+
+def main():
+    unittest.main()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/yardstick/common/openstack_utils.py b/yardstick/common/openstack_utils.py
index 5026e819d..aa369b896 100644
--- a/yardstick/common/openstack_utils.py
+++ b/yardstick/common/openstack_utils.py
@@ -8,17 +8,26 @@
 ##############################################################################
 
 from __future__ import absolute_import
+
 import os
+import time
 import logging
 
 from keystoneauth1 import loading
 from keystoneauth1 import session
+from novaclient import client as novaclient
+from glanceclient import client as glanceclient
+from neutronclient.neutron import client as neutronclient
 
 log = logging.getLogger(__name__)
 
 DEFAULT_HEAT_API_VERSION = '1'
+DEFAULT_API_VERSION = '2'
 
 
+# *********************************************
+#   CREDENTIALS
+# *********************************************
 def get_credentials():
     """Returns a creds dictionary filled with parsed from env"""
     creds = {}
@@ -64,7 +73,7 @@ def get_credentials():
                       "ca_file": cacert})
         creds.update({"insecure": "True", "https_insecure": "True"})
         if not os.path.isfile(cacert):
-            log.info("WARNING: The 'OS_CACERT' environment variable is set\
+            log.info("WARNING: The 'OS_CACERT' environment variable is set \
                       to %s but the file does not exist.", cacert)
 
     return creds
@@ -89,9 +98,338 @@ def get_endpoint(service_type, endpoint_type='publicURL'):
                                       endpoint_type=endpoint_type)
 
 
-def get_heat_api_version():
-    api_version = os.getenv('HEAT_API_VERSION')
-    if api_version is not None:
+# *********************************************
+#   CLIENTS
+# *********************************************
+def get_heat_api_version():     # pragma: no cover
+    try:
+        api_version = os.environ['HEAT_API_VERSION']
+    except KeyError:
+        return DEFAULT_HEAT_API_VERSION
+    else:
         log.info("HEAT_API_VERSION is set in env as '%s'", api_version)
         return api_version
-    return DEFAULT_HEAT_API_VERSION
+
+
+def get_nova_client_version():      # pragma: no cover
+    try:
+        api_version = os.environ['OS_COMPUTE_API_VERSION']
+    except KeyError:
+        return DEFAULT_API_VERSION
+    else:
+        log.info("OS_COMPUTE_API_VERSION is set in env as '%s'", api_version)
+        return api_version
+
+
+def get_nova_client():      # pragma: no cover
+    sess = get_session()
+    return novaclient.Client(get_nova_client_version(), session=sess)
+
+
+def get_neutron_client_version():   # pragma: no cover
+    try:
+        api_version = os.environ['OS_NETWORK_API_VERSION']
+    except KeyError:
+        return DEFAULT_API_VERSION
+    else:
+        log.info("OS_NETWORK_API_VERSION is set in env as '%s'", api_version)
+        return api_version
+
+
+def get_neutron_client():   # pragma: no cover
+    sess = get_session()
+    return neutronclient.Client(get_neutron_client_version(), session=sess)
+
+
+def get_glance_client_version():    # pragma: no cover
+    try:
+        api_version = os.environ['OS_IMAGE_API_VERSION']
+    except KeyError:
+        return DEFAULT_API_VERSION
+    else:
+        log.info("OS_IMAGE_API_VERSION is set in env as '%s'", api_version)
+        return api_version
+
+
+def get_glance_client():    # pragma: no cover
+    sess = get_session()
+    return glanceclient.Client(get_glance_client_version(), session=sess)
+
+
+# *********************************************
+#   NOVA
+# *********************************************
+def get_instances(nova_client):     # pragma: no cover
+    try:
+        return nova_client.servers.list(search_opts={'all_tenants': 1})
+    except Exception:
+        log.exception("Error [get_instances(nova_client)]")
+
+
+def get_instance_status(nova_client, instance):     # pragma: no cover
+    try:
+        return nova_client.servers.get(instance.id).status
+    except Exception:
+        log.exception("Error [get_instance_status(nova_client)]")
+
+
+def get_instance_by_name(nova_client, instance_name):   # pragma: no cover
+    try:
+        return nova_client.servers.find(name=instance_name)
+    except Exception:
+        log.exception("Error [get_instance_by_name(nova_client, '%s')]",
+                      instance_name)
+
+
+def get_aggregates(nova_client):    # pragma: no cover
+    try:
+        return nova_client.aggregates.list()
+    except Exception:
+        log.exception("Error [get_aggregates(nova_client)]")
+
+
+def get_availability_zones(nova_client):    # pragma: no cover
+    try:
+        return nova_client.availability_zones.list()
+    except Exception:
+        log.exception("Error [get_availability_zones(nova_client)]")
+
+
+def get_availability_zone_names(nova_client):   # pragma: no cover
+    try:
+        return [az.zoneName for az in get_availability_zones(nova_client)]
+    except Exception:
+        log.exception("Error [get_availability_zone_names(nova_client)]")
+
+
+def create_aggregate(nova_client, aggregate_name, av_zone):  # pragma: no cover
+    try:
+        nova_client.aggregates.create(aggregate_name, av_zone)
+    except Exception:
+        log.exception("Error [create_aggregate(nova_client, %s, %s)]",
+                      aggregate_name, av_zone)
+        return False
+    else:
+        return True
+
+
+def get_aggregate_id(nova_client, aggregate_name):      # pragma: no cover
+    try:
+        aggregates = get_aggregates(nova_client)
+        _id = next((ag.id for ag in aggregates if ag.name == aggregate_name))
+    except Exception:
+        log.exception("Error [get_aggregate_id(nova_client, %s)]",
+                      aggregate_name)
+    else:
+        return _id
+
+
+def add_host_to_aggregate(nova_client, aggregate_name,
+                          compute_host):    # pragma: no cover
+    try:
+        aggregate_id = get_aggregate_id(nova_client, aggregate_name)
+        nova_client.aggregates.add_host(aggregate_id, compute_host)
+    except Exception:
+        log.exception("Error [add_host_to_aggregate(nova_client, %s, %s)]",
+                      aggregate_name, compute_host)
+        return False
+    else:
+        return True
+
+
+def create_aggregate_with_host(nova_client, aggregate_name, av_zone,
+                               compute_host):    # pragma: no cover
+    try:
+        create_aggregate(nova_client, aggregate_name, av_zone)
+        add_host_to_aggregate(nova_client, aggregate_name, compute_host)
+    except Exception:
+        log.exception("Error [create_aggregate_with_host("
+                      "nova_client, %s, %s, %s)]",
+                      aggregate_name, av_zone, compute_host)
+        return False
+    else:
+        return True
+
+
+def create_instance(flavor_name,
+                    image_id,
+                    network_id,
+                    instance_name="instance-vm",
+                    confdrive=True,
+                    userdata=None,
+                    av_zone='',
+                    fixed_ip=None,
+                    files=None):    # pragma: no cover
+    nova_client = get_nova_client()
+    try:
+        flavor = nova_client.flavors.find(name=flavor_name)
+    except:
+        flavors = nova_client.flavors.list()
+        log.exception("Error: Flavor '%s' not found. Available flavors are: "
+                      "\n%s", flavor_name, flavors)
+        return None
+    if fixed_ip is not None:
+        nics = {"net-id": network_id, "v4-fixed-ip": fixed_ip}
+    else:
+        nics = {"net-id": network_id}
+    if userdata is None:
+        instance = nova_client.servers.create(
+            name=instance_name,
+            flavor=flavor,
+            image=image_id,
+            nics=[nics],
+            availability_zone=av_zone,
+            files=files
+        )
+    else:
+        instance = nova_client.servers.create(
+            name=instance_name,
+            flavor=flavor,
+            image=image_id,
+            nics=[nics],
+            config_drive=confdrive,
+            userdata=userdata,
+            availability_zone=av_zone,
+            files=files
+        )
+    return instance
+
+
+def create_instance_and_wait_for_active(flavor_name,
+                                        image_id,
+                                        network_id,
+                                        instance_name="instance-vm",
+                                        config_drive=False,
+                                        userdata="",
+                                        av_zone='',
+                                        fixed_ip=None,
+                                        files=None):    # pragma: no cover
+    SLEEP = 3
+    VM_BOOT_TIMEOUT = 180
+    nova_client = get_nova_client()
+    instance = create_instance(flavor_name,
+                               image_id,
+                               network_id,
+                               instance_name,
+                               config_drive,
+                               userdata,
+                               av_zone=av_zone,
+                               fixed_ip=fixed_ip,
+                               files=files)
+    count = VM_BOOT_TIMEOUT / SLEEP
+    for n in range(count, -1, -1):
+        status = get_instance_status(nova_client, instance)
+        if status.lower() == "active":
+            return instance
+        elif status.lower() == "error":
+            log.error("The instance %s went to ERROR status.", instance_name)
+            return None
+        time.sleep(SLEEP)
+    log.error("Timeout booting the instance %s.", instance_name)
+    return None
+
+
+def delete_instance(nova_client, instance_id):      # pragma: no cover
+    try:
+        nova_client.servers.force_delete(instance_id)
+    except Exception:
+        log.exception("Error [delete_instance(nova_client, '%s')]",
+                      instance_id)
+        return False
+    else:
+        return True
+
+
+def remove_host_from_aggregate(nova_client, aggregate_name,
+                               compute_host):  # pragma: no cover
+    try:
+        aggregate_id = get_aggregate_id(nova_client, aggregate_name)
+        nova_client.aggregates.remove_host(aggregate_id, compute_host)
+    except Exception:
+        log.exception("Error remove_host_from_aggregate(nova_client, %s, %s)",
+                      aggregate_name, compute_host)
+        return False
+    else:
+        return True
+
+
+def remove_hosts_from_aggregate(nova_client,
+                                aggregate_name):   # pragma: no cover
+    aggregate_id = get_aggregate_id(nova_client, aggregate_name)
+    hosts = nova_client.aggregates.get(aggregate_id).hosts
+    assert(
+        all(remove_host_from_aggregate(nova_client, aggregate_name, host)
+            for host in hosts))
+
+
+def delete_aggregate(nova_client, aggregate_name):  # pragma: no cover
+    try:
+        remove_hosts_from_aggregate(nova_client, aggregate_name)
+        nova_client.aggregates.delete(aggregate_name)
+    except Exception:
+        log.exception("Error [delete_aggregate(nova_client, %s)]",
+                      aggregate_name)
+        return False
+    else:
+        return True
+
+
+def get_server_by_name(name):   # pragma: no cover
+    try:
+        return get_nova_client().servers.list(search_opts={'name': name})[0]
+    except IndexError:
+        log.exception('Failed to get nova client')
+        raise
+
+
+def get_image_by_name(name):    # pragma: no cover
+    images = get_nova_client().images.list()
+    try:
+        return next((a for a in images if a.name == name))
+    except StopIteration:
+        log.exception('No image matched')
+
+
+def get_flavor_by_name(name):   # pragma: no cover
+    flavors = get_nova_client().flavors.list()
+    try:
+        return next((a for a in flavors if a.name == name))
+    except StopIteration:
+        log.exception('No flavor matched')
+
+
+def check_status(status, name, iterations, interval):   # pragma: no cover
+    for i in range(iterations):
+        try:
+            server = get_server_by_name(name)
+        except IndexError:
+            log.error('Cannot found %s server', name)
+            raise
+
+        if server.status == status:
+            return True
+
+        time.sleep(interval)
+    return False
+
+
+# *********************************************
+#   NEUTRON
+# *********************************************
+def get_network_id(neutron_client, network_name):       # pragma: no cover
+    networks = neutron_client.list_networks()['networks']
+    return next((n['id'] for n in networks if n['name'] == network_name), None)
+
+
+def get_port_id_by_ip(neutron_client, ip_address):      # pragma: no cover
+    ports = neutron_client.list_ports()['ports']
+    return next((i['id'] for i in ports for j in i.get(
+        'fixed_ips') if j['ip_address'] == ip_address), None)
+
+
+# *********************************************
+#   GLANCE
+# *********************************************
+def get_image_id(glance_client, image_name):    # pragma: no cover
+    images = glance_client.images.list()
+    return next((i.id for i in images if i.name == image_name), None)
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index 04536190b..3c5895f1e 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -26,9 +26,6 @@ import sys
 from functools import reduce
 
 import yaml
-from keystoneauth1 import identity
-from keystoneauth1 import session
-from neutronclient.v2_0 import client
 from oslo_utils import importutils
 from oslo_serialization import jsonutils
 
@@ -134,20 +131,6 @@ def source_env(env_file):
     return env
 
 
-def get_openstack_session():
-    auth = identity.Password(auth_url=os.environ.get('OS_AUTH_URL'),
-                             username=os.environ.get('OS_USERNAME'),
-                             password=os.environ.get('OS_PASSWORD'),
-                             tenant_name=os.environ.get('OS_TENANT_NAME'))
-    return session.Session(auth=auth)
-
-
-def get_neutron_client():
-    sess = get_openstack_session()
-    neutron_client = client.Client(session=sess)
-    return neutron_client
-
-
 def read_json_from_file(path):
     with open(path, 'r') as f:
         return jsonutils.load(f)