Merge "Rally keystone test scenarios for v3"

5 files changed, 188 insertions, 124 deletions

diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml
index 15e0d3a1..a67b03ed 100755
--- a/functest/ci/config_functest.yaml
+++ b/functest/ci/config_functest.yaml
@@ -88,6 +88,7 @@ onos_sfc:
     image_file_name: firewall_block_image.img
 
 tempest:
+    deployment_name: opnfv-tempest
     identity:
         tenant_name: tempest
         tenant_description: Tenant for Tempest test suite
diff --git a/functest/ci/prepare_env.py b/functest/ci/prepare_env.py
index 77bb14a8..74c751af 100755
--- a/functest/ci/prepare_env.py
+++ b/functest/ci/prepare_env.py
@@ -234,32 +234,36 @@ def install_rally():
     rally_conf = os_utils.get_credentials_for_rally()
     with open('rally_conf.json', 'w') as fp:
         json.dump(rally_conf, fp)
-    cmd = "rally deployment create --file=rally_conf.json --name="
-    cmd += CONST.rally_deployment_name
+    cmd = ("rally deployment create "
+           "--file=rally_conf.json --name={}"
+           .format(CONST.rally_deployment_name))
     ft_utils.execute_command(cmd,
-                             error_msg="Problem creating Rally deployment")
-
-    logger.info("Installing tempest from existing repo...")
-    cmd = ("rally verify install --source " +
-           CONST.dir_repo_tempest +
-           " --system-wide")
-    ft_utils.execute_command(cmd,
-                             error_msg="Problem installing Tempest.")
+                             error_msg=("Problem while creating "
+                                        "Rally deployment"))
 
     cmd = "rally deployment check"
     ft_utils.execute_command(cmd,
                              error_msg=("OpenStack not responding or "
                                         "faulty Rally deployment."))
 
-    cmd = "rally show images"
+    cmd = "rally deployment list"
     ft_utils.execute_command(cmd,
                              error_msg=("Problem while listing "
-                                        "OpenStack images."))
+                                        "Rally deployment."))
 
-    cmd = "rally show flavors"
+    cmd = "rally plugin list | head -5"
     ft_utils.execute_command(cmd,
                              error_msg=("Problem while showing "
-                                        "OpenStack flavors."))
+                                        "Rally plugins."))
+
+
+def install_tempest():
+    logger.info("Installing tempest from existing repo...")
+    cmd = ("rally verify create-verifier --source {0} "
+           "--name {1} --type tempest"
+           .format(CONST.dir_repo_tempest, CONST.tempest_deployment_name))
+    ft_utils.execute_command(cmd,
+                             error_msg="Problem while installing Tempest.")
 
 
 def check_environment():
@@ -274,7 +278,7 @@ def check_environment():
             logger.error(msg_not_active)
             sys.exit(1)
 
-    logger.info("Functest environment installed.")
+    logger.info("Functest environment is installed.")
 
 
 def main(**kwargs):
@@ -290,6 +294,7 @@ def main(**kwargs):
         patch_config_file()
         verify_deployment()
         install_rally()
+        install_tempest()
 
         with open(CONST.env_active, "w") as env_file:
             env_file.write("1")
diff --git a/functest/opnfv_tests/openstack/rally/run_rally-cert.py b/functest/opnfv_tests/openstack/rally/run_rally-cert.py
index ec22b52d..b02fd427 100755
--- a/functest/opnfv_tests/openstack/rally/run_rally-cert.py
+++ b/functest/opnfv_tests/openstack/rally/run_rally-cert.py
@@ -1,19 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 #
-# Copyright (c) 2015 Orange
-# guyrodrigue.koffi@orange.com
-# morgan.richomme@orange.com
-# All rights reserved. This program and the accompanying materials
+# Copyright (c) 2015 All rights reserved
+# This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
 #
-# 0.1 (05/2015) initial commit
-# 0.2 (28/09/2015) extract Tempest, format json result, add ceilometer suite
-# 0.3 (19/10/2015) remove Tempest from run_rally
-# and push result into test DB
+# http://www.apache.org/licenses/LICENSE-2.0
 #
-""" tests configuration """
 
 import argparse
 import json
@@ -391,10 +384,11 @@ def run_task(test_name):
         logger.info('No tests for scenario "{}"'.format(test_name))
         return
 
-    cmd_line = ("rally task start --abort-on-sla-failure " +
-                "--task {} ".format(task_file) +
-                "--task-args \"{}\" ".format(build_task_args(test_name)))
-    logger.debug('running command line : {}'.format(cmd_line))
+    cmd_line = ("rally task start --abort-on-sla-failure "
+                "--task {0} "
+                "--task-args \"{1}\""
+                .format(task_file, build_task_args(test_name)))
+    logger.debug('running command line: {}'.format(cmd_line))
 
     p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                          stderr=RALLY_STDERR, shell=True)
@@ -404,10 +398,11 @@ def run_task(test_name):
 
     if task_id is None:
         logger.error('Failed to retrieve task_id, validating task...')
-        cmd_line = ("rally task validate " +
-                    "--task {} ".format(task_file) +
-                    "--task-args \"{}\" ".format(build_task_args(test_name)))
-        logger.debug('running command line : {}'.format(cmd_line))
+        cmd_line = ("rally task validate "
+                    "--task {0} "
+                    "--task-args \"{1}\""
+                    .format(task_file, build_task_args(test_name)))
+        logger.debug('running command line: {}'.format(cmd_line))
         p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT, shell=True)
         output = get_cmd_output(p)
@@ -425,12 +420,12 @@ def run_task(test_name):
     cmd_line = "rally task report {} --out {}".format(task_id,
                                                       report_html_dir)
 
-    logger.debug('running command line : {}'.format(cmd_line))
+    logger.debug('running command line: {}'.format(cmd_line))
     os.popen(cmd_line)
 
     # get and save rally operation JSON result
     cmd_line = "rally task results %s" % task_id
-    logger.debug('running command line : {}'.format(cmd_line))
+    logger.debug('running command line: {}'.format(cmd_line))
     cmd = os.popen(cmd_line)
     json_results = cmd.read()
     report_json_name = 'opnfv-{}.json'.format(test_name)
diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py
index 6aa39ea9..67b52796 100644
--- a/functest/opnfv_tests/openstack/tempest/conf_utils.py
+++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py
@@ -14,9 +14,10 @@ import shutil
 
 import opnfv.utils.constants as releng_constants
 
+from functest.utils.constants import CONST
 import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_utils as os_utils
-from functest.utils.constants import CONST
+
 
 IMAGE_ID_ALT = None
 FLAVOR_ID_ALT = None
@@ -43,16 +44,17 @@ def configure_tempest(logger, deployment_dir, IMAGE_ID=None, FLAVOR_ID=None):
     """
     Add/update needed parameters into tempest.conf file generated by Rally
     """
-    tempest_conf_file = deployment_dir + "/tempest.conf"
+    tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
     if os.path.isfile(tempest_conf_file):
-        logger.debug("Deleting old tempest.conf file...")
-        os.remove(tempest_conf_file)
-
-    logger.debug("Generating new tempest.conf file...")
-    cmd = "rally verify genconfig"
+        logger.debug("Verifier is already configured.")
+        logger.debug("Reconfiguring the current verifier...")
+        cmd = "rally verify configure-verifier --reconfigure"
+    else:
+        logger.info("Configuring the verifier...")
+        cmd = "rally verify configure-verifier"
     ft_utils.execute_command(cmd)
 
-    logger.debug("Finding tempest.conf file...")
+    logger.debug("Looking for tempest.conf file...")
     if not os.path.isfile(tempest_conf_file):
         logger.error("Tempest configuration file %s NOT found."
                      % tempest_conf_file)
@@ -100,8 +102,8 @@ def configure_tempest(logger, deployment_dir, IMAGE_ID=None, FLAVOR_ID=None):
         config.write(config_file)
 
     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
-    shutil.copyfile(
-        tempest_conf_file, TEMPEST_RESULTS_DIR + '/tempest.conf')
+    shutil.copyfile(tempest_conf_file,
+                    os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 
     return releng_constants.EXIT_OK
 
@@ -114,7 +116,7 @@ def configure_tempest_multisite(logger, deployment_dir):
     configure_tempest(logger, deployment_dir)
 
     logger.debug("Finding tempest.conf file...")
-    tempest_conf_old = os.path.join(deployment_dir, '/tempest.conf')
+    tempest_conf_old = os.path.join(deployment_dir, 'tempest.conf')
     if not os.path.isfile(tempest_conf_old):
         logger.error("Tempest configuration file %s NOT found."
                      % tempest_conf_old)
@@ -122,7 +124,7 @@ def configure_tempest_multisite(logger, deployment_dir):
 
     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
     cur_path = os.path.split(os.path.realpath(__file__))[0]
-    tempest_conf_file = os.path.join(cur_path, '/tempest_multisite.conf')
+    tempest_conf_file = os.path.join(cur_path, 'tempest_multisite.conf')
     shutil.copyfile(tempest_conf_old, tempest_conf_file)
 
     logger.debug("Updating selected tempest.conf parameters...")
diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 20b1ebb4..0014b718 100644
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -16,8 +16,8 @@ import time
 
 import yaml
 
-import conf_utils
-import functest.core.testcase_base as testcase_base
+from functest.core import testcase_base
+from functest.opnfv_tests.openstack.tempest import conf_utils
 from functest.utils.constants import CONST
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
@@ -35,12 +35,33 @@ class TempestCommon(testcase_base.TestcaseBase):
         self.OPTION = ""
         self.FLAVOR_ID = None
         self.IMAGE_ID = None
-        self.DEPLOYMENT_DIR = self.get_deployment_dir()
+        self.VERIFIER_ID = self.get_verifier_id()
+        self.VERIFIER_REPO_DIR = self.get_verifier_repo_dir()
+        self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
+        self.DEPLOYMENT_DIR = self.get_verifier_deployment_dir()
+        self.VERIFICATION_ID = None
 
     @staticmethod
-    def get_deployment_dir():
+    def get_verifier_id():
         """
-        Returns current Rally deployment directory
+        Returns verifer id for current Tempest
+        """
+        cmd = ("rally verify list-verifiers | awk '/" +
+               CONST.tempest_deployment_name +
+               "/ {print $2}'")
+        p = subprocess.Popen(cmd, shell=True,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        deployment_uuid = p.stdout.readline().rstrip()
+        if deployment_uuid == "":
+            logger.error("Tempest verifier not found.")
+            raise Exception('Error with command:%s' % cmd)
+        return deployment_uuid
+
+    @staticmethod
+    def get_verifier_deployment_id():
+        """
+        Returns deployment id for active Rally deployment
         """
         cmd = ("rally deployment list | awk '/" +
                CONST.rally_deployment_name +
@@ -51,9 +72,35 @@ class TempestCommon(testcase_base.TestcaseBase):
         deployment_uuid = p.stdout.readline().rstrip()
         if deployment_uuid == "":
             logger.error("Rally deployment not found.")
-            exit(-1)
+            raise Exception('Error with command:%s' % cmd)
+        return deployment_uuid
+
+    def get_verifier_repo_dir(self):
+        """
+        Returns installed verfier repo directory for Tempest
+        """
+        if not self.VERIFIER_ID:
+            self.VERIFIER_ID = self.get_verifier_id()
+
+        return os.path.join(CONST.dir_rally_inst,
+                            'verification',
+                            'verifier-{}'.format(self.VERIFIER_ID),
+                            'repo')
+
+    def get_verifier_deployment_dir(self):
+        """
+        Returns Rally deployment directory for current verifier
+        """
+        if not self.VERIFIER_ID:
+            self.VERIFIER_ID = self.get_verifier_id()
+
+        if not self.DEPLOYMENT_ID:
+            self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
+
         return os.path.join(CONST.dir_rally_inst,
-                            "tempest/for-deployment-" + deployment_uuid)
+                            'verification',
+                            'verifier-{}'.format(self.VERIFIER_ID),
+                            'for-deployment-{}'.format(self.DEPLOYMENT_ID))
 
     @staticmethod
     def read_file(filename):
@@ -81,12 +128,11 @@ class TempestCommon(testcase_base.TestcaseBase):
                          CONST.tempest_identity_user_name)
 
         logger.debug("Creating private network for Tempest suite")
-        network_dic = \
-            os_utils.create_shared_network_full(
-                CONST.tempest_private_net_name,
-                CONST.tempest_private_subnet_name,
-                CONST.tempest_router_name,
-                CONST.tempest_private_subnet_cidr)
+        network_dic = os_utils.create_shared_network_full(
+            CONST.tempest_private_net_name,
+            CONST.tempest_private_subnet_name,
+            CONST.tempest_router_name,
+            CONST.tempest_private_subnet_cidr)
         if not network_dic:
             return testcase_base.TestcaseBase.EX_RUN_ERROR
 
@@ -112,7 +158,7 @@ class TempestCommon(testcase_base.TestcaseBase):
 
         return testcase_base.TestcaseBase.EX_OK
 
-    def generate_test_list(self, DEPLOYMENT_DIR):
+    def generate_test_list(self, verifier_repo_dir):
         logger.debug("Generating test case list...")
         if self.MODE == 'defcore':
             shutil.copyfile(
@@ -134,8 +180,11 @@ class TempestCommon(testcase_base.TestcaseBase):
                 testr_mode = ""
             else:
                 testr_mode = 'tempest.api.' + self.MODE
-            cmd = ("cd " + DEPLOYMENT_DIR + ";" + "testr list-tests " +
-                   testr_mode + ">" + conf_utils.TEMPEST_RAW_LIST + ";cd")
+            cmd = ("cd {0};"
+                   "testr list-tests {1} > {2};"
+                   "cd -;".format(verifier_repo_dir,
+                                  testr_mode,
+                                  conf_utils.TEMPEST_RAW_LIST))
             ft_utils.execute_command(cmd)
 
         return testcase_base.TestcaseBase.EX_OK
@@ -163,7 +212,7 @@ class TempestCommon(testcase_base.TestcaseBase):
                         for test in tests:
                             black_tests.append(test)
                         break
-        except:
+        except Exception:
             black_tests = []
             logger.debug("Tempest blacklist file does not exist.")
 
@@ -176,36 +225,15 @@ class TempestCommon(testcase_base.TestcaseBase):
         result_file.close()
         return testcase_base.TestcaseBase.EX_OK
 
-    def run(self):
-
-        self.start_time = time.time()
-
-        if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
-            os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
-
-        # Pre-configuration
-        res = self.create_tempest_resources()
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = conf_utils.configure_tempest(logger,
-                                           self.DEPLOYMENT_DIR,
-                                           self.IMAGE_ID,
-                                           self.FLAVOR_ID)
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = self.generate_test_list(self.DEPLOYMENT_DIR)
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = self.apply_tempest_blacklist()
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
+    def _parse_verification_id(line):
+        first_pos = line.index("UUID=") + len("UUID=")
+        last_pos = line.index(") for deployment")
+        return line[first_pos:last_pos]
 
-        self.OPTION += (" --tests-file %s " % conf_utils.TEMPEST_LIST)
+    def run_verifier_tests(self):
+        self.OPTION += (" --load-list {}".format(conf_utils.TEMPEST_LIST))
 
-        cmd_line = "rally verify start " + self.OPTION + " --system-wide"
+        cmd_line = "rally verify start " + self.OPTION
         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 
         header = ("Tempest environment:\n"
@@ -215,14 +243,15 @@ class TempestCommon(testcase_base.TestcaseBase):
                    CONST.NODE_NAME,
                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 
-        f_stdout = open(conf_utils.TEMPEST_RESULTS_DIR + "/tempest.log", 'w+')
+        f_stdout = open(
+            os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
         f_stderr = open(
-            conf_utils.TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+')
-        f_env = open(conf_utils.TEMPEST_RESULTS_DIR + "/environment.log", 'w+')
+            os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
+                         "tempest-error.log"), 'w+')
+        f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
+                                  "environment.log"), 'w+')
         f_env.write(header)
 
-        # subprocess.call(cmd_line, shell=True,
-        # stdout=f_stdout, stderr=f_stderr)
         p = subprocess.Popen(
             cmd_line, shell=True,
             stdout=subprocess.PIPE,
@@ -233,6 +262,12 @@ class TempestCommon(testcase_base.TestcaseBase):
             for line in iter(p.stdout.readline, b''):
                 if re.search("\} tempest\.", line):
                     logger.info(line.replace('\n', ''))
+                elif re.search('Starting verification', line):
+                    logger.info(line.replace('\n', ''))
+                    first_pos = line.index("UUID=") + len("UUID=")
+                    last_pos = line.index(") for deployment")
+                    self.VERIFICATION_ID = line[first_pos:last_pos]
+                    logger.debug('Verication UUID: %s' % self.VERIFICATION_ID)
                 f_stdout.write(line)
         p.wait()
 
@@ -240,37 +275,33 @@ class TempestCommon(testcase_base.TestcaseBase):
         f_stderr.close()
         f_env.close()
 
-        cmd_line = "rally verify show"
-        output = ""
+    def parse_verifier_result(self):
+        if not self.VERIFICATION_ID:
+            raise Exception('Verification UUID not found')
+
+        cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
+        logger.info("Showing result for a verification: '%s'." % cmd_line)
         p = subprocess.Popen(cmd_line,
                              shell=True,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.STDOUT)
         for line in p.stdout:
-            if re.search("Tests\:", line):
+            new_line = line.replace(' ', '').split('|')
+            if 'Tests' in new_line:
                 break
-            output += line
-        logger.info(output)
-
-        cmd_line = "rally verify list"
-        cmd = os.popen(cmd_line)
-        output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|")
-        # Format:
-        # | UUID | Deployment UUID | smoke | tests | failures | Created at |
-        # Duration | Status  |
-        num_tests = output[4]
-        num_failures = output[5]
-        duration = output[7]
-        # Compute duration (lets assume it does not take more than 60 min)
-        dur_min = int(duration.split(':')[1])
-        dur_sec_float = float(duration.split(':')[2])
-        dur_sec_int = int(round(dur_sec_float, 0))
-        dur_sec_int = dur_sec_int + 60 * dur_min
+
+            logger.info(line)
+            if 'Testscount' in new_line:
+                num_tests = new_line[2]
+            elif 'Success' in new_line:
+                num_success = new_line[2]
+            elif 'Skipped' in new_line:
+                num_skipped = new_line[2]
 
         try:
-            diff = (int(num_tests) - int(num_failures))
-            success_rate = 100 * diff / int(num_tests)
-        except:
+            num_executed = int(num_tests) - int(num_skipped)
+            success_rate = 100 * int(num_success) / int(num_executed)
+        except Exception:
             success_rate = 0
 
         self.criteria = ft_utils.check_success_rate(
@@ -278,6 +309,36 @@ class TempestCommon(testcase_base.TestcaseBase):
         logger.info("Tempest %s success_rate is %s%%, is marked as %s"
                     % (self.case_name, success_rate, self.criteria))
 
+    def run(self):
+
+        self.start_time = time.time()
+
+        if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
+            os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
+
+        # Pre-configuration
+        res = self.create_tempest_resources()
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = conf_utils.configure_tempest(logger,
+                                           self.DEPLOYMENT_DIR,
+                                           self.IMAGE_ID,
+                                           self.FLAVOR_ID)
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = self.generate_test_list(self.VERIFIER_REPO_DIR)
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = self.apply_tempest_blacklist()
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        self.run_verifier_tests()
+        self.parse_verifier_result()
+
         self.stop_time = time.time()
 
         if self.criteria == "PASS":
@@ -292,7 +353,7 @@ class TempestSmokeSerial(TempestCommon):
         TempestCommon.__init__(self)
         self.case_name = "tempest_smoke_serial"
         self.MODE = "smoke"
-        self.OPTION = "--concur 1"
+        self.OPTION = "--concurrency 1"
 
 
 class TempestSmokeParallel(TempestCommon):
@@ -318,7 +379,7 @@ class TempestMultisite(TempestCommon):
         TempestCommon.__init__(self)
         self.case_name = "multisite"
         self.MODE = "feature_multisite"
-        self.OPTION = "--concur 1"
+        self.OPTION = "--concurrency 1"
         conf_utils.configure_tempest_multisite(logger, self.DEPLOYMENT_DIR)