Rally keystone test scenarios for v3

JIRA: FUNCTEST-652

1. Updated the Rally repository to master
the keystone test scenarios of TAG= 0.7.0 cannot run on Newton
also, 0.7.0 is the last tag that exists
2. Update the Tempest installation with updated rally command
Rally team said "there was no possibility to do it in backward
compatible way(rally verify re-design)"
3. The `rally show` was deprecated[1] in the master and corresponding
logic is updated
4. Update rally commands

[1]
https://github.com/openstack/rally/commit/f2fc3015beb4cfd8053db728765714500db6eeb2

Change-Id: Ief9e36a16dd92e4449c0f76579e7191a93aa3e21
Signed-off-by: helenyao <yaohelan@huawei.com>

5 files changed, 188 insertions, 124 deletions

diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml
index 15e0d3a1..a67b03ed 100755
--- a/functest/ci/config_functest.yaml
+++ b/functest/ci/config_functest.yaml
@@ -88,6 +88,7 @@ onos_sfc:
     image_file_name: firewall_block_image.img
 
 tempest:
+    deployment_name: opnfv-tempest
     identity:
         tenant_name: tempest
         tenant_description: Tenant for Tempest test suite
diff --git a/functest/ci/prepare_env.py b/functest/ci/prepare_env.py
index 3df3a0e0..45e19a46 100755
--- a/functest/ci/prepare_env.py
+++ b/functest/ci/prepare_env.py
@@ -227,32 +227,36 @@ def install_rally():
     rally_conf = os_utils.get_credentials_for_rally()
     with open('rally_conf.json', 'w') as fp:
         json.dump(rally_conf, fp)
-    cmd = "rally deployment create --file=rally_conf.json --name="
-    cmd += CONST.rally_deployment_name
+    cmd = ("rally deployment create "
+           "--file=rally_conf.json --name={}"
+           .format(CONST.rally_deployment_name))
     ft_utils.execute_command(cmd,
-                             error_msg="Problem creating Rally deployment")
-
-    logger.info("Installing tempest from existing repo...")
-    cmd = ("rally verify install --source " +
-           CONST.dir_repo_tempest +
-           " --system-wide")
-    ft_utils.execute_command(cmd,
-                             error_msg="Problem installing Tempest.")
+                             error_msg=("Problem while creating "
+                                        "Rally deployment"))
 
     cmd = "rally deployment check"
     ft_utils.execute_command(cmd,
                              error_msg=("OpenStack not responding or "
                                         "faulty Rally deployment."))
 
-    cmd = "rally show images"
+    cmd = "rally deployment list"
     ft_utils.execute_command(cmd,
                              error_msg=("Problem while listing "
-                                        "OpenStack images."))
+                                        "Rally deployment."))
 
-    cmd = "rally show flavors"
+    cmd = "rally plugin list | head -5"
     ft_utils.execute_command(cmd,
                              error_msg=("Problem while showing "
-                                        "OpenStack flavors."))
+                                        "Rally plugins."))
+
+
+def install_tempest():
+    logger.info("Installing tempest from existing repo...")
+    cmd = ("rally verify create-verifier --source {0} "
+           "--name {1} --type tempest"
+           .format(CONST.dir_repo_tempest, CONST.tempest_deployment_name))
+    ft_utils.execute_command(cmd,
+                             error_msg="Problem while installing Tempest.")
 
 
 def check_environment():
@@ -267,7 +271,7 @@ def check_environment():
             logger.error(msg_not_active)
             sys.exit(1)
 
-    logger.info("Functest environment installed.")
+    logger.info("Functest environment is installed.")
 
 
 def main():
@@ -283,6 +287,7 @@ def main():
         patch_config_file()
         verify_deployment()
         install_rally()
+        install_tempest()
 
         with open(CONST.env_active, "w") as env_file:
             env_file.write("1")
diff --git a/functest/opnfv_tests/openstack/rally/run_rally-cert.py b/functest/opnfv_tests/openstack/rally/run_rally-cert.py
index ec22b52d..b02fd427 100755
--- a/functest/opnfv_tests/openstack/rally/run_rally-cert.py
+++ b/functest/opnfv_tests/openstack/rally/run_rally-cert.py
@@ -1,19 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 #
-# Copyright (c) 2015 Orange
-# guyrodrigue.koffi@orange.com
-# morgan.richomme@orange.com
-# All rights reserved. This program and the accompanying materials
+# Copyright (c) 2015 All rights reserved
+# This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
 #
-# 0.1 (05/2015) initial commit
-# 0.2 (28/09/2015) extract Tempest, format json result, add ceilometer suite
-# 0.3 (19/10/2015) remove Tempest from run_rally
-# and push result into test DB
+# http://www.apache.org/licenses/LICENSE-2.0
 #
-""" tests configuration """
 
 import argparse
 import json
@@ -391,10 +384,11 @@ def run_task(test_name):
         logger.info('No tests for scenario "{}"'.format(test_name))
         return
 
-    cmd_line = ("rally task start --abort-on-sla-failure " +
-                "--task {} ".format(task_file) +
-                "--task-args \"{}\" ".format(build_task_args(test_name)))
-    logger.debug('running command line : {}'.format(cmd_line))
+    cmd_line = ("rally task start --abort-on-sla-failure "
+                "--task {0} "
+                "--task-args \"{1}\""
+                .format(task_file, build_task_args(test_name)))
+    logger.debug('running command line: {}'.format(cmd_line))
 
     p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                          stderr=RALLY_STDERR, shell=True)
@@ -404,10 +398,11 @@ def run_task(test_name):
 
     if task_id is None:
         logger.error('Failed to retrieve task_id, validating task...')
-        cmd_line = ("rally task validate " +
-                    "--task {} ".format(task_file) +
-                    "--task-args \"{}\" ".format(build_task_args(test_name)))
-        logger.debug('running command line : {}'.format(cmd_line))
+        cmd_line = ("rally task validate "
+                    "--task {0} "
+                    "--task-args \"{1}\""
+                    .format(task_file, build_task_args(test_name)))
+        logger.debug('running command line: {}'.format(cmd_line))
         p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
                              stderr=subprocess.STDOUT, shell=True)
         output = get_cmd_output(p)
@@ -425,12 +420,12 @@ def run_task(test_name):
     cmd_line = "rally task report {} --out {}".format(task_id,
                                                       report_html_dir)
 
-    logger.debug('running command line : {}'.format(cmd_line))
+    logger.debug('running command line: {}'.format(cmd_line))
     os.popen(cmd_line)
 
     # get and save rally operation JSON result
     cmd_line = "rally task results %s" % task_id
-    logger.debug('running command line : {}'.format(cmd_line))
+    logger.debug('running command line: {}'.format(cmd_line))
     cmd = os.popen(cmd_line)
     json_results = cmd.read()
     report_json_name = 'opnfv-{}.json'.format(test_name)
diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py
index 6aa39ea9..67b52796 100644
--- a/functest/opnfv_tests/openstack/tempest/conf_utils.py
+++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py
@@ -14,9 +14,10 @@ import shutil
 
 import opnfv.utils.constants as releng_constants
 
+from functest.utils.constants import CONST
 import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_utils as os_utils
-from functest.utils.constants import CONST
+
 
 IMAGE_ID_ALT = None
 FLAVOR_ID_ALT = None
@@ -43,16 +44,17 @@ def configure_tempest(logger, deployment_dir, IMAGE_ID=None, FLAVOR_ID=None):
     """
     Add/update needed parameters into tempest.conf file generated by Rally
     """
-    tempest_conf_file = deployment_dir + "/tempest.conf"
+    tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
     if os.path.isfile(tempest_conf_file):
-        logger.debug("Deleting old tempest.conf file...")
-        os.remove(tempest_conf_file)
-
-    logger.debug("Generating new tempest.conf file...")
-    cmd = "rally verify genconfig"
+        logger.debug("Verifier is already configured.")
+        logger.debug("Reconfiguring the current verifier...")
+        cmd = "rally verify configure-verifier --reconfigure"
+    else:
+        logger.info("Configuring the verifier...")
+        cmd = "rally verify configure-verifier"
     ft_utils.execute_command(cmd)
 
-    logger.debug("Finding tempest.conf file...")
+    logger.debug("Looking for tempest.conf file...")
     if not os.path.isfile(tempest_conf_file):
         logger.error("Tempest configuration file %s NOT found."
                      % tempest_conf_file)
@@ -100,8 +102,8 @@ def configure_tempest(logger, deployment_dir, IMAGE_ID=None, FLAVOR_ID=None):
         config.write(config_file)
 
     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
-    shutil.copyfile(
-        tempest_conf_file, TEMPEST_RESULTS_DIR + '/tempest.conf')
+    shutil.copyfile(tempest_conf_file,
+                    os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 
     return releng_constants.EXIT_OK
 
@@ -114,7 +116,7 @@ def configure_tempest_multisite(logger, deployment_dir):
     configure_tempest(logger, deployment_dir)
 
     logger.debug("Finding tempest.conf file...")
-    tempest_conf_old = os.path.join(deployment_dir, '/tempest.conf')
+    tempest_conf_old = os.path.join(deployment_dir, 'tempest.conf')
     if not os.path.isfile(tempest_conf_old):
         logger.error("Tempest configuration file %s NOT found."
                      % tempest_conf_old)
@@ -122,7 +124,7 @@ def configure_tempest_multisite(logger, deployment_dir):
 
     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
     cur_path = os.path.split(os.path.realpath(__file__))[0]
-    tempest_conf_file = os.path.join(cur_path, '/tempest_multisite.conf')
+    tempest_conf_file = os.path.join(cur_path, 'tempest_multisite.conf')
     shutil.copyfile(tempest_conf_old, tempest_conf_file)
 
     logger.debug("Updating selected tempest.conf parameters...")
diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 20b1ebb4..0014b718 100644
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -16,8 +16,8 @@ import time
 
 import yaml
 
-import conf_utils
-import functest.core.testcase_base as testcase_base
+from functest.core import testcase_base
+from functest.opnfv_tests.openstack.tempest import conf_utils
 from functest.utils.constants import CONST
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
@@ -35,12 +35,33 @@ class TempestCommon(testcase_base.TestcaseBase):
         self.OPTION = ""
         self.FLAVOR_ID = None
         self.IMAGE_ID = None
-        self.DEPLOYMENT_DIR = self.get_deployment_dir()
+        self.VERIFIER_ID = self.get_verifier_id()
+        self.VERIFIER_REPO_DIR = self.get_verifier_repo_dir()
+        self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
+        self.DEPLOYMENT_DIR = self.get_verifier_deployment_dir()
+        self.VERIFICATION_ID = None
 
     @staticmethod
-    def get_deployment_dir():
+    def get_verifier_id():
         """
-        Returns current Rally deployment directory
+        Returns verifer id for current Tempest
+        """
+        cmd = ("rally verify list-verifiers | awk '/" +
+               CONST.tempest_deployment_name +
+               "/ {print $2}'")
+        p = subprocess.Popen(cmd, shell=True,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT)
+        deployment_uuid = p.stdout.readline().rstrip()
+        if deployment_uuid == "":
+            logger.error("Tempest verifier not found.")
+            raise Exception('Error with command:%s' % cmd)
+        return deployment_uuid
+
+    @staticmethod
+    def get_verifier_deployment_id():
+        """
+        Returns deployment id for active Rally deployment
         """
         cmd = ("rally deployment list | awk '/" +
                CONST.rally_deployment_name +
@@ -51,9 +72,35 @@ class TempestCommon(testcase_base.TestcaseBase):
         deployment_uuid = p.stdout.readline().rstrip()
         if deployment_uuid == "":
             logger.error("Rally deployment not found.")
-            exit(-1)
+            raise Exception('Error with command:%s' % cmd)
+        return deployment_uuid
+
+    def get_verifier_repo_dir(self):
+        """
+        Returns installed verfier repo directory for Tempest
+        """
+        if not self.VERIFIER_ID:
+            self.VERIFIER_ID = self.get_verifier_id()
+
+        return os.path.join(CONST.dir_rally_inst,
+                            'verification',
+                            'verifier-{}'.format(self.VERIFIER_ID),
+                            'repo')
+
+    def get_verifier_deployment_dir(self):
+        """
+        Returns Rally deployment directory for current verifier
+        """
+        if not self.VERIFIER_ID:
+            self.VERIFIER_ID = self.get_verifier_id()
+
+        if not self.DEPLOYMENT_ID:
+            self.DEPLOYMENT_ID = self.get_verifier_deployment_id()
+
         return os.path.join(CONST.dir_rally_inst,
-                            "tempest/for-deployment-" + deployment_uuid)
+                            'verification',
+                            'verifier-{}'.format(self.VERIFIER_ID),
+                            'for-deployment-{}'.format(self.DEPLOYMENT_ID))
 
     @staticmethod
     def read_file(filename):
@@ -81,12 +128,11 @@ class TempestCommon(testcase_base.TestcaseBase):
                          CONST.tempest_identity_user_name)
 
         logger.debug("Creating private network for Tempest suite")
-        network_dic = \
-            os_utils.create_shared_network_full(
-                CONST.tempest_private_net_name,
-                CONST.tempest_private_subnet_name,
-                CONST.tempest_router_name,
-                CONST.tempest_private_subnet_cidr)
+        network_dic = os_utils.create_shared_network_full(
+            CONST.tempest_private_net_name,
+            CONST.tempest_private_subnet_name,
+            CONST.tempest_router_name,
+            CONST.tempest_private_subnet_cidr)
         if not network_dic:
             return testcase_base.TestcaseBase.EX_RUN_ERROR
 
@@ -112,7 +158,7 @@ class TempestCommon(testcase_base.TestcaseBase):
 
         return testcase_base.TestcaseBase.EX_OK
 
-    def generate_test_list(self, DEPLOYMENT_DIR):
+    def generate_test_list(self, verifier_repo_dir):
         logger.debug("Generating test case list...")
         if self.MODE == 'defcore':
             shutil.copyfile(
@@ -134,8 +180,11 @@ class TempestCommon(testcase_base.TestcaseBase):
                 testr_mode = ""
             else:
                 testr_mode = 'tempest.api.' + self.MODE
-            cmd = ("cd " + DEPLOYMENT_DIR + ";" + "testr list-tests " +
-                   testr_mode + ">" + conf_utils.TEMPEST_RAW_LIST + ";cd")
+            cmd = ("cd {0};"
+                   "testr list-tests {1} > {2};"
+                   "cd -;".format(verifier_repo_dir,
+                                  testr_mode,
+                                  conf_utils.TEMPEST_RAW_LIST))
             ft_utils.execute_command(cmd)
 
         return testcase_base.TestcaseBase.EX_OK
@@ -163,7 +212,7 @@ class TempestCommon(testcase_base.TestcaseBase):
                         for test in tests:
                             black_tests.append(test)
                         break
-        except:
+        except Exception:
             black_tests = []
             logger.debug("Tempest blacklist file does not exist.")
 
@@ -176,36 +225,15 @@ class TempestCommon(testcase_base.TestcaseBase):
         result_file.close()
         return testcase_base.TestcaseBase.EX_OK
 
-    def run(self):
-
-        self.start_time = time.time()
-
-        if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
-            os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
-
-        # Pre-configuration
-        res = self.create_tempest_resources()
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = conf_utils.configure_tempest(logger,
-                                           self.DEPLOYMENT_DIR,
-                                           self.IMAGE_ID,
-                                           self.FLAVOR_ID)
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = self.generate_test_list(self.DEPLOYMENT_DIR)
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
-
-        res = self.apply_tempest_blacklist()
-        if res != testcase_base.TestcaseBase.EX_OK:
-            return res
+    def _parse_verification_id(line):
+        first_pos = line.index("UUID=") + len("UUID=")
+        last_pos = line.index(") for deployment")
+        return line[first_pos:last_pos]
 
-        self.OPTION += (" --tests-file %s " % conf_utils.TEMPEST_LIST)
+    def run_verifier_tests(self):
+        self.OPTION += (" --load-list {}".format(conf_utils.TEMPEST_LIST))
 
-        cmd_line = "rally verify start " + self.OPTION + " --system-wide"
+        cmd_line = "rally verify start " + self.OPTION
         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 
         header = ("Tempest environment:\n"
@@ -215,14 +243,15 @@ class TempestCommon(testcase_base.TestcaseBase):
                    CONST.NODE_NAME,
                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 
-        f_stdout = open(conf_utils.TEMPEST_RESULTS_DIR + "/tempest.log", 'w+')
+        f_stdout = open(
+            os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
         f_stderr = open(
-            conf_utils.TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+')
-        f_env = open(conf_utils.TEMPEST_RESULTS_DIR + "/environment.log", 'w+')
+            os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
+                         "tempest-error.log"), 'w+')
+        f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
+                                  "environment.log"), 'w+')
         f_env.write(header)
 
-        # subprocess.call(cmd_line, shell=True,
-        # stdout=f_stdout, stderr=f_stderr)
         p = subprocess.Popen(
             cmd_line, shell=True,
             stdout=subprocess.PIPE,
@@ -233,6 +262,12 @@ class TempestCommon(testcase_base.TestcaseBase):
             for line in iter(p.stdout.readline, b''):
                 if re.search("\} tempest\.", line):
                     logger.info(line.replace('\n', ''))
+                elif re.search('Starting verification', line):
+                    logger.info(line.replace('\n', ''))
+                    first_pos = line.index("UUID=") + len("UUID=")
+                    last_pos = line.index(") for deployment")
+                    self.VERIFICATION_ID = line[first_pos:last_pos]
+                    logger.debug('Verication UUID: %s' % self.VERIFICATION_ID)
                 f_stdout.write(line)
         p.wait()
 
@@ -240,37 +275,33 @@ class TempestCommon(testcase_base.TestcaseBase):
         f_stderr.close()
         f_env.close()
 
-        cmd_line = "rally verify show"
-        output = ""
+    def parse_verifier_result(self):
+        if not self.VERIFICATION_ID:
+            raise Exception('Verification UUID not found')
+
+        cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
+        logger.info("Showing result for a verification: '%s'." % cmd_line)
         p = subprocess.Popen(cmd_line,
                              shell=True,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
+                             stderr=subprocess.STDOUT)
         for line in p.stdout:
-            if re.search("Tests\:", line):
+            new_line = line.replace(' ', '').split('|')
+            if 'Tests' in new_line:
                 break
-            output += line
-        logger.info(output)
-
-        cmd_line = "rally verify list"
-        cmd = os.popen(cmd_line)
-        output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|")
-        # Format:
-        # | UUID | Deployment UUID | smoke | tests | failures | Created at |
-        # Duration | Status  |
-        num_tests = output[4]
-        num_failures = output[5]
-        duration = output[7]
-        # Compute duration (lets assume it does not take more than 60 min)
-        dur_min = int(duration.split(':')[1])
-        dur_sec_float = float(duration.split(':')[2])
-        dur_sec_int = int(round(dur_sec_float, 0))
-        dur_sec_int = dur_sec_int + 60 * dur_min
+
+            logger.info(line)
+            if 'Testscount' in new_line:
+                num_tests = new_line[2]
+            elif 'Success' in new_line:
+                num_success = new_line[2]
+            elif 'Skipped' in new_line:
+                num_skipped = new_line[2]
 
         try:
-            diff = (int(num_tests) - int(num_failures))
-            success_rate = 100 * diff / int(num_tests)
-        except:
+            num_executed = int(num_tests) - int(num_skipped)
+            success_rate = 100 * int(num_success) / int(num_executed)
+        except Exception:
             success_rate = 0
 
         self.criteria = ft_utils.check_success_rate(
@@ -278,6 +309,36 @@ class TempestCommon(testcase_base.TestcaseBase):
         logger.info("Tempest %s success_rate is %s%%, is marked as %s"
                     % (self.case_name, success_rate, self.criteria))
 
+    def run(self):
+
+        self.start_time = time.time()
+
+        if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
+            os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
+
+        # Pre-configuration
+        res = self.create_tempest_resources()
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = conf_utils.configure_tempest(logger,
+                                           self.DEPLOYMENT_DIR,
+                                           self.IMAGE_ID,
+                                           self.FLAVOR_ID)
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = self.generate_test_list(self.VERIFIER_REPO_DIR)
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        res = self.apply_tempest_blacklist()
+        if res != testcase_base.TestcaseBase.EX_OK:
+            return res
+
+        self.run_verifier_tests()
+        self.parse_verifier_result()
+
         self.stop_time = time.time()
 
         if self.criteria == "PASS":
@@ -292,7 +353,7 @@ class TempestSmokeSerial(TempestCommon):
         TempestCommon.__init__(self)
         self.case_name = "tempest_smoke_serial"
         self.MODE = "smoke"
-        self.OPTION = "--concur 1"
+        self.OPTION = "--concurrency 1"
 
 
 class TempestSmokeParallel(TempestCommon):
@@ -318,7 +379,7 @@ class TempestMultisite(TempestCommon):
         TempestCommon.__init__(self)
         self.case_name = "multisite"
         self.MODE = "feature_multisite"
-        self.OPTION = "--concur 1"
+        self.OPTION = "--concurrency 1"
         conf_utils.configure_tempest_multisite(logger, self.DEPLOYMENT_DIR)