Added handling of blacklist for tempest

It is now possible to exclude some specific test cases ("black list")
from the test case suites.

JIRA: FUNCTEST-197

Change-Id: I2946a010793da0a3d3ef1d96aa0ca0840e8a761d
Signed-off-by: vitikkan <viktor.tikkanen@nokia.com>

1 files changed, 72 insertions, 39 deletions

diff --git a/testcases/VIM/OpenStack/CI/libraries/run_tempest.py b/testcases/VIM/OpenStack/CI/libraries/run_tempest.py
index b042b49c3..9eca6cbd8 100644
--- a/testcases/VIM/OpenStack/CI/libraries/run_tempest.py
+++ b/testcases/VIM/OpenStack/CI/libraries/run_tempest.py
@@ -85,8 +85,11 @@ RESULTS_DIR = functest_yaml.get("general").get("directories").get(
 TEMPEST_RESULTS_DIR = RESULTS_DIR + '/tempest'
 TEST_LIST_DIR = functest_yaml.get("general").get("directories").get(
     "dir_tempest_cases")
-TEMPEST_LIST_FILE = REPO_PATH + TEST_LIST_DIR + 'test_list.txt'
+TEMPEST_CUSTOM = REPO_PATH + TEST_LIST_DIR + 'test_list.txt'
+TEMPEST_BLACKLIST = REPO_PATH + TEST_LIST_DIR + 'blacklist.txt'
 TEMPEST_DEFCORE = REPO_PATH + TEST_LIST_DIR + 'defcore_req.txt'
+TEMPEST_RAW_LIST = TEMPEST_RESULTS_DIR + '/test_raw_list.txt'
+TEMPEST_LIST = TEMPEST_RESULTS_DIR + '/test_list.txt'
 
 
 def get_info(file_result):
@@ -169,15 +172,7 @@ def create_tempest_resources():
         exit(-1)
 
 
-def configure_tempest(mode):
-    """
-    Add/update needed parameters into tempest.conf file generated by Rally
-    """
-
-    logger.debug("Generating tempest.conf file...")
-    cmd = "rally verify genconfig"
-    ft_utils.execute_command(cmd, logger)
-
+def get_deployment_dir():
     logger.debug("Resolving deployment UUID and directory...")
     cmd = "rally deployment list | awk '/" + DEPLOYMENT_MAME + "/ {print $2}'"
     p = subprocess.Popen(cmd, shell=True,
@@ -185,26 +180,28 @@ def configure_tempest(mode):
                          stderr=subprocess.STDOUT)
     deployment_uuid = p.stdout.readline().rstrip()
     if deployment_uuid == "":
-        logger.debug("   Rally deployment NOT found")
-        return False
+        logger.error("Rally deployment NOT found.")
+        exit(-1)
     deployment_dir = (RALLY_INSTALLATION_DIR + "/tempest/for-deployment-" +
                       deployment_uuid)
+    return deployment_dir
+
+
+def configure_tempest(deployment_dir):
+    """
+    Add/update needed parameters into tempest.conf file generated by Rally
+    """
+
+    logger.debug("Generating tempest.conf file...")
+    cmd = "rally verify genconfig"
+    ft_utils.execute_command(cmd, logger)
 
     logger.debug("Finding tempest.conf file...")
     tempest_conf_file = deployment_dir + "/tempest.conf"
     if not os.path.isfile(tempest_conf_file):
-        logger.error("   Tempest configuration file %s NOT found."
+        logger.error("Tempest configuration file %s NOT found."
                      % tempest_conf_file)
-        return False
-
-    logger.debug("Generating test case list...")
-    cmd = "cd " + deployment_dir + ";"
-    if mode == 'smoke':
-        cmd += "testr list-tests smoke >" + TEMPEST_LIST_FILE + ";cd"
-        ft_utils.execute_command(cmd, logger)
-    elif mode == 'full':
-        cmd += "testr list-tests >" + TEMPEST_LIST_FILE + ";cd"
-        ft_utils.execute_command(cmd, logger)
+        exit(-1)
 
     logger.debug("Updating selected tempest.conf parameters...")
     config = ConfigParser.RawConfigParser()
@@ -221,6 +218,49 @@ def configure_tempest(mode):
     return True
 
 
+def read_file(filename):
+    with open(filename) as src:
+        return [line.strip() for line in src.readlines()]
+
+
+def generate_test_list(deployment_dir, mode):
+    logger.debug("Generating test case list...")
+    if mode == 'defcore':
+        shutil.copyfile(TEMPEST_DEFCORE, TEMPEST_RAW_LIST)
+    elif mode == 'custom':
+        if os.path.isfile(TEMPEST_CUSTOM):
+            shutil.copyfile(TEMPEST_CUSTOM, TEMPEST_RAW_LIST)
+        else:
+            logger.error("Tempest test list file %s NOT found."
+                         % TEMPEST_CUSTOM)
+            exit(-1)
+    else:
+        if mode == 'smoke':
+            testr_mode = "smoke"
+        elif mode == 'full':
+            testr_mode = ""
+        else:
+            testr_mode = 'tempest.api.' + mode
+        cmd = ("cd " + deployment_dir + ";" + "testr list-tests " +
+               testr_mode + ">" + TEMPEST_RAW_LIST + ";cd")
+        ft_utils.execute_command(cmd, logger)
+
+
+def apply_tempest_blacklist():
+    logger.debug("Applying tempest blacklist...")
+    cases_file = read_file(TEMPEST_RAW_LIST)
+    result_file = open(TEMPEST_LIST, 'w')
+    try:
+        black_file = read_file(TEMPEST_BLACKLIST)
+    except:
+        black_file = ''
+        logger.debug("Tempest blacklist file does not exist.")
+    for line in cases_file:
+        if line not in black_file:
+            result_file.write(str(line) + '\n')
+    result_file.close()
+
+
 def run_tempest(OPTION):
     #
     # the "main" function of the script which launches Rally to run Tempest
@@ -229,8 +269,6 @@ def run_tempest(OPTION):
     #
     logger.info("Starting Tempest test suite: '%s'." % OPTION)
     cmd_line = "rally verify start " + OPTION + " --system-wide"
-    logger.debug('Executing command : {}'.format(cmd_line))
-
     CI_DEBUG = os.environ.get("CI_DEBUG")
     if CI_DEBUG == "true" or CI_DEBUG == "True":
         ft_utils.execute_command(cmd_line, logger, exit_on_error=True)
@@ -304,25 +342,20 @@ def main():
                      "Possible values are:\n" + str(modes))
         exit(-1)
 
-    if args.mode == 'custom' or args.mode == 'smoke' or args.mode == 'full':
-        MODE = "--tests-file " + TEMPEST_LIST_FILE
-    elif args.mode == 'defcore':
-        MODE = "--tests-file " + TEMPEST_DEFCORE
-    else:
-        MODE = "--set " + args.mode
-
-    if args.serial:
-        MODE += " --concur 1"
-
     if not os.path.exists(TEMPEST_RESULTS_DIR):
         os.makedirs(TEMPEST_RESULTS_DIR)
 
+    deployment_dir = get_deployment_dir()
+    configure_tempest(deployment_dir)
     create_tempest_resources()
-    configure_tempest(args.mode)
-    run_tempest(MODE)
+    generate_test_list(deployment_dir, args.mode)
+    apply_tempest_blacklist()
+
+    MODE = "--tests-file " + TEMPEST_LIST
+    if args.serial:
+        MODE += " --concur 1"
 
-    if args.noclean:
-        exit(0)
+    run_tempest(MODE)
 
 
 if __name__ == '__main__':