From d416cc4c34a33c5316669f2c384f4e4f240becef Mon Sep 17 00:00:00 2001 From: Morgan Richomme Date: Fri, 22 Sep 2017 16:25:21 +0200 Subject: cloudify_ims reporting fixes + minor fixes * unknown => real test case names * pylint corrections Change-Id: I75446e4006d41fe1bb25106420910025afa01489 Signed-off-by: Morgan Richomme --- .../reporting/functest/reporting-status.py | 96 ++++----- .../reporting/reporting/functest/reporting-vims.py | 232 +++++++++++---------- .../functest/template/index-vims-tmpl.html | 13 +- .../test/reporting/reporting/functest/testCase.py | 5 +- 4 files changed, 182 insertions(+), 164 deletions(-) (limited to 'utils') diff --git a/utils/test/reporting/reporting/functest/reporting-status.py b/utils/test/reporting/reporting/functest/reporting-status.py index 02bf67d0e..808c84144 100755 --- a/utils/test/reporting/reporting/functest/reporting-status.py +++ b/utils/test/reporting/reporting/functest/reporting-status.py @@ -22,7 +22,7 @@ Functest reporting status """ # Logger -logger = rp_utils.getLogger("Functest-Status") +LOGGER = rp_utils.getLogger("Functest-Status") # Initialization testValid = [] @@ -46,16 +46,16 @@ exclude_virtual = rp_utils.get_config('functest.exclude_virtual') functest_yaml_config = rp_utils.getFunctestConfig() -logger.info("*******************************************") -logger.info("* *") -logger.info("* Generating reporting scenario status *") -logger.info("* Data retention: %s days *" % period) -logger.info("* Log level: %s *" % log_level) -logger.info("* *") -logger.info("* Virtual PODs exluded: %s *" % exclude_virtual) -logger.info("* NOHA scenarios excluded: %s *" % exclude_noha) -logger.info("* *") -logger.info("*******************************************") +LOGGER.info("*******************************************") +LOGGER.info("* *") +LOGGER.info("* Generating reporting scenario status *") +LOGGER.info("* Data retention: %s days *", period) +LOGGER.info("* Log level: %s *", log_level) +LOGGER.info("* *") +LOGGER.info("* Virtual PODs exluded: %s *", exclude_virtual) +LOGGER.info("* NOHA scenarios excluded: %s *", exclude_noha) +LOGGER.info("* *") +LOGGER.info("*******************************************") # Retrieve test cases of Tier 1 (smoke) config_tiers = functest_yaml_config.get("tiers") @@ -75,9 +75,9 @@ for tier in config_tiers: elif tier['order'] == 2: for case in tier['testcases']: if case['case_name'] not in blacklist: - testValid.append(tc.TestCase(case['case_name'], - case['case_name'], - case['dependencies'])) + otherTestCases.append(tc.TestCase(case['case_name'], + case['case_name'], + case['dependencies'])) elif tier['order'] > 2: for case in tier['testcases']: if case['case_name'] not in blacklist: @@ -85,7 +85,7 @@ for tier in config_tiers: "functest", case['dependencies'])) -logger.debug("Functest reporting start") +LOGGER.debug("Functest reporting start") # For all the versions for version in versions: @@ -101,7 +101,7 @@ for version in versions: # initiate scenario file if it does not exist if not os.path.isfile(scenario_file_name): with open(scenario_file_name, "a") as my_file: - logger.debug("Create scenario file: %s" % scenario_file_name) + LOGGER.debug("Create scenario file: %s", scenario_file_name) my_file.write("date,scenario,installer,detail,score\n") for installer in installers: @@ -113,10 +113,10 @@ for version in versions: version) # get nb of supported architecture (x86, aarch64) architectures = rp_utils.getArchitectures(scenario_results) - logger.info("Supported architectures: {}".format(architectures)) + LOGGER.info("Supported architectures: %s", architectures) for architecture in architectures: - logger.info("architecture: {}".format(architecture)) + LOGGER.info("Architecture: %s", architecture) # Consider only the results for the selected architecture # i.e drop x86 for aarch64 and vice versa filter_results = rp_utils.filterArchitecture(scenario_results, @@ -133,10 +133,10 @@ for version in versions: # For all the scenarios get results for s, s_result in filter_results.items(): - logger.info("---------------------------------") - logger.info("installer %s, version %s, scenario %s:" % - (installer, version, s)) - logger.debug("Scenario results: %s" % s_result) + LOGGER.info("---------------------------------") + LOGGER.info("installer %s, version %s, scenario %s:", + installer, version, s) + LOGGER.debug("Scenario results: %s", s_result) # Green or Red light for a given scenario nb_test_runnable_for_this_scenario = 0 @@ -146,11 +146,11 @@ for version in versions: s_url = "" if len(s_result) > 0: build_tag = s_result[len(s_result)-1]['build_tag'] - logger.debug("Build tag: %s" % build_tag) + LOGGER.debug("Build tag: %s", build_tag) s_url = rp_utils.getJenkinsUrl(build_tag) if s_url is None: s_url = "http://testresultS.opnfv.org/reporting" - logger.info("last jenkins url: %s" % s_url) + LOGGER.info("last jenkins url: %s", s_url) testCases2BeDisplayed = [] # Check if test case is runnable / installer, scenario # for the test case used for Scenario validation @@ -160,24 +160,24 @@ for version in versions: for test_case in testValid: test_case.checkRunnable(installer, s, test_case.getConstraints()) - logger.debug("testcase %s (%s) is %s" % - (test_case.getDisplayName(), - test_case.getName(), - test_case.isRunnable)) + LOGGER.debug("testcase %s (%s) is %s", + test_case.getDisplayName(), + test_case.getName(), + test_case.isRunnable) time.sleep(1) if test_case.isRunnable: name = test_case.getName() displayName = test_case.getDisplayName() project = test_case.getProject() nb_test_runnable_for_this_scenario += 1 - logger.info(" Searching results for case %s " % - (displayName)) + LOGGER.info(" Searching results for case %s ", + displayName) result = rp_utils.getResult(name, installer, s, version) # if no result set the value to 0 if result < 0: result = 0 - logger.info(" >>>> Test score = " + str(result)) + LOGGER.info(" >>>> Test score = " + str(result)) test_case.setCriteria(result) test_case.setIsRunnable(True) testCases2BeDisplayed.append(tc.TestCase(name, @@ -193,17 +193,17 @@ for version in versions: for test_case in otherTestCases: test_case.checkRunnable(installer, s, test_case.getConstraints()) - logger.debug("testcase %s (%s) is %s" % - (test_case.getDisplayName(), - test_case.getName(), - test_case.isRunnable)) + LOGGER.debug("testcase %s (%s) is %s", + test_case.getDisplayName(), + test_case.getName(), + test_case.isRunnable) time.sleep(1) if test_case.isRunnable: name = test_case.getName() displayName = test_case.getDisplayName() project = test_case.getProject() - logger.info(" Searching results for case %s " % - (displayName)) + LOGGER.info(" Searching results for case %s ", + displayName) result = rp_utils.getResult(name, installer, s, version) # at least 1 result for the test @@ -218,13 +218,13 @@ for version in versions: True, 4)) else: - logger.debug("No results found") + LOGGER.debug("No results found") items[s] = testCases2BeDisplayed except Exception: - logger.error("Error: installer %s, version %s, scenario %s" - % (installer, version, s)) - logger.error("No data available: %s" % (sys.exc_info()[0])) + LOGGER.error("Error installer %s, version %s, scenario %s", + installer, version, s) + LOGGER.error("No data available: %s", sys.exc_info()[0]) # ********************************************** # Evaluate the results for scenario validation @@ -243,11 +243,11 @@ for version in versions: s_status = "KO" if scenario_score < scenario_criteria: - logger.info(">>>> scenario not OK, score = %s/%s" % - (scenario_score, scenario_criteria)) + LOGGER.info(">>>> scenario not OK, score = %s/%s", + scenario_score, scenario_criteria) s_status = "KO" else: - logger.info(">>>>> scenario OK, save the information") + LOGGER.info(">>>>> scenario OK, save the information") s_status = "OK" path_validation_file = ("./display/" + version + "/functest/" + @@ -270,7 +270,7 @@ for version in versions: s_score, s_score_percent, s_url) - logger.info("--------------------------") + LOGGER.info("--------------------------") templateLoader = jinja2.FileSystemLoader(".") templateEnv = jinja2.Environment( @@ -294,9 +294,9 @@ for version in versions: installer_display + ".html", "wb") as fh: fh.write(outputText) - logger.info("Manage export CSV & PDF") + LOGGER.info("Manage export CSV & PDF") rp_utils.export_csv(scenario_file_name, installer_display, version) - logger.error("CSV generated...") + LOGGER.error("CSV generated...") # Generate outputs for export # pdf @@ -306,4 +306,4 @@ for version in versions: pdf_doc_name = ("./display/" + version + "/functest/status-" + installer_display + ".pdf") rp_utils.export_pdf(pdf_path, pdf_doc_name) - logger.info("PDF generated...") + LOGGER.info("PDF generated...") diff --git a/utils/test/reporting/reporting/functest/reporting-vims.py b/utils/test/reporting/reporting/functest/reporting-vims.py index 14fddbe25..3b25e911d 100755 --- a/utils/test/reporting/reporting/functest/reporting-vims.py +++ b/utils/test/reporting/reporting/functest/reporting-vims.py @@ -1,112 +1,128 @@ +#!/usr/bin/python +# +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +""" +vIMS reporting status +""" from urllib2 import Request, urlopen, URLError import json import jinja2 -# manage conf -import utils.reporting_utils as rp_utils - -logger = rp_utils.getLogger("vIMS") - - -def sig_test_format(sig_test): - nbPassed = 0 - nbFailures = 0 - nbSkipped = 0 - for data_test in sig_test: - if data_test['result'] == "Passed": - nbPassed += 1 - elif data_test['result'] == "Failed": - nbFailures += 1 - elif data_test['result'] == "Skipped": - nbSkipped += 1 - total_sig_test_result = {} - total_sig_test_result['passed'] = nbPassed - total_sig_test_result['failures'] = nbFailures - total_sig_test_result['skipped'] = nbSkipped - return total_sig_test_result - -period = rp_utils.get_config('general.period') -versions = rp_utils.get_config('general.versions') -url_base = rp_utils.get_config('testapi.url') - -logger.info("****************************************") -logger.info("* Generating reporting vIMS *") -logger.info("* Data retention = %s days *" % period) -logger.info("* *") -logger.info("****************************************") - -installers = rp_utils.get_config('general.installers') -step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"] -logger.info("Start processing....") +import reporting.utils.reporting_utils as rp_utils + +LOGGER = rp_utils.getLogger("vIMS") + +PERIOD = rp_utils.get_config('general.period') +VERSIONS = rp_utils.get_config('general.versions') +URL_BASE = rp_utils.get_config('testapi.url') + +LOGGER.info("****************************************") +LOGGER.info("* Generating reporting vIMS *") +LOGGER.info("* Data retention = %s days *", PERIOD) +LOGGER.info("* *") +LOGGER.info("****************************************") + +INSTALLERS = rp_utils.get_config('general.installers') +STEP_ORDER = ["initialisation", "orchestrator", "vnf", "test_vnf"] +LOGGER.info("Start vIMS reporting processing....") # For all the versions -for version in versions: - for installer in installers: - logger.info("Search vIMS results for installer: %s, version: %s" - % (installer, version)) - request = Request("http://" + url_base + '?case=vims&installer=' + - installer + '&version=' + version) - - try: - response = urlopen(request) - k = response.read() - results = json.loads(k) - except URLError as e: - logger.error("Error code: %s" % e) - - test_results = results['results'] - - logger.debug("Results found: %s" % test_results) - - scenario_results = {} - for r in test_results: - if not r['scenario'] in scenario_results.keys(): - scenario_results[r['scenario']] = [] - scenario_results[r['scenario']].append(r) - - for s, s_result in scenario_results.items(): - scenario_results[s] = s_result[0:5] - logger.debug("Search for success criteria") - for result in scenario_results[s]: - result["start_date"] = result["start_date"].split(".")[0] - sig_test = result['details']['sig_test']['result'] - if not sig_test == "" and isinstance(sig_test, list): - format_result = sig_test_format(sig_test) - if format_result['failures'] > format_result['passed']: - result['details']['sig_test']['duration'] = 0 - result['details']['sig_test']['result'] = format_result - nb_step_ok = 0 - nb_step = len(result['details']) - - for step_name, step_result in result['details'].items(): - if step_result['duration'] != 0: - nb_step_ok += 1 - m, s = divmod(step_result['duration'], 60) - m_display = "" - if int(m) != 0: - m_display += str(int(m)) + "m " - - step_result['duration_display'] = (m_display + - str(int(s)) + "s") - - result['pr_step_ok'] = 0 - if nb_step != 0: - result['pr_step_ok'] = (float(nb_step_ok) / nb_step) * 100 - try: - logger.debug("Scenario %s, Installer %s" - % (s_result[1]['scenario'], installer)) - res = result['details']['orchestrator']['duration'] - logger.debug("Orchestrator deployment: %s s" - % res) - logger.debug("vIMS deployment: %s s" - % result['details']['vIMS']['duration']) - logger.debug("Signaling testing: %s s" - % result['details']['sig_test']['duration']) - logger.debug("Signaling testing results: %s" - % format_result) - except Exception: - logger.error("Data badly formatted") - logger.debug("----------------------------------------") +for version in VERSIONS: + for installer in INSTALLERS: + + # get nb of supported architecture (x86, aarch64) + # get scenarios + scenario_results = rp_utils.getScenarios("functest", + "cloudify_ims", + installer, + version) + + architectures = rp_utils.getArchitectures(scenario_results) + LOGGER.info("Supported architectures: %s", architectures) + + for architecture in architectures: + LOGGER.info("Architecture: %s", architecture) + # Consider only the results for the selected architecture + # i.e drop x86 for aarch64 and vice versa + filter_results = rp_utils.filterArchitecture(scenario_results, + architecture) + scenario_stats = rp_utils.getScenarioStats(filter_results) + items = {} + scenario_result_criteria = {} + + # in case of more than 1 architecture supported + # precise the architecture + installer_display = installer + if "fuel" in installer: + installer_display = installer + "@" + architecture + + LOGGER.info("Search vIMS results for installer: %s, version: %s", + installer, version) + request = Request("http://" + URL_BASE + '?case=cloudify_ims&' + 'installer=' + installer + '&version=' + version) + try: + response = urlopen(request) + k = response.read() + results = json.loads(k) + except URLError as err: + LOGGER.error("Error code: %s", err) + + test_results = results['results'] + + # LOGGER.debug("Results found: %s" % test_results) + + scenario_results = {} + for r in test_results: + if not r['scenario'] in scenario_results.keys(): + scenario_results[r['scenario']] = [] + scenario_results[r['scenario']].append(r) + + # LOGGER.debug("scenario result: %s" % scenario_results) + + for s, s_result in scenario_results.items(): + scenario_results[s] = s_result[0:5] + for result in scenario_results[s]: + try: + format_result = result['details']['test_vnf']['result'] + + # round durations of the different steps + result['details']['orchestrator']['duration'] = round( + result['details']['orchestrator']['duration'], 1) + result['details']['vnf']['duration'] = round( + result['details']['vnf']['duration'], 1) + result['details']['test_vnf']['duration'] = round( + result['details']['test_vnf']['duration'], 1) + + res_orch = \ + result['details']['orchestrator']['duration'] + res_vnf = result['details']['vnf']['duration'] + res_test_vnf = \ + result['details']['test_vnf']['duration'] + res_signaling = \ + result['details']['test_vnf']['result']['failures'] + + # Manage test result status + if res_signaling != 0: + LOGGER.debug("At least 1 signalig test FAIL") + result['details']['test_vnf']['status'] = "FAIL" + else: + LOGGER.debug("All signalig tests PASS") + result['details']['test_vnf']['status'] = "PASS" + + LOGGER.debug("Scenario %s, Installer %s", + s_result[1]['scenario'], installer) + LOGGER.debug("Orchestrator deployment: %ss", res_orch) + LOGGER.debug("vIMS deployment: %ss", res_vnf) + LOGGER.debug("VNF testing: %ss", res_test_vnf) + LOGGER.debug("VNF testing results: %s", format_result) + except Exception as err: # pylint: disable=broad-except + LOGGER.error("Uncomplete data %s", err) + LOGGER.debug("----------------------------------------") templateLoader = jinja2.FileSystemLoader(".") templateEnv = jinja2.Environment(loader=templateLoader, @@ -116,11 +132,11 @@ for version in versions: template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_results=scenario_results, - step_order=step_order, - installer=installer) - + step_order=STEP_ORDER, + installer=installer_display) + LOGGER.debug("Generate html page for %s", installer_display) with open("./display/" + version + "/functest/vims-" + - installer + ".html", "wb") as fh: + installer_display + ".html", "wb") as fh: fh.write(outputText) -logger.info("vIMS report succesfully generated") +LOGGER.info("vIMS report succesfully generated") diff --git a/utils/test/reporting/reporting/functest/template/index-vims-tmpl.html b/utils/test/reporting/reporting/functest/template/index-vims-tmpl.html index cd51607b7..9bd2b2f66 100644 --- a/utils/test/reporting/reporting/functest/template/index-vims-tmpl.html +++ b/utils/test/reporting/reporting/functest/template/index-vims-tmpl.html @@ -22,11 +22,12 @@ @@ -58,17 +59,17 @@ Step Status - Duration + Duration(s) Result {% for step_od_name in step_order -%} {% if step_od_name in result.details.keys() -%} {% set step_result = result.details[step_od_name] -%} - {% if step_result.duration != 0 -%} + {% if step_result.status == "PASS" -%} {{step_od_name}} - {{step_result.duration_display}} + {{step_result.duration}} {{step_result.result}} {%- else -%} diff --git a/utils/test/reporting/reporting/functest/testCase.py b/utils/test/reporting/reporting/functest/testCase.py index 9834f0753..a182dd4cf 100644 --- a/utils/test/reporting/reporting/functest/testCase.py +++ b/utils/test/reporting/reporting/functest/testCase.py @@ -50,9 +50,10 @@ class TestCase(object): 'gluon_vping': 'Netready', 'fds': 'FDS', 'cloudify_ims': 'vIMS (Cloudify)', - 'orchestra_ims': 'OpenIMS (OpenBaton)', + 'orchestra_openims': 'OpenIMS (OpenBaton)', + 'orchestra_clearwaterims': 'vIMS (OpenBaton)', 'opera_ims': 'vIMS (Open-O)', - 'vyos_vrouter': 'vyos', + 'vyos_vrouter': 'vyos (Cloudify)', 'barometercollectd': 'Barometer', 'odl_netvirt': 'Netvirt', 'security_scan': 'Security'} -- cgit 1.2.3-korg