diff options
| -rwxr-xr-x | jjb/apex/apex-upload-artifact.sh | 6 | ||||
| -rw-r--r-- | jjb/armband/armband-ci-jobs.yml | 6 | ||||
| -rw-r--r-- | jjb/opnfv/installer-params.yml | 16 | ||||
| -rw-r--r-- | jjb/opnfv/test-sign.yml | 42 | ||||
| -rw-r--r-- | jjb/sandbox/sandbox-verify-jobs.yml | 280 | ||||
| -rwxr-xr-x | utils/test-sign-artifact.sh | 26 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-status.py | 68 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-tempest.py | 71 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-vims.py | 68 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reportingConf.py | 8 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reportingUtils.py | 38 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-status-tmpl.html | 2 | ||||
| -rw-r--r-- | utils/test/reporting/functest/testCase.py | 58 |
13 files changed, 570 insertions, 119 deletions
diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index 49e19bb12..d45c7c01b 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -23,8 +23,8 @@ fi signrpm () { for artifact in $RPM_LIST $SRPM_LIST; do echo "Signing artifact: ${artifact}" - gpg2 -vvv --batch \ - --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ --passphrase besteffort \ --detach-sig $artifact gsutil cp "$artifact".sig gs://$GS_URL/$(basename "$artifact".sig) @@ -33,7 +33,7 @@ done } signiso () { -time gpg2 -vvv --batch \ +time gpg2 -vvv --batch --yes --no-tty \ --default-key opnfv-helpdesk@rt.linuxfoundation.org \ --passphrase notreallysecure \ --detach-sig $BUILD_DIRECTORY/release/OPNFV-CentOS-7-x86_64-$OPNFV_ARTIFACT_VERSION.iso diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 53c652eaf..ba02af8d6 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -36,8 +36,14 @@ #-------------------------------- scenario: # HA scenarios + - 'os-nosdn-nofeature-ha': + auto-trigger-name: 'daily-trigger-disabled' - 'os-odl_l2-nofeature-ha': auto-trigger-name: 'armband-{scenario}-{pod}-{stream}-trigger' + - 'os-odl_l3-nofeature-ha': + auto-trigger-name: 'daily-trigger-disabled' + - 'os-odl_l2-bgpvpn-ha': + auto-trigger-name: 'daily-trigger-disabled' # NOHA scenarios - 'os-odl_l2-nofeature-noha': diff --git a/jjb/opnfv/installer-params.yml b/jjb/opnfv/installer-params.yml index f95d79f93..60fee9263 100644 --- a/jjb/opnfv/installer-params.yml +++ b/jjb/opnfv/installer-params.yml @@ -93,3 +93,19 @@ name: CPU_ARCHITECTURE default: 'amd64' description: "CPU Architecture to use for Ubuntu distro " + +- parameter: + name: 'sandbox-defaults' + parameters: + - string: + name: INSTALLER_IP + default: '10.20.0.2' + description: 'IP of the installer' + - string: + name: INSTALLER_TYPE + default: sandbox + description: 'Installer used for deploying OPNFV on this POD' + - string: + name: EXTERNAL_NETWORK + default: 'admin_floating_net' + description: 'external network for test' diff --git a/jjb/opnfv/test-sign.yml b/jjb/opnfv/test-sign.yml new file mode 100644 index 000000000..b27d75777 --- /dev/null +++ b/jjb/opnfv/test-sign.yml @@ -0,0 +1,42 @@ +- project: + name: test-sign + + project: 'releng' + + jobs: + - 'test-sign-daily-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + + +- job-template: + name: 'test-sign-daily-{stream}' + + # Job template for daily builders + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: master + + disabled: false + + parameters: + - project-parameter: + project: '{project}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + triggers: + - timed: 'H H * * *' + + builders: + - shell: | + $WORKSPACE/utils/test-sign-artifact.sh diff --git a/jjb/sandbox/sandbox-verify-jobs.yml b/jjb/sandbox/sandbox-verify-jobs.yml new file mode 100644 index 000000000..b1e8e93cd --- /dev/null +++ b/jjb/sandbox/sandbox-verify-jobs.yml @@ -0,0 +1,280 @@ +- project: + name: 'sandbox-verify-jobs' + + project: 'sandbox' + + installer: 'sandbox' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + +# what are the verification activities we do for this installer + activity: + - 'basic' + - 'build' + - 'deploy' + - 'test' + + jobs: + - 'sandbox-verify-{stream}' + - 'sandbox-verify-{activity}-{stream}' + +- job-template: + name: 'sandbox-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + builders: + - description-setter: + description: "POD: $NODE_NAME" + - '{project}-verify-builder' + - trigger-builds: + - project: 'sandbox-verify-basic-{stream}' + block: true + - trigger-builds: + - project: 'sandbox-verify-build-{stream}' + block: true + - trigger-builds: + - project: 'sandbox-verify-deploy-{stream}' + block: true + - trigger-builds: + - project: 'sandbox-verify-test-{stream}' + block: true + +- job-template: + name: 'sandbox-verify-{activity}-{stream}' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{installer}-defaults' + - '{project}-verify-{activity}-parameter' + + builders: + - description-setter: + description: "POD: $NODE_NAME" + - '{project}-verify-{activity}-builder' + +##################################### +# parameter builders +##################################### +- parameter: + name: 'sandbox-verify-basic-parameter' + parameters: + - 'opnfv-build-ubuntu-defaults' + +- parameter: + name: 'sandbox-verify-build-parameter' + parameters: + - 'opnfv-build-ubuntu-defaults' + +- parameter: + name: 'sandbox-verify-deploy-parameter' + parameters: + - 'opnfv-build-centos-defaults' + +- parameter: + name: 'sandbox-verify-test-parameter' + parameters: + - 'opnfv-build-centos-defaults' +##################################### +# builder builders +##################################### +- builder: + name: 'sandbox-verify-builder' + builders: + - shell: | + #!/bin/bash + + # this is the builder for the parent/upstream job which we do nothing + cd $WORKSPACE + echo + echo "Commit Message is" + echo "-------------------------------------" + echo $GERRIT_CHANGE_COMMIT_MESSAGE + echo "-------------------------------------" + echo + echo "Repo contents" + echo "-------------------------------------" + ls -al + echo "-------------------------------------" + echo + echo "Changed files are" + echo "-------------------------------------" + git diff origin/master --name-only + echo "-------------------------------------" + echo + echo "Change introduced" + echo "-------------------------------------" + git diff origin/master + echo "-------------------------------------" + +- builder: + name: 'sandbox-verify-basic-builder' + builders: + - shell: | + #!/bin/bash + + # this is where we check the commit message, unit test, etc. + cd $WORKSPACE + echo + echo "Commit Message is" + echo "-------------------------------------" + echo $GERRIT_CHANGE_COMMIT_MESSAGE + echo "-------------------------------------" + echo + echo "Repo contents" + echo "-------------------------------------" + ls -al + echo "-------------------------------------" + echo + echo "Changed files are" + echo "-------------------------------------" + git diff origin/master --name-only + echo "-------------------------------------" + echo + echo "Change introduced" + echo "-------------------------------------" + git diff origin/master + echo "-------------------------------------" + +- builder: + name: 'sandbox-verify-build-builder' + builders: + - shell: | + #!/bin/bash + + # this is where we do the build + cd $WORKSPACE + echo + echo "Commit Message is" + echo "-------------------------------------" + echo $GERRIT_CHANGE_COMMIT_MESSAGE + echo "-------------------------------------" + echo + echo "Repo contents" + echo "-------------------------------------" + ls -al + echo "-------------------------------------" + echo + echo "Changed files are" + echo "-------------------------------------" + git diff origin/master --name-only + echo "-------------------------------------" + echo + echo "Change introduced" + echo "-------------------------------------" + git diff origin/master + echo "-------------------------------------" + +- builder: + name: 'sandbox-verify-deploy-builder' + builders: + - shell: | + #!/bin/bash + + # this is where we start the virtual deployment + cd $WORKSPACE + echo + echo "Commit Message is" + echo "-------------------------------------" + echo $GERRIT_CHANGE_COMMIT_MESSAGE + echo "-------------------------------------" + echo + echo "Repo contents" + echo "-------------------------------------" + ls -al + echo "-------------------------------------" + echo + echo "Changed files are" + echo "-------------------------------------" + git diff origin/master --name-only + echo "-------------------------------------" + echo + echo "Change introduced" + echo "-------------------------------------" + git diff origin/master + echo "-------------------------------------" + +- builder: + name: 'sandbox-verify-test-builder' + builders: + - shell: | + #!/bin/bash + + # this is where we do functest smoketest + cd $WORKSPACE + echo + echo "Commit Message is" + echo "-------------------------------------" + echo $GERRIT_CHANGE_COMMIT_MESSAGE + echo "-------------------------------------" + echo + echo "Repo contents" + echo "-------------------------------------" + ls -al + echo "-------------------------------------" + echo + echo "Changed files are" + echo "-------------------------------------" + git diff origin/master --name-only + echo "-------------------------------------" + echo + echo "Change introduced" + echo "-------------------------------------" + git diff origin/master + echo "-------------------------------------" diff --git a/utils/test-sign-artifact.sh b/utils/test-sign-artifact.sh new file mode 100755 index 000000000..f09b7f4e2 --- /dev/null +++ b/utils/test-sign-artifact.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +export PATH=$PATH:/usr/local/bin/ + +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi + +artifact="foo" +echo foo > foo + +testsign () { + echo "Signing artifact: ${artifact}" + gpg2 -vvv --batch \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $artifact +} + +testsign + diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index adbee36aa..622c375cc 100644 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -8,8 +8,6 @@ # import datetime import jinja2 -import logging -import os import requests import sys import time @@ -21,17 +19,7 @@ import testCase as tc import scenarioResult as sr # Logger -logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") -logger = logging.getLogger() - -fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE)) -fileHandler.setFormatter(logFormatter) -logger.addHandler(fileHandler) - -consoleHandler = logging.StreamHandler() -consoleHandler.setFormatter(logFormatter) -logger.addHandler(consoleHandler) -logger.setLevel(conf.LOG_LEVEL) +logger = utils.getLogger("Status") # Initialization testValid = [] @@ -48,11 +36,11 @@ response = requests.get(cf) functest_yaml_config = yaml.load(response.text) -logger.info("****************************************") -logger.info("* Generating reporting..... *") -logger.info("* Data retention = %s days *" % conf.PERIOD) -logger.info("* *") -logger.info("****************************************") +logger.info("*******************************************") +logger.info("* Generating reporting scenario status *") +logger.info("* Data retention = %s days *" % conf.PERIOD) +logger.info("* *") +logger.info("*******************************************") # Retrieve test cases of Tier 1 (smoke) config_tiers = functest_yaml_config.get("tiers") @@ -111,17 +99,22 @@ for version in conf.versions: for test_case in testValid: test_case.checkRunnable(installer, s, test_case.getConstraints()) - logger.debug("testcase %s is %s" % (test_case.getName(), - test_case.isRunnable)) + logger.debug("testcase %s is %s" % + (test_case.getDisplayName(), + test_case.isRunnable)) time.sleep(1) if test_case.isRunnable: dbName = test_case.getDbName() name = test_case.getName() + displayName = test_case.getDisplayName() project = test_case.getProject() nb_test_runnable_for_this_scenario += 1 logger.info(" Searching results for case %s " % - (dbName)) + (displayName)) result = utils.getResult(dbName, installer, s, version) + # if no result set the value to 0 + if result < 0: + result = 0 logger.info(" >>>> Test score = " + str(result)) test_case.setCriteria(result) test_case.setIsRunnable(True) @@ -144,18 +137,23 @@ for version in conf.versions: if test_case.isRunnable: dbName = test_case.getDbName() name = test_case.getName() + displayName = test_case.getDisplayName() project = test_case.getProject() logger.info(" Searching results for case %s " % - (dbName)) + (displayName)) result = utils.getResult(dbName, installer, s, version) - test_case.setCriteria(result) - test_case.setIsRunnable(True) - testCases2BeDisplayed.append(tc.TestCase(name, - project, - "", - result, - True, - 4)) + # at least 1 result for the test + if result > -1: + test_case.setCriteria(result) + test_case.setIsRunnable(True) + testCases2BeDisplayed.append(tc.TestCase(name, + project, + "", + result, + True, + 4)) + else: + logger.debug("No results found") items[s] = testCases2BeDisplayed except: @@ -182,7 +180,7 @@ for version in conf.versions: else: logger.info(">>>>> scenario OK, save the information") s_status = "OK" - path_validation_file = ("./release/" + version + + path_validation_file = (conf.REPORTING_PATH + "/release/" + version + "/validated_scenario_history.txt") with open(path_validation_file, "a") as f: time_format = "%Y-%m-%d %H:%M" @@ -193,12 +191,10 @@ for version in conf.versions: scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score) logger.info("--------------------------") - templateLoader = jinja2.FileSystemLoader(os.path.dirname - (os.path.abspath - (__file__))) + templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH) templateEnv = jinja2.Environment(loader=templateLoader) - TEMPLATE_FILE = "./template/index-status-tmpl.html" + TEMPLATE_FILE = "/template/index-status-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_stats=scenario_stats, @@ -208,6 +204,6 @@ for version in conf.versions: period=conf.PERIOD, version=version) - with open("./release/" + version + + with open(conf.REPORTING_PATH + "/release/" + version + "/index-status-" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/functest/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py index a065ef442..e3f4e3306 100644 --- a/utils/test/reporting/functest/reporting-tempest.py +++ b/utils/test/reporting/functest/reporting-tempest.py @@ -1,28 +1,44 @@ from urllib2 import Request, urlopen, URLError import json import jinja2 -import os +import reportingConf as conf +import reportingUtils as utils -installers = ["apex", "compass", "fuel", "joid"] +installers = conf.installers items = ["tests", "Success rate", "duration"] -PERIOD = 7 -print "Generate Tempest automatic reporting" +PERIOD = conf.PERIOD +criteria_nb_test = 165 +criteria_duration = 1800 +criteria_success_rate = 90 + +logger = utils.getLogger("Tempest") +logger.info("************************************************") +logger.info("* Generating reporting Tempest_smoke_serial *") +logger.info("* Data retention = %s days *" % PERIOD) +logger.info("* *") +logger.info("************************************************") + +logger.info("Success criteria:") +logger.info("nb tests executed > %s s " % criteria_nb_test) +logger.info("test duration < %s s " % criteria_duration) +logger.info("success rate > %s " % criteria_success_rate) + for installer in installers: # we consider the Tempest results of the last PERIOD days - url = "http://testresults.opnfv.org/test/api/v1/results?case=tempest_smoke_serial" - request = Request(url + '&period=' + str(PERIOD) - + '&installer=' + installer + '&version=master') - + url = conf.URL_BASE + "?case=tempest_smoke_serial" + request = Request(url + '&period=' + str(PERIOD) + + '&installer=' + installer + '&version=master') + logger.info("Search tempest_smoke_serial results for installer %s" + % installer) try: response = urlopen(request) k = response.read() results = json.loads(k) except URLError, e: - print 'No kittez. Got an error code:', e + logger.error("Error code: %s" % e) test_results = results['results'] - test_results.reverse() scenario_results = {} criteria = {} @@ -48,8 +64,8 @@ for installer in installers: nb_tests_run = result['details']['tests'] nb_tests_failed = result['details']['failures'] if nb_tests_run != 0: - success_rate = 100*(int(nb_tests_run) - - int(nb_tests_failed))/int(nb_tests_run) + success_rate = 100*(int(nb_tests_run) - + int(nb_tests_failed)) / int(nb_tests_run) else: success_rate = 0 @@ -63,40 +79,49 @@ for installer in installers: crit_time = False # Expect that at least 165 tests are run - if nb_tests_run >= 165: + if nb_tests_run >= criteria_nb_test: crit_tests = True # Expect that at least 90% of success - if success_rate >= 90: + if success_rate >= criteria_success_rate: crit_rate = True # Expect that the suite duration is inferior to 30m - if result['details']['duration'] < 1800: + if result['details']['duration'] < criteria_duration: crit_time = True result['criteria'] = {'tests': crit_tests, 'Success rate': crit_rate, 'duration': crit_time} - # error management + try: + logger.debug("Scenario %s, Installer %s" + % (s_result[1]['scenario'], installer)) + logger.debug("Nb Test run: %s" % nb_tests_run) + logger.debug("Test duration: %s" + % result['details']['duration']) + logger.debug("Success rate: %s" % success_rate) + except: + logger.error("Data format error") + + # Error management # **************** try: errors = result['details']['errors'] result['errors'] = errors.replace('{0}', '') except: - print "Error field not present (Brahamputra runs?)" + logger.error("Error field not present (Brahamputra runs?)") - mypath = os.path.abspath(__file__) - tplLoader = jinja2.FileSystemLoader(os.path.dirname(mypath)) - templateEnv = jinja2.Environment(loader=tplLoader) + templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH) + templateEnv = jinja2.Environment(loader=templateLoader) - TEMPLATE_FILE = "./template/index-tempest-tmpl.html" + TEMPLATE_FILE = "/template/index-tempest-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_results=scenario_results, items=items, installer=installer) - with open("./release/master/index-tempest-" + + with open(conf.REPORTING_PATH + "/release/master/index-tempest-" + installer + ".html", "wb") as fh: fh.write(outputText) -print "Tempest automatic reporting Done" +logger.info("Tempest automatic reporting succesfully generated.") diff --git a/utils/test/reporting/functest/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py index 4033687e8..d0436ed14 100644 --- a/utils/test/reporting/functest/reporting-vims.py +++ b/utils/test/reporting/functest/reporting-vims.py @@ -1,7 +1,11 @@ from urllib2 import Request, urlopen, URLError import json import jinja2 -import os +import reportingConf as conf +import reportingUtils as utils + +logger = utils.getLogger("vIMS") + def sig_test_format(sig_test): nbPassed = 0 @@ -9,7 +13,7 @@ def sig_test_format(sig_test): nbSkipped = 0 for data_test in sig_test: if data_test['result'] == "Passed": - nbPassed+= 1 + nbPassed += 1 elif data_test['result'] == "Failed": nbFailures += 1 elif data_test['result'] == "Skipped": @@ -20,21 +24,29 @@ def sig_test_format(sig_test): total_sig_test_result['skipped'] = nbSkipped return total_sig_test_result -installers = ["fuel", "compass", "joid", "apex"] -step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"] +logger.info("****************************************") +logger.info("* Generating reporting vIMS *") +logger.info("* Data retention = %s days *" % conf.PERIOD) +logger.info("* *") +logger.info("****************************************") +installers = conf.installers +step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"] +logger.info("Start processing....") for installer in installers: - request = Request('http://testresults.opnfv.org/test/api/v1/results?case=vims&installer=' + installer) + logger.info("Search vIMS results for installer %s" % installer) + request = Request(conf.URL_BASE + '?case=vims&installer=' + installer) try: response = urlopen(request) k = response.read() results = json.loads(k) except URLError, e: - print 'No kittez. Got an error code:', e + logger.error("Error code: %s" % e) test_results = results['results'] - test_results.reverse() + + logger.debug("Results found: %s" % test_results) scenario_results = {} for r in test_results: @@ -44,6 +56,7 @@ for installer in installers: for s, s_result in scenario_results.items(): scenario_results[s] = s_result[0:5] + logger.debug("Search for success criteria") for result in scenario_results[s]: result["start_date"] = result["start_date"].split(".")[0] sig_test = result['details']['sig_test']['result'] @@ -67,17 +80,34 @@ for installer in installers: result['pr_step_ok'] = 0 if nb_step != 0: result['pr_step_ok'] = (float(nb_step_ok)/nb_step)*100 - - - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) - templateEnv = jinja2.Environment( loader=templateLoader ) - - TEMPLATE_FILE = "./template/index-vims-tmpl.html" - template = templateEnv.get_template( TEMPLATE_FILE ) - - outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer) - - with open("./release/master/index-vims-" + installer + ".html", "wb") as fh: + try: + logger.debug("Scenario %s, Installer %s" + % (s_result[1]['scenario'], installer)) + logger.debug("Orchestrator deployment: %s s" + % result['details']['orchestrator']['duration']) + logger.debug("vIMS deployment: %s s" + % result['details']['vIMS']['duration']) + logger.debug("Signaling testing: %s s" + % result['details']['sig_test']['duration']) + logger.debug("Signaling testing results: %s" + % format_result) + except: + logger.error("Data badly formatted") + logger.debug("------------------------------------------------") + + templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH) + templateEnv = jinja2.Environment(loader=templateLoader) + + TEMPLATE_FILE = "/template/index-vims-tmpl.html" + template = templateEnv.get_template(TEMPLATE_FILE) + + outputText = template.render(scenario_results=scenario_results, + step_order=step_order, + installer=installer) + + with open(conf.REPORTING_PATH + + "/release/master/index-vims-" + + installer + ".html", "wb") as fh: fh.write(outputText) - +logger.info("vIMS report succesfully generated") diff --git a/utils/test/reporting/functest/reportingConf.py b/utils/test/reporting/functest/reportingConf.py index 61410b414..a58eeecc9 100644 --- a/utils/test/reporting/functest/reportingConf.py +++ b/utils/test/reporting/functest/reportingConf.py @@ -13,14 +13,16 @@ installers = ["apex", "compass", "fuel", "joid"] # installers = ["apex"] # list of test cases declared in testcases.yaml but that must not be # taken into account for the scoring -blacklist = ["odl", "ovno", "security_scan"] +blacklist = ["odl", "ovno", "security_scan", "copper", "moon"] # versions = ["brahmaputra", "master"] versions = ["master"] PERIOD = 10 MAX_SCENARIO_CRITERIA = 18 # get the last 5 test results to determinate the success criteria NB_TESTS = 5 +# REPORTING_PATH = "/usr/share/nginx/html/reporting/functest" +REPORTING_PATH = "." URL_BASE = 'http://testresults.opnfv.org/test/api/v1/results' TEST_CONF = "https://git.opnfv.org/cgit/functest/plain/ci/testcases.yaml" -LOG_LEVEL = "INFO" -LOG_FILE = "reporting.log" +LOG_LEVEL = "ERROR" +LOG_FILE = REPORTING_PATH + "/reporting.log" diff --git a/utils/test/reporting/functest/reportingUtils.py b/utils/test/reporting/functest/reportingUtils.py index 2f06b8449..5051ffa95 100644 --- a/utils/test/reporting/functest/reportingUtils.py +++ b/utils/test/reporting/functest/reportingUtils.py @@ -7,8 +7,26 @@ # http://www.apache.org/licenses/LICENSE-2.0 # from urllib2 import Request, urlopen, URLError +import logging import json -import reportingConf +import reportingConf as conf + + +def getLogger(module): + logFormatter = logging.Formatter("%(asctime)s [" + + module + + "] [%(levelname)-5.5s] %(message)s") + logger = logging.getLogger() + + fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE)) + fileHandler.setFormatter(logFormatter) + logger.addHandler(fileHandler) + + consoleHandler = logging.StreamHandler() + consoleHandler.setFormatter(logFormatter) + logger.addHandler(consoleHandler) + logger.setLevel(conf.LOG_LEVEL) + return logger def getApiResults(case, installer, scenario, version): @@ -19,10 +37,10 @@ def getApiResults(case, installer, scenario, version): # urllib2.install_opener(opener) # url = "http://127.0.0.1:8000/results?case=" + case + \ # "&period=30&installer=" + installer - url = (reportingConf.URL_BASE + "?case=" + case + - "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer + + url = (conf.URL_BASE + "?case=" + case + + "&period=" + str(conf.PERIOD) + "&installer=" + installer + "&scenario=" + scenario + "&version=" + version + - "&last=" + str(reportingConf.NB_TESTS)) + "&last=" + str(conf.NB_TESTS)) request = Request(url) try: @@ -38,9 +56,8 @@ def getApiResults(case, installer, scenario, version): def getScenarios(case, installer, version): case = case.getName() - print case - url = (reportingConf.URL_BASE + "?case=" + case + - "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer + + url = (conf.URL_BASE + "?case=" + case + + "&period=" + str(conf.PERIOD) + "&installer=" + installer + "&version=" + version) request = Request(url) @@ -115,11 +132,16 @@ def getResult(testCase, installer, scenario, version): # 2: <4 successful consecutive runs but passing the criteria # 1: close to pass the success criteria # 0: 0% success, not passing + # -1: no run available test_result_indicator = 0 nbTestOk = getNbtestOk(scenario_results) + # print "Nb test OK (last 10 days):"+ str(nbTestOk) # check that we have at least 4 runs - if nbTestOk < 1: + if len(scenario_results) < 1: + # No results available + test_result_indicator = -1 + elif nbTestOk < 1: test_result_indicator = 0 elif nbTestOk < 2: test_result_indicator = 1 diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index 89a1d1527..0c3fa9426 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -76,7 +76,7 @@ {% for test in items[scenario] -%} <th> {% if test.getCriteria() > -1 -%} - {{test.getDbName() }} + {{test.getDisplayName() }} {%- endif %} {% if test.getTier() > 3 -%} * diff --git a/utils/test/reporting/functest/testCase.py b/utils/test/reporting/functest/testCase.py index f0e8f5995..e19853a09 100644 --- a/utils/test/reporting/functest/testCase.py +++ b/utils/test/reporting/functest/testCase.py @@ -19,6 +19,28 @@ class TestCase(object): self.criteria = criteria self.isRunnable = isRunnable self.tier = tier + display_name_matrix = {'healthcheck': 'healthcheck', + 'vping_ssh': 'vPing (ssh)', + 'vping_userdata': 'vPing (userdata)', + 'odl': 'ODL', + 'onos': 'ONOS', + 'ocl': 'OCL', + 'tempest_smoke_serial': 'Tempest (smoke)', + 'tempest_full_parallel': 'Tempest (full)', + 'rally_sanity': 'Rally (smoke)', + 'bgpvpn': 'bgpvpn', + 'rally_full': 'Rally (full)', + 'vims': 'vIMS', + 'doctor': 'Doctor', + 'promise': 'Promise', + 'moon': 'moon', + 'copper': 'copper', + 'security_scan': 'security' + } + try: + self.displayName = display_name_matrix[self.name] + except: + self.displayName = "unknown" def getName(self): return self.name @@ -74,10 +96,10 @@ class TestCase(object): self.isRunnable = is_runnable def toString(self): - testcase = ("Name=" + self.name + ";Criteria=" + str(self.criteria) - + ";Project=" + self.project + ";Constraints=" - + str(self.constraints) + ";IsRunnable" - + str(self.isRunnable)) + testcase = ("Name=" + self.name + ";Criteria=" + + str(self.criteria) + ";Project=" + self.project + + ";Constraints=" + str(self.constraints) + + ";IsRunnable" + str(self.isRunnable)) return testcase def getDbName(self): @@ -98,31 +120,15 @@ class TestCase(object): 'rally_full': 'rally_full', 'vims': 'vims', 'doctor': 'doctor-notification', - 'promise': 'promise' + 'promise': 'promise', + 'moon': 'moon', + 'copper': 'copper', + 'security_scan': 'security' } try: return test_match_matrix[self.name] except: return "unknown" - def getTestDisplayName(self): - # Correspondance name of the test case / name in the DB - test_match_matrix = {'healthcheck': 'healthcheck', - 'vping_ssh': 'vPing (ssh)', - 'vping_userdata': 'vPing (userdata)', - 'odl': 'ODL', - 'onos': 'ONOS', - 'ocl': 'OCL', - 'tempest_smoke_serial': 'Tempest (smoke)', - 'tempest_full_parallel': 'Tempest (full)', - 'rally_sanity': 'Rally (smoke)', - 'bgpvpn': 'bgpvpn', - 'rally_full': 'Rally (full)', - 'vims': 'vIMS', - 'doctor': 'Doctor', - 'promise': 'Promise' - } - try: - return test_match_matrix[self.name] - except: - return "unknown" + def getDisplayName(self): + return self.displayName |