diff options
| -rw-r--r-- | jjb/compass4nfv/compass-ci-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-project-jobs.yml | 2 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-status.py | 92 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reportingConf.py | 10 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reportingUtils.py | 11 | ||||
| -rw-r--r-- | utils/test/result_collection_api/docker/Dockerfile | 52 | ||||
| -rwxr-xr-x | utils/test/result_collection_api/docker/prepare-env.sh | 16 | ||||
| -rwxr-xr-x | utils/test/result_collection_api/docker/start-server.sh | 4 |
8 files changed, 145 insertions, 44 deletions
diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 8a43355ca..52d678554 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -215,8 +215,8 @@ - choice: name: COMPASS_OPENSTACK_VERSION choices: - - 'liberty' - 'mitaka' + - 'liberty' ######################## # trigger macros diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index a0438ee1f..8cd67ba0b 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -243,8 +243,8 @@ - choice: name: COMPASS_OPENSTACK_VERSION choices: - - 'liberty' - 'mitaka' + - 'liberty' - choice: name: COMPASS_OS_VERSION choices: diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index 2ce5efbd0..adbee36aa 100644 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -8,6 +8,7 @@ # import datetime import jinja2 +import logging import os import requests import sys @@ -19,7 +20,21 @@ import reportingConf as conf import testCase as tc import scenarioResult as sr -testCases4Validation = [] +# Logger +logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s") +logger = logging.getLogger() + +fileHandler = logging.FileHandler("{0}/{1}".format('.', conf.LOG_FILE)) +fileHandler.setFormatter(logFormatter) +logger.addHandler(fileHandler) + +consoleHandler = logging.StreamHandler() +consoleHandler.setFormatter(logFormatter) +logger.addHandler(consoleHandler) +logger.setLevel(conf.LOG_LEVEL) + +# Initialization +testValid = [] otherTestCases = [] # init just tempest to get the list of scenarios @@ -28,16 +43,16 @@ tempest = tc.TestCase("tempest_smoke_serial", "functest", -1) # Retrieve the Functest configuration to detect which tests are relevant # according to the installer, scenario -# cf = "https://git.opnfv.org/cgit/functest/plain/ci/config_functest.yaml" -cf = "https://git.opnfv.org/cgit/functest/plain/ci/testcases.yaml" +cf = conf.TEST_CONF response = requests.get(cf) + functest_yaml_config = yaml.load(response.text) -print "****************************************" -print "* Generating reporting..... *" -print ("* Data retention = %s days *" % conf.PERIOD) -print "* *" -print "****************************************" +logger.info("****************************************") +logger.info("* Generating reporting..... *") +logger.info("* Data retention = %s days *" % conf.PERIOD) +logger.info("* *") +logger.info("****************************************") # Retrieve test cases of Tier 1 (smoke) config_tiers = functest_yaml_config.get("tiers") @@ -50,19 +65,22 @@ config_tiers = functest_yaml_config.get("tiers") for tier in config_tiers: if tier['order'] > 0 and tier['order'] < 3: for case in tier['testcases']: - testCases4Validation.append(tc.TestCase(case['name'], - "functest", - case['dependencies'])) + if case['name'] not in conf.blacklist: + testValid.append(tc.TestCase(case['name'], + "functest", + case['dependencies'])) elif tier['order'] == 3: for case in tier['testcases']: - testCases4Validation.append(tc.TestCase(case['name'], - case['name'], - case['dependencies'])) + if case['name'] not in conf.blacklist: + testValid.append(tc.TestCase(case['name'], + case['name'], + case['dependencies'])) elif tier['order'] > 3: for case in tier['testcases']: - otherTestCases.append(tc.TestCase(case['name'], - "functest", - case['dependencies'])) + if case['name'] not in conf.blacklist: + otherTestCases.append(tc.TestCase(case['name'], + "functest", + case['dependencies'])) # For all the versions for version in conf.versions: @@ -84,27 +102,27 @@ for version in conf.versions: # Check if test case is runnable / installer, scenario # for the test case used for Scenario validation try: - print ("---------------------------------") - print ("installer %s, version %s, scenario %s:" % - (installer, version, s)) + logger.info("---------------------------------") + logger.info("installer %s, version %s, scenario %s:" % + (installer, version, s)) # 1) Manage the test cases for the scenario validation # concretely Tiers 0-3 - for test_case in testCases4Validation: + for test_case in testValid: test_case.checkRunnable(installer, s, test_case.getConstraints()) - print ("testcase %s is %s" % (test_case.getName(), - test_case.isRunnable)) + logger.debug("testcase %s is %s" % (test_case.getName(), + test_case.isRunnable)) time.sleep(1) if test_case.isRunnable: dbName = test_case.getDbName() name = test_case.getName() project = test_case.getProject() nb_test_runnable_for_this_scenario += 1 - print (" Searching results for case %s " % - (dbName)) + logger.info(" Searching results for case %s " % + (dbName)) result = utils.getResult(dbName, installer, s, version) - print " >>>> Test result=" + str(result) + logger.info(" >>>> Test score = " + str(result)) test_case.setCriteria(result) test_case.setIsRunnable(True) testCases2BeDisplayed.append(tc.TestCase(name, @@ -120,15 +138,15 @@ for version in conf.versions: for test_case in otherTestCases: test_case.checkRunnable(installer, s, test_case.getConstraints()) - print ("testcase %s is %s" % (test_case.getName(), - test_case.isRunnable)) + logger.info("testcase %s is %s" % + (test_case.getName(), test_case.isRunnable)) time.sleep(1) if test_case.isRunnable: dbName = test_case.getDbName() name = test_case.getName() project = test_case.getProject() - print (" Searching results for case %s " % - (dbName)) + logger.info(" Searching results for case %s " % + (dbName)) result = utils.getResult(dbName, installer, s, version) test_case.setCriteria(result) test_case.setIsRunnable(True) @@ -141,9 +159,9 @@ for version in conf.versions: items[s] = testCases2BeDisplayed except: - print ("Error: installer %s, version %s, scenario %s" % - (installer, version, s)) - print "No data available , error %s " % (sys.exc_info()[0]) + logger.error("Error: installer %s, version %s, scenario %s" % + (installer, version, s)) + logger.error("No data available: %s " % (sys.exc_info()[0])) # ********************************************** # Evaluate the results for scenario validation @@ -158,11 +176,11 @@ for version in conf.versions: s_score = str(scenario_score) + "/" + str(scenario_criteria) s_status = "KO" if scenario_score < scenario_criteria: - print (">>>> scenario not OK, score = %s/%s" % - (scenario_score, scenario_criteria)) + logger.info(">>>> scenario not OK, score = %s/%s" % + (scenario_score, scenario_criteria)) s_status = "KO" else: - print ">>>>> scenario OK, save the information" + logger.info(">>>>> scenario OK, save the information") s_status = "OK" path_validation_file = ("./release/" + version + "/validated_scenario_history.txt") @@ -173,7 +191,7 @@ for version in conf.versions: f.write(info) scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score) - print "--------------------------" + logger.info("--------------------------") templateLoader = jinja2.FileSystemLoader(os.path.dirname (os.path.abspath diff --git a/utils/test/reporting/functest/reportingConf.py b/utils/test/reporting/functest/reportingConf.py index 649246d64..61410b414 100644 --- a/utils/test/reporting/functest/reportingConf.py +++ b/utils/test/reporting/functest/reportingConf.py @@ -10,9 +10,17 @@ # # **************************************************** installers = ["apex", "compass", "fuel", "joid"] -# installers = ["compass"] +# installers = ["apex"] +# list of test cases declared in testcases.yaml but that must not be +# taken into account for the scoring +blacklist = ["odl", "ovno", "security_scan"] # versions = ["brahmaputra", "master"] versions = ["master"] PERIOD = 10 MAX_SCENARIO_CRITERIA = 18 +# get the last 5 test results to determinate the success criteria +NB_TESTS = 5 URL_BASE = 'http://testresults.opnfv.org/test/api/v1/results' +TEST_CONF = "https://git.opnfv.org/cgit/functest/plain/ci/testcases.yaml" +LOG_LEVEL = "INFO" +LOG_FILE = "reporting.log" diff --git a/utils/test/reporting/functest/reportingUtils.py b/utils/test/reporting/functest/reportingUtils.py index 0db570f32..2f06b8449 100644 --- a/utils/test/reporting/functest/reportingUtils.py +++ b/utils/test/reporting/functest/reportingUtils.py @@ -21,7 +21,8 @@ def getApiResults(case, installer, scenario, version): # "&period=30&installer=" + installer url = (reportingConf.URL_BASE + "?case=" + case + "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer + - "&scenario=" + scenario + "&version=" + version) + "&scenario=" + scenario + "&version=" + version + + "&last=" + str(reportingConf.NB_TESTS)) request = Request(url) try: @@ -104,7 +105,7 @@ def getResult(testCase, installer, scenario, version): # print "nb of results:" + str(len(test_results)) for r in test_results: - # print r["creation_date"] + # print r["start_date"] # print r["criteria"] scenario_results.append({r["start_date"]: r["criteria"]}) # sort results @@ -116,7 +117,7 @@ def getResult(testCase, installer, scenario, version): # 0: 0% success, not passing test_result_indicator = 0 nbTestOk = getNbtestOk(scenario_results) - # print "Nb test OK:"+ str(nbTestOk) + # print "Nb test OK (last 10 days):"+ str(nbTestOk) # check that we have at least 4 runs if nbTestOk < 1: test_result_indicator = 0 @@ -126,7 +127,9 @@ def getResult(testCase, installer, scenario, version): # Test the last 4 run if (len(scenario_results) > 3): last4runResults = scenario_results[-4:] - if getNbtestOk(last4runResults): + nbTestOkLast4 = getNbtestOk(last4runResults) + # print "Nb test OK (last 4 run):"+ str(nbTestOkLast4) + if nbTestOkLast4 > 3: test_result_indicator = 3 else: test_result_indicator = 2 diff --git a/utils/test/result_collection_api/docker/Dockerfile b/utils/test/result_collection_api/docker/Dockerfile new file mode 100644 index 000000000..ffee4c231 --- /dev/null +++ b/utils/test/result_collection_api/docker/Dockerfile @@ -0,0 +1,52 @@ +####################################################### +# Docker container for OPNFV-TESTAPI +####################################################### +# Purpose: run opnfv-testapi for gathering test results +# +# Maintained by SerenaFeng +# Build: +# $ docker build -t opnfv/testapi:tag . +# +# Execution: +# $ docker run -dti -p 8000:8000 \ +# -e "swagger_url=http://10.63.243.17:8000" \ +# -e "mongodb_url=mongodb://10.63.243.17:27017/" \ +# -e "api_port=8000" +# opnfv/testapi:tag +# +# NOTE: providing swagger_url, api_port, mongodb_url is optional. +# If not provided, it will use the default one +# configured in config.ini +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +FROM ubuntu:14.04 +MAINTAINER SerenaFeng <feng.xiaowei@zte.com.cn> +LABEL version="v1" description="OPNFV TestAPI Docker container" + +ENV HOME /home + +# Packaged dependencies +RUN apt-get update && apt-get install -y \ +curl \ +git \ +gcc \ +wget \ +python-dev \ +python-pip \ +crudini \ +--no-install-recommends + +RUN pip install --upgrade pip + +RUN git config --global http.sslVerify false +RUN git clone https://gerrit.opnfv.org/gerrit/releng /home/releng + +WORKDIR /home/releng/utils/test/result_collection_api/ +RUN pip install -r requirements.txt +RUN python setup.py install +CMD ["bash", "docker/start-server.sh"] diff --git a/utils/test/result_collection_api/docker/prepare-env.sh b/utils/test/result_collection_api/docker/prepare-env.sh new file mode 100755 index 000000000..99433cc8c --- /dev/null +++ b/utils/test/result_collection_api/docker/prepare-env.sh @@ -0,0 +1,16 @@ +#!/bin/bash +FILE=/etc/opnfv_testapi/config.ini + + +if [ "$mongodb_url" != "" ]; then + sudo crudini --set --existing $FILE mongo url $mongodb_url +fi + +if [ "$swagger_url" != "" ]; then + sudo crudini --set --existing $FILE swagger base_url $swagger_url +fi + +if [ "$api_port" != "" ];then + sudo crudini --set --existing $FILE api port $api_port +fi + diff --git a/utils/test/result_collection_api/docker/start-server.sh b/utils/test/result_collection_api/docker/start-server.sh new file mode 100755 index 000000000..8bf6084ae --- /dev/null +++ b/utils/test/result_collection_api/docker/start-server.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +bash docker/prepare-env.sh +opnfv-testapi |