diff options
-rw-r--r-- | jjb/apex/apex.yml | 26 | ||||
-rw-r--r-- | jjb/bottlenecks/bottlenecks-ci-jobs.yml | 2 | ||||
-rwxr-xr-x | jjb/fuel/fuel-build.sh | 12 | ||||
-rw-r--r-- | jjb/fuel/fuel-ci-jobs.yml | 16 | ||||
-rwxr-xr-x | jjb/fuel/fuel-project-jobs.yml | 3 | ||||
-rw-r--r-- | jjb/functest/functest-ci-jobs.yml | 34 | ||||
-rw-r--r-- | jjb/opnfv/opnfv-docker.yml | 4 | ||||
-rwxr-xr-x | utils/calculate_version.sh | 15 | ||||
-rw-r--r-- | utils/test/reporting/functest/img/icon-nok.png | bin | 0 -> 2317 bytes | |||
-rw-r--r-- | utils/test/reporting/functest/img/icon-ok.png | bin | 0 -> 4063 bytes | |||
-rw-r--r-- | utils/test/reporting/functest/reporting-status.py | 66 | ||||
-rw-r--r-- | utils/test/reporting/functest/template/index-status-tmpl.html | 13 | ||||
-rw-r--r-- | utils/test/result_collection_api/resources/handlers.py | 6 | ||||
-rw-r--r-- | utils/test/result_collection_api/resources/models.py | 24 |
14 files changed, 161 insertions, 60 deletions
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 61a50daea..3ddc7b2cd 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -216,16 +216,16 @@ - trigger-builds: - project: 'apex-deploy-virtual-os-nosdn-nofeature-ha-{stream1}' predefined-parameters: | - BUILD_DIRECTORY=apex-verify-{stream1}/build_output + BUILD_DIRECTORY=apex-verify-{stream1} OPNFV_CLEAN=yes git-revision: false block: true - - trigger-builds: - - project: 'functest-apex-{slave}-suite-{stream1}' - predefined-parameters: | - DEPLOY_SCENARIO=os-nosdn-nofeature-ha - FUNCTEST_SUITE_NAME=vping_userdata - block: true +# - trigger-builds: +# - project: 'functest-apex-{slave}-suite-{stream1}' +# predefined-parameters: | +# DEPLOY_SCENARIO=os-nosdn-nofeature-ha +# FUNCTEST_SUITE_NAME=vping_userdata +# block: true - 'apex-workspace-cleanup' - job-template: @@ -329,7 +329,7 @@ - trigger-builds: - project: 'apex-deploy-virtual-os-nosdn-nofeature-ha-{stream}' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build_output + BUILD_DIRECTORY=apex-build-{stream} OPNFV_CLEAN=yes git-revision: false block: true @@ -429,7 +429,7 @@ # branch: branch (eg. stable) node: '{slave}' - disabled: false + disabled: true scm: - git-scm: @@ -605,7 +605,7 @@ - trigger-builds: - project: 'apex-deploy-virtual-nosdn-nofeature-ha-{stream1}' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream1}/build_output + BUILD_DIRECTORY=apex-build-{stream1} OPNFV_CLEAN=yes git-revision: true block: true @@ -732,7 +732,7 @@ # start the build cd $WORKSPACE/ci ./build.sh $BUILD_ARGS - RPM_VERSION=$(grep Version $BUILD_DIRECTORY/../build/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') + RPM_VERSION=$(grep Version $BUILD_DIRECTORY/build/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') # list the contents of BUILD_OUTPUT directory ls -al $BUILD_DIRECTORY # save information regarding artifact into file @@ -859,10 +859,10 @@ echo "BUILD DIRECTORY modified to $BUILD_DIRECTORY" fi - if [[ -f ${BUILD_DIRECTORY}/../opnfv.properties ]]; then + if [[ -f ${BUILD_DIRECTORY}/opnfv.properties ]]; then # if opnfv.properties exists then use the # local build. Source the file so we get local OPNFV vars - source ${BUILD_DIRECTORY}/../opnfv.properties + source ${BUILD_DIRECTORY}/opnfv.properties RPM_INSTALL_PATH=${BUILD_DIRECTORY}/$(basename $OPNFV_RPM_URL) else if [[ $BUILD_DIRECTORY == *verify* ]]; then diff --git a/jjb/bottlenecks/bottlenecks-ci-jobs.yml b/jjb/bottlenecks/bottlenecks-ci-jobs.yml index 9d881233d..deeebfdf4 100644 --- a/jjb/bottlenecks/bottlenecks-ci-jobs.yml +++ b/jjb/bottlenecks/bottlenecks-ci-jobs.yml @@ -23,7 +23,7 @@ branch: 'stable/{stream}' gs-pathname: '/{stream}' gs-packagepath: '/{stream}/{suite}' - docker-tag: 'brahmaputra' + docker-tag: 'stable' #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh index eaf1f83e7..95ebaa5a9 100755 --- a/jjb/fuel/fuel-build.sh +++ b/jjb/fuel/fuel-build.sh @@ -35,11 +35,13 @@ CURRENT_SHA1=$(git rev-parse HEAD) FORCE_BUILD=${FORCE_BUILD:-false} if [[ "$CURRENT_SHA1" == "$LATEST_ISO_SHA1" && "$FORCE_BUILD" == "false" ]]; then - echo "An ISO has already been built for this commit" - echo " $LATEST_ISO_URL" - echo "Nothing new to build. Exiting." - touch $WORKSPACE/.noupload - exit 0 + echo "***************************************************" + echo " An ISO has already been built for this commit" + echo " $LATEST_ISO_URL" + echo "***************************************************" +# echo "Nothing new to build. Exiting." +# touch $WORKSPACE/.noupload +# exit 0 else echo "This commit has not been built yet or forced build! Proceeding with the build." /bin/rm -f $LATEST_ISO_PROPERTIES diff --git a/jjb/fuel/fuel-ci-jobs.yml b/jjb/fuel/fuel-ci-jobs.yml index e1816b528..77b711e74 100644 --- a/jjb/fuel/fuel-ci-jobs.yml +++ b/jjb/fuel/fuel-ci-jobs.yml @@ -278,35 +278,35 @@ - trigger: name: 'fuel-os-nosdn-nofeature-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 12 * * *' + - timed: '' - trigger: name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 15 * * *' + - timed: '' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 18 * * *' + - timed: '' - trigger: name: 'fuel-os-onos-nofeature-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 21 * * *' + - timed: '' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 0 * * *' + - timed: '' - trigger: name: 'fuel-os-odl_l2-sfc-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 3 * * *' + - timed: '' - trigger: name: 'fuel-os-nosdn-kvm-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 6 * * *' + - timed: '' - trigger: name: 'fuel-os-nosdn-ovs-ha-ericsson-pod2-brahmaputra-trigger' triggers: - - timed: '0 9 * * *' + - timed: '' - trigger: name: 'fuel-os-nosdn-kvm-noha-ericsson-pod2-brahmaputra-trigger' triggers: diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml index 0d768203f..f377c9bed 100755 --- a/jjb/fuel/fuel-project-jobs.yml +++ b/jjb/fuel/fuel-project-jobs.yml @@ -60,8 +60,7 @@ branch: '{branch}' triggers: - - pollscm: - cron: '0 H/4 * * *' + - timed: '0 H/4 * * *' wrappers: - timeout: diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index a0a1326ba..8bcf4c9de 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -55,6 +55,9 @@ - virtual: installer: joid <<: *brahmaputra + - arm-pod1: + installer: fuel + <<: *brahmaputra #-------------------------------- # master #-------------------------------- @@ -194,7 +197,7 @@ description: "Push the results of all the tests to the resultDB" - string: name: CI_DEBUG - default: 'false' + default: 'true' description: "Show debug output information" ######################## # trigger macros @@ -221,8 +224,13 @@ - shell: | #!/bin/bash set -e - echo "Functest: run $FUNCTEST_SUITE_NAME" - cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME" + branch=${GIT_BRANCH##*/} + echo "Functest: run $FUNCTEST_SUITE_NAME on branch ${branch}" + if [[ ${branch} == *"brahmaputra"* ]]; then + cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME" + else + cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t $FUNCTEST_SUITE_NAME" + fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -232,9 +240,13 @@ - shell: | #!/bin/bash set +e - flags="-s" - [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+=" -r" - cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh ${flags}" + branch=${GIT_BRANCH##*/} + [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+="-r" + if [[ ${branch} == *"brahmaputra"* ]]; then + cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh -s ${flags}" + else + cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t all ${flags}" + fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -282,11 +294,13 @@ mkdir -p ${dir_result} sudo rm -rf ${dir_result}/* res_volume="-v ${dir_result}:/home/opnfv/functest/results" + custom_params= + test -f ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG} && custom_params=$(cat ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG}) echo "Functest: Pulling image opnfv/functest:${DOCKER_TAG}" docker pull opnfv/functest:$DOCKER_TAG >${redirect} - cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} ${res_volume} opnfv/functest:${DOCKER_TAG} /bin/bash" + cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} ${res_volume} ${custom_params} opnfv/functest:${DOCKER_TAG} /bin/bash" echo "Functest: Running docker run command: ${cmd}" ${cmd} >${redirect} docker ps -a >${redirect} @@ -306,7 +320,11 @@ echo "The container opnfv/functest with ID=${container_id} has not been properly started. Exiting..." exit 1 fi - cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" + if [[ ${branch} == *"brahmaputra"* ]]; then + cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" + else + cmd="python ${FUNCTEST_REPO_DIR}/ci/prepare_env.py start" + fi echo "Executing command inside the docker: ${cmd}" docker exec ${container_id} ${cmd} diff --git a/jjb/opnfv/opnfv-docker.yml b/jjb/opnfv/opnfv-docker.yml index 130edd480..936f22040 100644 --- a/jjb/opnfv/opnfv-docker.yml +++ b/jjb/opnfv/opnfv-docker.yml @@ -40,7 +40,7 @@ description: "To enable/disable pushing the image to Dockerhub." - string: name: BASE_VERSION - default: "brahmaputra.1" + default: "brahmaputra.3" description: "Base version to be used." - string: name: DOCKER_REPO_NAME @@ -83,7 +83,7 @@ description: "To enable/disable pushing the image to Dockerhub." - string: name: BASE_VERSION - default: "brahmaputra.2" + default: "brahmaputra.3" description: "Base version to be used." - string: name: DOCKER_REPO_NAME diff --git a/utils/calculate_version.sh b/utils/calculate_version.sh index 4bfd13f7a..608a3bd73 100755 --- a/utils/calculate_version.sh +++ b/utils/calculate_version.sh @@ -36,14 +36,15 @@ function docker_version() { tag_json=$(curl $url_tag 2>/dev/null | python -mjson.tool | grep ${BASE_VERSION} | head -1) #e.g. tag_json= "name": "brahmaputra.0.2", if [ "${tag_json}" == "" ]; then - error "The Docker Image ${docker_image} does not have a TAG with base version ${BASE_VERSION}" + echo ${BASE_VERSION}.0 + else + tag=$(echo $tag_json | awk '{print $2}' | sed 's/\,//' | sed 's/\"//g') + #e.g.: tag=brahmaputra.0.2 + tag_current_version=$(echo $tag | sed 's/.*\.//') + tag_new_version=$(($tag_current_version+1)) + #e.g.: tag=brahmaputra.0.3 + echo ${BASE_VERSION}.${tag_new_version} fi - tag=$(echo $tag_json | awk '{print $2}' | sed 's/\,//' | sed 's/\"//g') - #e.g.: tag=brahmaputra.0.2 - tag_current_version=$(echo $tag | sed 's/.*\.//') - tag_new_version=$(($tag_current_version+1)) - #e.g.: tag=brahmaputra.0.3 - echo ${BASE_VERSION}.${tag_new_version} } diff --git a/utils/test/reporting/functest/img/icon-nok.png b/utils/test/reporting/functest/img/icon-nok.png Binary files differnew file mode 100644 index 000000000..526b5294b --- /dev/null +++ b/utils/test/reporting/functest/img/icon-nok.png diff --git a/utils/test/reporting/functest/img/icon-ok.png b/utils/test/reporting/functest/img/icon-ok.png Binary files differnew file mode 100644 index 000000000..3a9de2e89 --- /dev/null +++ b/utils/test/reporting/functest/img/icon-ok.png diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index 2eb0f50d5..9271717bb 100644 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -1,4 +1,5 @@ from urllib2 import Request, urlopen, URLError +import datetime import json import jinja2 import os @@ -20,12 +21,14 @@ installers = ["apex", "compass", "fuel", "joid"] versions = ["brahmaputra", "master"] # versions = ["master"] PERIOD = 10 +MAX_SCENARIO_CRITERIA = 18 # Correspondance between the name of the test case and the name in the DB # ideally we should modify the DB to avoid such interface.... # '<name in the DB':'<name in the config'> # I know it is uggly... -test_match_matrix = {'vPing': 'vping_ssh', +test_match_matrix = {'healthcheck': 'healthcheck', + 'vPing': 'vping_ssh', 'vPing_userdata': 'vping_userdata', 'ODL': 'odl', 'ONOS': 'onos', @@ -102,6 +105,20 @@ class TestCase(object): self.isRunnable = is_runnable +class ScenarioResult(object): + def __init__(self, status, score=0): + self.status = status + self.score = score + + def getStatus(self): + return self.status + + def getScore(self): + return self.score + +# ***************************************************************************** + + def getApiResults(case, installer, scenario, version): case = case.getName() results = json.dumps([]) @@ -239,13 +256,16 @@ tempest = TestCase("Tempest", "functest", -1) # Retrieve the Functest configuration to detect which tests are relevant # according to the installer, scenario -cf = "https://git.opnfv.org/cgit/functest/plain/testcases/config_functest.yaml" +cf = "https://git.opnfv.org/cgit/functest/plain/ci/config_functest.yaml" response = requests.get(cf) functest_yaml_config = yaml.load(response.text) print "****************************************" print "* Generating reporting..... *" +print ("* Data retention = %s days *" % PERIOD) +print "* *" print "****************************************" + # For all the versions for version in versions: # For all the installers @@ -253,11 +273,16 @@ for version in versions: # get scenarios scenario_results = getScenarios(tempest, installer, version) scenario_stats = getScenarioStats(scenario_results) - items = {} + scenario_result_criteria = {} + # For all the scenarios get results for s, s_result in scenario_results.items(): testCases = [] + # Green or Red light for a given scenario + nb_test_runnable_for_this_scenario = 0 + scenario_score = 0 + # For each scenario declare the test cases # Functest cases for test_case in functest_test_list: @@ -276,31 +301,58 @@ for version in versions: test_case.checkRunnable(installer, s, functest_yaml_config) # print "testcase %s is %s" % (test_case.getName(), # test_case.isRunnable) - print "--------------------------" print ("installer %s, version %s, scenario %s:" % (installer, version, s)) for testCase in testCases: time.sleep(1) if testCase.isRunnable: + nb_test_runnable_for_this_scenario += 1 print (" Searching results for case %s " % (testCase.getName())) result = getResult(testCase, installer, s, version) testCase.setCriteria(result) items[s] = testCases - print "--------------------------" + scenario_score = scenario_score + result except: print ("installer %s, version %s, scenario %s" % (installer, version, s)) print "No data available , error %s " % (sys.exc_info()[0]) - print "****************************************" - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + # the validation criteria = nb runnable tests x 3 + scenario_criteria = nb_test_runnable_for_this_scenario * 3 + # if 0 runnable tests set criteria at a high value + if scenario_criteria < 1: + scenario_criteria = MAX_SCENARIO_CRITERIA + + s_score = str(scenario_score) + "/" + str(scenario_criteria) + s_status = "KO" + if scenario_score < scenario_criteria: + print (">>>> scenario not OK, score = %s/%s" % + (scenario_score, scenario_criteria)) + s_status = "KO" + else: + print ">>>>> scenario OK, save the information" + s_status = "OK" + with open("./release/" + version + + "/validated_scenario_history.txt", "a") as f: + time_format = "%Y-%m-%d %H:%M" + info = (datetime.datetime.now().strftime(time_format) + + ";" + installer + ";" + s + "\n") + f.write(info) + + scenario_result_criteria[s] = ScenarioResult(s_status, s_score) + print "--------------------------" + + templateLoader = jinja2.FileSystemLoader(os.path.dirname + (os.path.abspath + (__file__))) templateEnv = jinja2.Environment(loader=templateLoader) TEMPLATE_FILE = "./template/index-status-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_stats=scenario_stats, + scenario_results=scenario_result_criteria, items=items, installer=installer, period=PERIOD, diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index 604f2c8e4..7a0656b74 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -40,12 +40,20 @@ <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div> <table class="table"> <tr> - <th width="80%">Scenario</th> - <th width="20%">Iteration</th> + <th width="60%">Scenario</th> + <th width="20%">Status</th> + <th width="10%">Score</th> + <th width="10%">Iteration</th> </tr> {% for scenario,iteration in scenario_stats.iteritems() -%} <tr class="tr-ok"> <td>{{scenario}}</td> + <td>{%if scenario_results[scenario].getStatus() is sameas "OK" -%} + <img src="../../img/icon-ok.png"> + {%- else -%} + <img src="../../img/icon-nok.png"> + {%- endif %}</td> + <td>{{scenario_results[scenario].getScore()}}</td> <td>{{iteration}}</td> </tr> {%- endfor %} @@ -53,7 +61,6 @@ </div> - {% for scenario, iteration in scenario_stats.iteritems() -%} <div class="scenario-part"> <div class="page-header"> diff --git a/utils/test/result_collection_api/resources/handlers.py b/utils/test/result_collection_api/resources/handlers.py index 1eda3b067..c1e8eb182 100644 --- a/utils/test/result_collection_api/resources/handlers.py +++ b/utils/test/result_collection_api/resources/handlers.py @@ -512,6 +512,8 @@ class TestResultsHandler(GenericApiHandler): - period : x (x last days) - scenario : the test scenario (previously version) - criteria : the global criteria status passed or failed + - trust_indicator : evaluate the stability of the test case to avoid + running systematically long and stable test case :param result_id: Get a result by ID @@ -531,6 +533,7 @@ class TestResultsHandler(GenericApiHandler): scenario_arg = self.get_query_argument("scenario", None) criteria_arg = self.get_query_argument("criteria", None) period_arg = self.get_query_argument("period", None) + trust_indicator_arg = self.get_query_argument("trust_indicator", None) # prepare request get_request = dict() @@ -559,6 +562,9 @@ class TestResultsHandler(GenericApiHandler): if criteria_arg is not None: get_request["criteria_tag"] = criteria_arg + if trust_indicator_arg is not None: + get_request["trust_indicator_arg"] = trust_indicator_arg + if period_arg is not None: try: period_arg = int(period_arg) diff --git a/utils/test/result_collection_api/resources/models.py b/utils/test/result_collection_api/resources/models.py index 35b6af11f..06e95f94f 100644 --- a/utils/test/result_collection_api/resources/models.py +++ b/utils/test/result_collection_api/resources/models.py @@ -153,6 +153,7 @@ class TestResult: self.build_tag = None
self.scenario = None
self.criteria = None
+ self.trust_indicator = None
@staticmethod
def test_result_from_dict(test_result_dict):
@@ -173,7 +174,21 @@ class TestResult: t.build_tag = test_result_dict.get('build_tag')
t.scenario = test_result_dict.get('scenario')
t.criteria = test_result_dict.get('criteria')
-
+ # 0 < trust indicator < 1
+ # if bad value => set this indicator to 0
+ if test_result_dict.get('trust_indicator') is not None:
+ if isinstance(test_result_dict.get('trust_indicator'),
+ (int, long, float)):
+ if test_result_dict.get('trust_indicator') < 0:
+ t.trust_indicator = 0
+ elif test_result_dict.get('trust_indicator') > 1:
+ t.trust_indicator = 1
+ else:
+ t.trust_indicator = test_result_dict.get('trust_indicator')
+ else:
+ t.trust_indicator = 0
+ else:
+ t.trust_indicator = 0
return t
def format(self):
@@ -188,7 +203,8 @@ class TestResult: "details": self.details,
"build_tag": self.build_tag,
"scenario": self.scenario,
- "criteria": self.criteria
+ "criteria": self.criteria,
+ "trust_indicator": self.trust_indicator
}
def format_http(self):
@@ -204,6 +220,6 @@ class TestResult: "details": self.details,
"build_tag": self.build_tag,
"scenario": self.scenario,
- "criteria": self.criteria
+ "criteria": self.criteria,
+ "trust_indicator": self.trust_indicator
}
-
|