diff options
| -rw-r--r-- | docs/etc/conf.py | 5 | ||||
| -rw-r--r-- | jjb/bottlenecks/bottlenecks.yml | 8 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-ci-jobs.yml | 6 | ||||
| -rw-r--r-- | jjb/fuel/fuel-ci-jobs-master.yml | 154 | ||||
| -rw-r--r-- | jjb/fuel/fuel-ci-jobs.yml | 86 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-deploy-master.sh (renamed from jjb/fuel/fuel-deploy-new.sh) | 12 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-deploy.sh | 2 | ||||
| -rw-r--r-- | jjb/joid/joid-ci-jobs.yml | 16 | ||||
| -rwxr-xr-x | utils/docs-build.sh | 15 | ||||
| -rwxr-xr-x | utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py | 114 |
10 files changed, 330 insertions, 88 deletions
diff --git a/docs/etc/conf.py b/docs/etc/conf.py index d10868de5..8540f9997 100644 --- a/docs/etc/conf.py +++ b/docs/etc/conf.py @@ -15,7 +15,10 @@ we encourage you to propose a change in the releng repository. For further guidance see the https://wiki.opnfv.org/documentation/tools page. ''' -extensions = ['sphinxcontrib.httpdomain'] +extensions = ['sphinxcontrib.httpdomain', + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon'] needs_sphinx = '1.3' master_doc = 'index' diff --git a/jjb/bottlenecks/bottlenecks.yml b/jjb/bottlenecks/bottlenecks.yml index 828fd214b..c29c5688b 100644 --- a/jjb/bottlenecks/bottlenecks.yml +++ b/jjb/bottlenecks/bottlenecks.yml @@ -25,9 +25,9 @@ - master: branch: '{stream}' gs-pathname: '' -# - brahmaputra: -# branch: 'stable/{stream}' -# gs-pathname: '/{stream}' + - brahmaputra: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' project: 'bottlenecks' ############################### @@ -243,7 +243,7 @@ - trigger: name: 'bottlenecks-trigger-lf' triggers: - - timed: '0 6 * * *' +# - timed: '0 6 * * *' - gerrit: trigger-on: - patchset-created-event: diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 693e1b0cd..f68597e3a 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -97,7 +97,7 @@ condition: SUCCESSFUL projects: - name: 'compass-deploy-{pod}-daily-{stream}' - current-parameters: true + current-parameters: false git-revision: true kill-phase-on: FAILURE - multijob: @@ -105,14 +105,14 @@ condition: COMPLETED projects: - name: 'functest-compass-{pod}-daily-{stream}' - current-parameters: true + current-parameters: false kill-phase-on: NEVER - multijob: name: yardstick condition: COMPLETED projects: - name: 'yardstick-compass-{pod}-daily-{stream}' - current-parameters: true + current-parameters: false kill-phase-on: NEVER - job-template: diff --git a/jjb/fuel/fuel-ci-jobs-master.yml b/jjb/fuel/fuel-ci-jobs-master.yml new file mode 100644 index 000000000..dffafe850 --- /dev/null +++ b/jjb/fuel/fuel-ci-jobs-master.yml @@ -0,0 +1,154 @@ +- project: + + name: 'fuel-master' + + project: 'fuel' + + installer: '{project}' + +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' +#-------------------------------- +# POD, INSTALLER, AND BRANCH MAPPING +#-------------------------------- + pod: + - ericsson-pod2: + <<: *master +#-------------------------------- + +# new scenario descriptions + scenario: + - 'os_ha' + - 'os_odl-l2_ha' + - 'os_odl-l3_ha' + - 'os_onos_ha' + - 'os_kvm_ha' + - 'os_ovs_ha' + - 'os_kvm_ovs_ha' + + jobs: + - 'fuel-{scenario}-{pod}-daily-{stream}' + - 'fuel-deploy-{pod}-daily-{stream}-development' + +######################## +# job templates - new type of defining the scenario +######################## +- job-template: + name: 'fuel-{scenario}-{pod}-daily-{stream}' + + project-type: multijob + + concurrent: false + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-os_.*' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + + parameters: + - project-parameter: + project: '{project}' + - '{pod}-defaults' + - '{installer}-defaults' + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - fuel-ci-parameter: + gs-pathname: '{gs-pathname}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + builders: + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'fuel-deploy-{pod}-daily-{stream}' + current-parameters: true + git-revision: true + kill-phase-on: FAILURE + - multijob: + name: functest + condition: COMPLETED + projects: + - name: 'functest-fuel-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + - multijob: + name: yardstick + condition: COMPLETED + projects: + - name: 'yardstick-fuel-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + +- job-template: + name: 'fuel-deploy-{pod}-daily-{stream}-development' + + concurrent: false + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: 'fuel' + - 'ericsson-pod2-defaults' + - 'fuel-defaults' + - fuel-ci-parameter-new: + gs-pathname: '' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: 'master' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + + builders: + - shell: + !include-raw-escape: ./fuel-download-artifact.sh + - shell: + !include-raw-escape: ./fuel-deploy-master.sh +######################## +# parameter macros +######################## +- parameter: + name: fuel-ci-parameter-new + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-ci-jobs.yml b/jjb/fuel/fuel-ci-jobs.yml index 452d03712..c90d930c1 100644 --- a/jjb/fuel/fuel-ci-jobs.yml +++ b/jjb/fuel/fuel-ci-jobs.yml @@ -36,31 +36,20 @@ # please check the triggers before enabling any of the controllers!!! sdn-controller: - - 'nosdn': - disabled: false -# commented out below controllers to get nosdn scenario settled first -# - 'odl': -# disabled: true -# - 'onos': -# disabled: true -# - 'opencontrail': -# disabled: true + - 'nosdn' jobs: - 'fuel-{sdn-controller}-{pod}-daily-{stream}' - 'fuel-deploy-{pod}-daily-{stream}' - - 'fuel-deploy-{pod}-daily-{stream}-wip' ######################## -# job templates +# job templates - old type of defining the scenario ######################## - job-template: name: 'fuel-{sdn-controller}-{pod}-daily-{stream}' project-type: multijob - disabled: '{obj:disabled}' - concurrent: false properties: @@ -68,6 +57,10 @@ enabled: true max-total: 1 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-{sdn-controller}-{pod}-daily-{stream}' wrappers: - build-name: @@ -130,6 +123,12 @@ enabled: true max-total: 1 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-deploy-{pod}-daily-{stream}.*' + - 'functest-fuel-{pod}-daily-{stream}' + - 'yardstick-fuel-{pod}-daily-{stream}' parameters: - project-parameter: @@ -147,52 +146,10 @@ wrappers: - build-name: - name: '$BUILD_NUMBER - SDN: $SDN_CONTROLLER Scenario: $DEPLOY_SCENARIO' - - builders: - - shell: - !include-raw-escape: ./fuel-download-artifact.sh - - shell: - !include-raw-escape: ./fuel-deploy.sh - - publishers: - - email: - recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com - -- job-template: - name: 'fuel-deploy-{pod}-daily-{stream}-wip' - - concurrent: false - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: 'fuel' - - 'ericsson-pod2-defaults' - - 'fuel-defaults' - - fuel-ci-parameter: - gs-pathname: '' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: 'master' - - wrappers: - - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' builders: - - shell: - !include-raw-escape: ./fuel-download-artifact.sh - - shell: - !include-raw-escape: ./fuel-deploy-new.sh + - 'builder-{stream}' publishers: - email: @@ -216,6 +173,23 @@ default: artifacts.opnfv.org/$PROJECT{gs-pathname} description: "URL to Google Storage." ######################## +# builder macros +######################## +- builder: + name: 'builder-master' + builders: + - shell: + !include-raw: ./fuel-download-artifact.sh + - shell: + !include-raw: ./fuel-deploy-master.sh +- builder: + name: 'builder-brahmaputra' + builders: + - shell: + !include-raw: ./fuel-download-artifact.sh + - shell: + !include-raw: ./fuel-deploy.sh +######################## # trigger macros ######################## # trigger for opnfv-jump-2 is set to run 1 hour ahead of others diff --git a/jjb/fuel/fuel-deploy-new.sh b/jjb/fuel/fuel-deploy-master.sh index 1e362bc8f..4a7d1d9fb 100755 --- a/jjb/fuel/fuel-deploy-new.sh +++ b/jjb/fuel/fuel-deploy-master.sh @@ -7,12 +7,12 @@ set -o pipefail source latest.properties # echo the info about artifact that is used during the deployment -echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3) for deployment" +echo "Using $OPNFV_ARTIFACT_URL for deployment" # checkout the commit that was used for building the downloaded artifact # to make sure the ISO and deployment mechanism uses same versions -#echo "Checking out $OPNFV_GIT_SHA1" -#git checkout $OPNFV_GIT_SHA1 --quiet +echo "Checking out $OPNFV_GIT_SHA1" +git checkout $OPNFV_GIT_SHA1 --quiet # set deployment parameters BRIDGE=pxebr @@ -30,15 +30,15 @@ chmod a+x $TMPDIR # clone the securedlab repo cd $WORKSPACE echo "Cloning securedlab repo" -git clone ssh://jenkins-ericsson@gerrit.opnfv.org:29418/securedlab +git clone ssh://jenkins-ericsson@gerrit.opnfv.org:29418/securedlab --quiet # construct the command -DEPLOY_COMMAND="sudo $WORKSPACE/ci/deploy.sh -b file://$WORKSPACE/securedlab -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i $WORKSPACE/opnfv.iso -H -B $BRIDGE -S $TMPDIR" +DEPLOY_COMMAND="sudo $WORKSPACE/ci/deploy.sh -b file://$WORKSPACE/securedlab -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i file://$WORKSPACE/opnfv.iso -H -B $BRIDGE -S $TMPDIR" # log info to console echo "Deployment parameters" -echo "Scenario: $DEPLOY_SCENARIO" echo "--------------------------------------------------------" +echo "Scenario: $DEPLOY_SCENARIO" echo "Lab: $LAB_NAME" echo "POD: $POD_NAME" echo "ISO: $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3)" diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index 334307448..dc3d81a3a 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -7,7 +7,7 @@ set -o pipefail source latest.properties # echo the info about artifact that is used during the deployment -echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3) for deployment" +echo "Using $OPNFV_ARTIFACT_URL for deployment" # checkout the commit that was used for building the downloaded artifact # to make sure the ISO and deployment mechanism uses same versions diff --git a/jjb/joid/joid-ci-jobs.yml b/jjb/joid/joid-ci-jobs.yml index 0cc931085..05064de1f 100644 --- a/jjb/joid/joid-ci-jobs.yml +++ b/jjb/joid/joid-ci-jobs.yml @@ -43,8 +43,8 @@ # commented out below controllers to get Arno scenario settled first # - 'nosdn': # disabled: true -# - 'onos': -# disabled: true + - 'onos': + disabled: false # - 'opencontrail': # disabled: true @@ -173,3 +173,15 @@ name: 'joid-orange-pod2-odl-trigger' triggers: - timed: '' +- trigger: + name: 'joid-intel-pod5-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-intel-pod6-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-orange-pod2-onos-trigger' + triggers: + - timed: '' diff --git a/utils/docs-build.sh b/utils/docs-build.sh index 914eaf207..55cb9f652 100755 --- a/utils/docs-build.sh +++ b/utils/docs-build.sh @@ -179,6 +179,21 @@ do [[ -n "$GERRIT_COMMENT" ]] && echo "$msg" >> "$GERRIT_COMMENT" } + # TODO: failures in ODT creation should be handled error and + # cause 'exit 1' before OPNFV B release. + tex=$(find $build -name '*.tex' | head -1) + odt=$(basename "${tex%.tex}.odt") + if [[ -e $tex ]] && which pandoc > /dev/null ; then + pandoc $tex -o $output/$odt || { + msg="Error: ODT creation for $dir has failed." + echo + echo "$msg" + echo + } + else + echo "Warn: tex file and/or 'pandoc' are not found, skip ODT creation." + fi + if is_top_dir "$dir" ; then # NOTE: Having top level document (docs/index.rst) is not recommended. # It may cause conflicts with other docs (mostly with HTML diff --git a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py index f9e434226..8d5326eb4 100755 --- a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py +++ b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py @@ -14,6 +14,9 @@ # a new method format_<Test_case>_for_dashboard(results) # v0.1: basic example with methods for Rubbos. # +import os +import requests +import json def get_bottlenecks_cases(): @@ -21,7 +24,7 @@ def get_bottlenecks_cases(): get the list of the supported test cases TODO: update the list when adding a new test case for the dashboard """ - return ["rubbos"] + return ["rubbos", "tu1", "tu3"] def check_bottlenecks_case_exist(case): @@ -54,7 +57,6 @@ def format_bottlenecks_for_dashboard(case, results): def format_rubbos_for_dashboard(results): - """ Post processing for the Rubbos test case """ @@ -76,24 +78,106 @@ def format_rubbos_for_dashboard(results): return test_data -# for local test -import json +def format_tu1_for_dashboard(results): + test_data = [{'description': 'Tu-1 performance result'}] + line_element = [] + bar_element = {} + last_result = results[-1]["details"] + for key in sorted(last_result): + bandwith = last_result[key]["Bandwidth"] + pktsize = int(key) + line_element.append({'x': pktsize, + 'y': bandwith * 1000}) + bar_element[key] = bandwith * 1000 + # graph1, line + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {'type': "graph", + 'xlabel': 'pktsize', + 'ylabel': 'bandwith(kpps)'}, + 'data_set': line_element}) + # graph2, bar + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {"type": "bar"}, + 'data_set': bar_element}) + return test_data + + +def format_tu3_for_dashboard(results): + test_data = [{'description': 'Tu-3 performance result'}] + new_element = [] + bar_element = {} + last_result = results[-1]["details"] + for key in sorted(last_result): + bandwith = last_result[key]["Bandwidth"] + pktsize = int(key) + new_element.append({'x': pktsize, + 'y': bandwith * 1000}) + bar_element[key] = bandwith * 1000 + # graph1, line + test_data.append({'name': "VM2VM max bidirectional throughput", + 'info': {'type': "graph", + 'xlabel': 'pktsize', + 'ylabel': 'bandwith(kpps)'}, + 'data_set': new_element}) + # graph2, bar + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {"type": "bar"}, + 'data_set': bar_element}) + return test_data + + +############################ For local test ################################ + +def _read_sample_output(filename): + curr_path = os.path.dirname(os.path.abspath(__file__)) + output = os.path.join(curr_path, filename) + with open(output) as f: + sample_output = f.read() + + result = json.loads(sample_output) + return result + + +# Copy form functest/testcases/Dashboard/dashboard_utils.py +# and did some minor modification for local test. +def _get_results(db_url, test_criteria): + test_project = test_criteria["project"] + testcase = test_criteria["testcase"] + + # Build headers + headers = {'Content-Type': 'application/json'} + + # build the request + # if criteria is all => remove criteria + url = db_url + "/results?project=" + test_project + "&case=" + testcase + + # Send Request to Test DB + myData = requests.get(url, headers=headers) + + # Get result as a json object + myNewData = json.loads(myData.text) + + # Get results + myDataResults = myNewData['test_results'] + return myDataResults def _test(): - print('Post processing for the Rubbos test case begin<--') - results = '[{"details":[{"client":200,"throughput":20},{"client":300,"throughput":50}],"project_name":' \ - '"bottlenecks","pod_name":"unknown-pod","version":"unknown","installer":"fuel","description":' \ - '"bottlenecks test cases result","_id":"56793f11514bc5068a345da4","creation_date":' \ - '"2015-12-22 12:16:17.131438","case_name":"rubbos"},{"details":[{"client":200,"throughput":25},' \ - '{"client":300,"throughput":52}],"project_name":"bottlenecks","pod_name":"unknown-pod","version":' \ - '"unknown","installer":"fuel","description":"bottlenecks test cases result","_id":' \ - '"56793f11514bc5068a345da4","creation_date":"2015-12-23 12:16:17.131438","case_name":"rubbos"}]' + db_url = "http://213.77.62.197" + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "rubbos"}) + test_result = format_rubbos_for_dashboard(results) + print json.dumps(test_result, indent=4) - print("the output is:") - print(format_rubbos_for_dashboard(json.loads(results))) - print('Post processing for the Rubbos test case end<--') + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "tu1"}) + #results = _read_sample_output("sample") + #print json.dumps(results, indent=4) + test_result = format_tu1_for_dashboard(results) + print json.dumps(test_result, indent=4) + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "tu3"}) + test_result = format_tu3_for_dashboard(results) + print json.dumps(test_result, indent=4) if __name__ == '__main__': _test() + |