diff options
42 files changed, 677 insertions, 942 deletions
diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh index 58d9f1a40..ad94ba3d7 100755 --- a/jjb/apex/apex-build.sh +++ b/jjb/apex/apex-build.sh @@ -28,8 +28,10 @@ fi BUILD_DIRECTORY=${WORKSPACE}/build # start the build -cd $WORKSPACE/ci -./build.sh $BUILD_ARGS +pushd ${BUILD_DIRECTORY} +make clean +popd +python3 apex/build.py $BUILD_ARGS RPM_VERSION=$(grep Version: $WORKSPACE/build/rpm_specs/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') # list the contents of BUILD_OUTPUT directory echo "Build Directory is ${BUILD_DIRECTORY}/../.build" diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index ce9544b28..a47e3a5db 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -37,7 +37,7 @@ if [[ "$ARTIFACT_VERSION" =~ dev ]]; then # Settings for deploying from git workspace DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy" NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network" - DEPLOY_CMD="${WORKSPACE}/ci/deploy.sh" + DEPLOY_CMD="opnfv-deploy --image-dir ${WORKSPACE}/.build" CLEAN_CMD="${WORKSPACE}/ci/clean.sh" RESOURCES="${WORKSPACE}/.build/" CONFIG="${WORKSPACE}/build" @@ -48,6 +48,11 @@ if [[ "$ARTIFACT_VERSION" =~ dev ]]; then # Ensure artifacts were downloaded and extracted correctly # TODO(trozet) add verification here + # Install dev build + mkdir -p ~/tmp + mv -f .build ~/tmp/ + sudo pip3 install --upgrade --force-reinstall . + mv -f ~/tmp/.build . else DEPLOY_SETTINGS_DIR="/etc/opnfv-apex/" NETWORK_SETTINGS_DIR="/etc/opnfv-apex/" @@ -65,7 +70,10 @@ fi # Install Dependencies # Make sure python34 dependencies are installed -for dep_pkg in epel-release python34 python34-PyYAML python34-setuptools; do +dependencies="epel-release python34 python34-devel libvirt-devel python34-pip \ +ansible python34-PyYAML python34-jinja2 python34-setuptools python-tox ansible" + +for dep_pkg in $dependencies; do if ! rpm -q ${dep_pkg} > /dev/null; then if ! sudo yum install -y ${dep_pkg}; then echo "Failed to install ${dep_pkg}" @@ -74,31 +82,12 @@ for dep_pkg in epel-release python34 python34-PyYAML python34-setuptools; do fi done -# Make sure jinja2 is installed -for python_pkg in jinja2; do - if ! python3.4 -c "import $python_pkg"; then - echo "$python_pkg package not found for python3.4, attempting to install..." - if ! sudo easy_install-3.4 $python_pkg; then - echo -e "Failed to install $python_pkg package for python3.4" - exit 1 - fi - fi -done - if [[ "$JOB_NAME" =~ "virtual" ]]; then # Make sure ipxe-roms-qemu package is updated to latest. # This package is needed for multi virtio nic PXE boot in virtual environment. sudo yum update -y ipxe-roms-qemu - if [ -z ${PYTHONPATH:-} ]; then - export PYTHONPATH=${WORKSPACE}/lib/python - else - export PYTHONPATH=$PYTHONPATH:${WORKSPACE}/lib/python - fi fi -# set env vars to deploy cmd -DEPLOY_CMD="BASE=${BASE} IMAGES=${IMAGES} LIB=${LIB} ${DEPLOY_CMD}" - if [ "$OPNFV_CLEAN" == 'yes' ]; then if sudo test -e '/root/inventory/pod_settings.yaml'; then clean_opts='-i /root/inventory/pod_settings.yaml' @@ -106,7 +95,7 @@ if [ "$OPNFV_CLEAN" == 'yes' ]; then clean_opts='' fi - sudo BASE=${BASE} LIB=${LIB} ${CLEAN_CMD} ${clean_opts} + sudo ${CLEAN_CMD} ${clean_opts} fi if echo ${DEPLOY_SCENARIO} | grep ipv6; then diff --git a/jjb/apex/apex-project-jobs.yml b/jjb/apex/apex-project-jobs.yml new file mode 100644 index 000000000..973ad913a --- /dev/null +++ b/jjb/apex/apex-project-jobs.yml @@ -0,0 +1,127 @@ +--- +- project: + name: 'apex-project-jobs' + project: 'apex' + + stream: + - master: &master + branch: 'master' + gs-pathname: '' + concurrent-builds: 3 + disabled: false + + - danube: &danube + branch: 'stable/danube' + gs-pathname: '/danube' + concurrent-builds: 1 + disabled: true + + jobs: + - 'apex-build-{stream}' + - 'apex-verify-iso-{stream}' + +# Build phase +- job-template: + name: 'apex-build-{stream}' + + # Job template for builds + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: 'apex-build-{stream}' + + disabled: false + + concurrent: true + + parameters: + - '{project}-defaults' + - project-parameter: + project: '{project}' + branch: '{branch}' + - apex-parameter: + gs-pathname: '{gs-pathname}' + + scm: + - git-scm-gerrit + + wrappers: + - timeout: + timeout: 150 + fail: true + + properties: + - logrotate-default + - throttle: + max-per-node: '{concurrent-builds}' + max-total: 10 + option: 'project' + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify-iso-{stream}' + + builders: + - 'apex-build' + - inject: + properties-content: ARTIFACT_TYPE=rpm + - 'apex-upload-artifact' + +# ISO verify job +- job-template: + name: 'apex-verify-iso-{stream}' + + # Job template for builds + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: 'apex-virtual-{stream}' + + disabled: false + + concurrent: true + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - apex-parameter: + gs-pathname: '{gs-pathname}' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." + + scm: + - git-scm + + properties: + - logrotate-default + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - 'apex-iso-verify' + - inject: + properties-content: ARTIFACT_TYPE=iso + - 'apex-upload-artifact' + +######################## +# builder macros +######################## +- builder: + name: 'apex-build' + builders: + - shell: + !include-raw: ./apex-build.sh + +- builder: + name: 'apex-iso-verify' + builders: + - shell: + !include-raw: ./apex-iso-verify.sh diff --git a/jjb/apex/apex-unit-test.sh b/jjb/apex/apex-unit-test.sh index abcddcab4..3112c9d36 100755 --- a/jjb/apex/apex-unit-test.sh +++ b/jjb/apex/apex-unit-test.sh @@ -8,9 +8,24 @@ echo "-------------------------------------------------------------------------- echo -pushd ci/ > /dev/null -./test.sh -popd +pushd build/ > /dev/null +for pkg in yamllint rpmlint iproute epel-release python34-devel python34-nose python34-PyYAML python-pep8 python34-mock python34-pip; do + if ! rpm -q ${pkg} > /dev/null; then + if ! sudo yum install -y ${pkg}; then + echo "Failed to install ${pkg} package..." + exit 1 + fi + fi +done + +# Make sure coverage is installed +if ! python3 -c "import coverage" &> /dev/null; then sudo pip3 install coverage; fi + +make rpmlint +make python-pep8-check +make yamllint +make python-tests +popd > /dev/null echo "--------------------------------------------------------" echo "Unit Tests Done!" diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index aeea1c9fb..9ccf8e39d 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -6,13 +6,11 @@ - 'apex-verify-gate-{stream}' - 'apex-verify-unit-tests-{stream}' - 'apex-runner-cperf-{stream}' - - 'apex-build-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-master' - 'apex-daily-danube' - 'apex-csit-promote-daily-{stream}' - 'apex-fdio-promote-daily-{stream}' - - 'apex-verify-iso-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' - 'apex-upload-snapshot' @@ -27,7 +25,6 @@ virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' - concurrent-builds: 3 scenario_stream: 'master' - danube: &danube @@ -37,7 +34,6 @@ virtual-slave: 'apex-virtual-danube' baremetal-slave: 'apex-baremetal-danube' verify-scenario: 'os-odl_l3-nofeature-ha' - concurrent-builds: 1 scenario_stream: 'danube' disabled: true @@ -234,17 +230,15 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' file-paths: - compare-type: ANT - pattern: 'ci/**' - - compare-type: ANT - pattern: 'build/**' + pattern: '**' + forbidden-file-paths: - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' + pattern: 'apex/tests/**' - compare-type: ANT - pattern: 'apex/**' + pattern: 'docs/**' properties: - logrotate-default @@ -500,97 +494,6 @@ abort-all-job: false git-revision: false -# Build phase -- job-template: - name: 'apex-build-{stream}' - - # Job template for builds - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: '{build-slave}' - - disabled: false - - concurrent: true - - parameters: - - '{project}-defaults' - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - scm: - - git-scm-gerrit - - wrappers: - - timeout: - timeout: 150 - fail: true - - properties: - - logrotate-default - - throttle: - max-per-node: {concurrent-builds} - max-total: 10 - option: 'project' - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-verify-iso-{stream}' - - builders: - - 'apex-build' - - inject: - properties-content: ARTIFACT_TYPE=rpm - - 'apex-upload-artifact' - -# ISO verify job -- job-template: - name: 'apex-verify-iso-{stream}' - - # Job template for builds - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: '{virtual-slave}' - - disabled: false - - concurrent: true - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from parameters macro." - - scm: - - git-scm - - properties: - - logrotate-default - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - 'apex-iso-verify' - - inject: - properties-content: ARTIFACT_TYPE=iso - - 'apex-upload-artifact' - # Deploy job - job-template: name: 'apex-deploy-{platform}-{stream}' @@ -1671,23 +1574,11 @@ !include-raw: ./apex-unit-test.sh - builder: - name: 'apex-build' - builders: - - shell: - !include-raw: ./apex-build.sh - -- builder: name: 'apex-workspace-cleanup' builders: - shell: !include-raw: ./apex-workspace-cleanup.sh -- builder: - name: 'apex-iso-verify' - builders: - - shell: - !include-raw: ./apex-iso-verify.sh - - builder: name: 'apex-upload-artifact' diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index 0d446dcd7..c878fe6af 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -6,13 +6,11 @@ - 'apex-verify-gate-{stream}' - 'apex-verify-unit-tests-{stream}' - 'apex-runner-cperf-{stream}' - - 'apex-build-{stream}' - 'apex-deploy-{platform}-{stream}' - 'apex-daily-master' - 'apex-daily-danube' - 'apex-csit-promote-daily-{stream}' - 'apex-fdio-promote-daily-{stream}' - - 'apex-verify-iso-{stream}' - 'apex-{scenario}-baremetal-{scenario_stream}' - 'apex-testsuite-{scenario}-baremetal-{scenario_stream}' - 'apex-upload-snapshot' @@ -27,7 +25,6 @@ virtual-slave: 'apex-virtual-master' baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' - concurrent-builds: 3 scenario_stream: 'master' - danube: &danube @@ -37,7 +34,6 @@ virtual-slave: 'apex-virtual-danube' baremetal-slave: 'apex-baremetal-danube' verify-scenario: 'os-odl_l3-nofeature-ha' - concurrent-builds: 1 scenario_stream: 'danube' disabled: true @@ -146,17 +142,15 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' file-paths: - compare-type: ANT - pattern: 'ci/**' - - compare-type: ANT - pattern: 'build/**' + pattern: '**' + forbidden-file-paths: - compare-type: ANT - pattern: 'lib/**' - - compare-type: ANT - pattern: 'config/**' + pattern: 'apex/tests/**' - compare-type: ANT - pattern: 'apex/**' + pattern: 'docs/**' properties: - logrotate-default @@ -412,97 +406,6 @@ abort-all-job: false git-revision: false -# Build phase -- job-template: - name: 'apex-build-{stream}' - - # Job template for builds - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: '{build-slave}' - - disabled: false - - concurrent: true - - parameters: - - '{project}-defaults' - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - scm: - - git-scm-gerrit - - wrappers: - - timeout: - timeout: 150 - fail: true - - properties: - - logrotate-default - - throttle: - max-per-node: {concurrent-builds} - max-total: 10 - option: 'project' - - build-blocker: - use-build-blocker: true - block-level: 'NODE' - blocking-jobs: - - 'apex-verify-iso-{stream}' - - builders: - - 'apex-build' - - inject: - properties-content: ARTIFACT_TYPE=rpm - - 'apex-upload-artifact' - -# ISO verify job -- job-template: - name: 'apex-verify-iso-{stream}' - - # Job template for builds - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: '{virtual-slave}' - - disabled: false - - concurrent: true - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - apex-parameter: - gs-pathname: '{gs-pathname}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from parameters macro." - - scm: - - git-scm - - properties: - - logrotate-default - - throttle: - max-per-node: 1 - max-total: 10 - option: 'project' - - builders: - - 'apex-iso-verify' - - inject: - properties-content: ARTIFACT_TYPE=iso - - 'apex-upload-artifact' - # Deploy job - job-template: name: 'apex-deploy-{platform}-{stream}' @@ -1098,23 +1001,11 @@ !include-raw: ./apex-unit-test.sh - builder: - name: 'apex-build' - builders: - - shell: - !include-raw: ./apex-build.sh - -- builder: name: 'apex-workspace-cleanup' builders: - shell: !include-raw: ./apex-workspace-cleanup.sh -- builder: - name: 'apex-iso-verify' - builders: - - shell: - !include-raw: ./apex-iso-verify.sh - - builder: name: 'apex-upload-artifact' diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index ded70b151..faa5971e1 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -50,7 +50,7 @@ # HA scenarios - 'os-nosdn-nofeature-ha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' - - 'os-odl_l3-nofeature-ha': + - 'os-odl-nofeature-ha': auto-trigger-name: '{installer}-{scenario}-{pod}-{stream}-trigger' # NOHA scenarios @@ -82,6 +82,7 @@ use-build-blocker: true blocking-jobs: - '{installer}-os-.*?-{pod}-daily-.*' + - 'armband-verify-.*' block-level: 'NODE' wrappers: @@ -95,14 +96,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO default: '{scenario}' - - armband-ci-parameter: - gs-pathname: '{gs-pathname}' builders: - trigger-builds: @@ -139,10 +139,10 @@ # 3.only proposed_tests testsuite here(refstack, ha, ipv6, bgpvpn) # 4.not used for release criteria or compliance, # only to debug the dovetail tool bugs with arm pods - # 5.only run against scenario os-(nosdn|odl_l3)-(nofeature-bgpvpn)-ha + # 5.only run against scenario os-(nosdn|odl)-(nofeature-bgpvpn)-ha - conditional-step: condition-kind: regex-match - regex: os-(nosdn|odl|odl_l3)-(nofeature|bgpvpn)-ha + regex: os-(nosdn|odl)-(nofeature|bgpvpn)-ha label: '{scenario}' steps: - trigger-builds: @@ -180,14 +180,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l3-nofeature-ha' - - armband-ci-parameter: - gs-pathname: '{gs-pathname}' + default: 'os-odl-nofeature-ha' scm: - git-scm @@ -206,21 +205,6 @@ - email-jenkins-admins-on-failure ######################## -# parameter macros -######################## -- parameter: - name: armband-ci-parameter - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." - - string: - name: SSH_KEY - default: "/tmp/mcp.rsa" - description: "Path to private SSH key to access environment nodes. For MCP deployments only." - -######################## # trigger macros ######################## # CI PODs @@ -236,7 +220,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-armband-baremetal-master-trigger' + name: 'fuel-os-odl-nofeature-ha-armband-baremetal-master-trigger' triggers: - timed: '0 16 * * *' #---------------------------------------------------------------------- @@ -251,7 +235,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-armband-baremetal-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-armband-baremetal-euphrates-trigger' triggers: - timed: '' #--------------------------------------------------------------- @@ -266,7 +250,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-armband-virtual-master-trigger' + name: 'fuel-os-odl-nofeature-ha-armband-virtual-master-trigger' triggers: - timed: '' #-------------------------------------------------------------------- @@ -281,6 +265,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-armband-virtual-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-armband-virtual-euphrates-trigger' triggers: - timed: '' diff --git a/jjb/armband/armband-verify-jobs.yml b/jjb/armband/armband-verify-jobs.yml index 56f70d68b..c43dc7f82 100644 --- a/jjb/armband/armband-verify-jobs.yml +++ b/jjb/armband/armband-verify-jobs.yml @@ -21,11 +21,11 @@ ##################################### phase: - 'basic': - slave-label: 'opnfv-build-enea' + slave-label: 'armband-virtual' - 'deploy-virtual': - slave-label: 'opnfv-build-enea' + slave-label: 'armband-virtual' - 'smoke-test': - slave-label: 'opnfv-build-enea' + slave-label: 'armband-virtual' ##################################### # jobs ##################################### @@ -50,6 +50,11 @@ enabled: true max-total: 4 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-os-.*?-virtual-daily-.*' + block-level: 'NODE' scm: - git-scm-gerrit @@ -94,8 +99,9 @@ - project-parameter: project: '{project}' branch: '{branch}' - - 'opnfv-build-enea-defaults' - - 'armband-verify-defaults': + - 'armband-virtual-defaults': + installer: '{installer}' + - '{installer}-defaults': gs-pathname: '{gs-pathname}' builders: @@ -155,7 +161,8 @@ - logrotate-default - throttle: enabled: true - max-total: 6 + max-total: 2 + max-per-node: 1 option: 'project' - build-blocker: use-build-blocker: true @@ -177,8 +184,9 @@ project: '{project}' branch: '{branch}' - '{slave-label}-defaults' - - '{installer}-defaults' - - 'armband-verify-defaults': + - 'armband-virtual-defaults': + installer: '{installer}' + - '{installer}-defaults': gs-pathname: '{gs-pathname}' builders: @@ -199,10 +207,8 @@ - builder: name: 'armband-verify-deploy-virtual-macro' builders: - - shell: | - #!/bin/bash - - echo "Not activated!" + - shell: + !include-raw: ../fuel/fuel-deploy.sh - builder: name: 'armband-verify-smoke-test-macro' @@ -211,13 +217,3 @@ #!/bin/bash echo "Not activated!" -##################################### -# parameter macros -##################################### -- parameter: - name: 'armband-verify-defaults' - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml index 55d629cb4..59479e73d 100644 --- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml +++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml @@ -77,7 +77,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|functest|octopus|pharos|releng|sandbox|yardstick' + project-pattern: 'apex|armband|bamboo|barometer|bottlenecks|calipso|compass4nfv|conductor|cooper|cperf|daisy|doctor|dovetail|dpacc|enfv|escalator|fds|fuel|functest|octopus|pharos|releng|sandbox|yardstick' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index 59482459e..691e76278 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -21,14 +21,14 @@ disabled: false jobs: - - 'compass-build-iso-{stream}' + - '{installer}-build-daily-{stream}' - 'compass-build-ppa-{stream}' ######################## # job templates ######################## - job-template: - name: 'compass-build-iso-{stream}' + name: '{installer}-build-daily-{stream}' disabled: '{obj:disabled}' diff --git a/jjb/daisy4nfv/daisy-project-jobs.yml b/jjb/daisy4nfv/daisy-project-jobs.yml index fd0da7928..09b0bc2e1 100644 --- a/jjb/daisy4nfv/daisy-project-jobs.yml +++ b/jjb/daisy4nfv/daisy-project-jobs.yml @@ -54,6 +54,8 @@ use-build-blocker: true blocking-jobs: - '{installer}-daily-.*' + - 'daisy4nfv-merge-build-.*' + - 'daisy4nfv-verify-build-.*' block-level: 'NODE' scm: diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml index 561ffbe24..f7a6c1363 100644 --- a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml @@ -154,6 +154,7 @@ use-build-blocker: true blocking-jobs: - '{alias}-merge-{phase}-.*' + - '{installer}-daily-.*' block-level: 'NODE' scm: diff --git a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml index dff0ff0a4..6f6ab0fb0 100644 --- a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml @@ -50,7 +50,7 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '{installer}-merge-build-.*' + - '{alias}-merge-build-.*' block-level: 'NODE' scm: @@ -147,8 +147,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '{installer}-merge-build-.*' + - '{alias}-merge-build-.*' - '{alias}-verify-build-.*' + - '{installer}-daily-.*' block-level: 'NODE' scm: diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 7dd6a2ddc..9c4e205ae 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -175,7 +175,7 @@ fi # Modify tempest_conf.yaml file tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml -if [ ${INSTALLER_TYPE} == 'compass' ]; then +if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then volume_device='vdb' else volume_device='vdc' diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 488505edf..c30cfed72 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -68,18 +68,12 @@ # HA scenarios - 'os-nosdn-nofeature-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-nofeature-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l3-nofeature-ha': + - 'os-odl-nofeature-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-sfc-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-nofeature-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-sfc-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-bgpvpn-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-ha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-ha': @@ -93,18 +87,12 @@ # NOHA scenarios - 'os-nosdn-nofeature-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-nofeature-noha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l3-nofeature-noha': + - 'os-odl-nofeature-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-sfc-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-nofeature-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-sfc-noha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - - 'os-odl_l2-bgpvpn-noha': - auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-noha': @@ -140,6 +128,7 @@ blocking-jobs: - 'fuel-os-.*?-{pod}-daily-.*' - 'fuel-os-.*?-{pod}-weekly-.*' + - 'fuel-verify-.*' block-level: 'NODE' wrappers: @@ -153,14 +142,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO default: '{scenario}' - - fuel-ci-parameter: - gs-pathname: '{gs-pathname}' builders: - description-setter: @@ -199,11 +187,11 @@ # 3.only debug testsuite here(refstack, ha, ipv6, bgpvpn) # 4.not used for release criteria or compliance, # only to debug the dovetail tool bugs with bgpvpn and nosdn-nofeature - # 5.only run against scenario os-odl_l2-bgpvpn-ha(regex used here, can extend to more scenarios future) - # 6.ZTE pod1, os-nosdn-nofeature-ha and os-odl_l2-bgpvpn-ha, run against danube + # 5.only run against scenario os-odl-bgpvpn-ha(regex used here, can extend to more scenarios future) + # 6.ZTE pod1, os-nosdn-nofeature-ha and os-odl-bgpvpn-ha, run against danube - conditional-step: condition-kind: regex-match - regex: os-(nosdn-nofeature|odl_l2-bgpvpn)-ha + regex: os-(nosdn-nofeature|odl-bgpvpn)-ha label: '{scenario}' steps: - trigger-builds: @@ -250,14 +238,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-nofeature-ha' - - fuel-ci-parameter: - gs-pathname: '{gs-pathname}' + default: 'os-odl-nofeature-ha' - string: name: DEPLOY_TIMEOUT default: '150' @@ -284,20 +271,6 @@ - email-jenkins-admins-on-failure ######################## -# parameter macros -######################## -- parameter: - name: fuel-ci-parameter - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." - - string: - name: SSH_KEY - default: "/tmp/mcp.rsa" - description: "Path to private SSH key to access environment nodes. For MCP deployments only." -######################## # trigger macros ######################## #----------------------------------------------- @@ -309,11 +282,7 @@ triggers: - timed: '5 20 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-baremetal-daily-master-trigger' - triggers: - - timed: '' # '5 23 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-baremetal-daily-master-trigger' + name: 'fuel-os-odl-nofeature-ha-baremetal-daily-master-trigger' triggers: - timed: '5 2 * * *' - trigger: @@ -329,14 +298,6 @@ triggers: - timed: '' # '5 8 * * *' - trigger: - name: 'fuel-os-odl_l2-sfc-ha-baremetal-daily-master-trigger' - triggers: - - timed: '' # '5 11 * * *' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-baremetal-daily-master-trigger' - triggers: - - timed: '' # '5 14 * * *' -- trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-master-trigger' triggers: - timed: '' # '5 17 * * *' @@ -354,11 +315,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-baremetal-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-baremetal-daily-master-trigger' + name: 'fuel-os-odl-nofeature-noha-baremetal-daily-master-trigger' triggers: - timed: '' - trigger: @@ -370,14 +327,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-baremetal-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-baremetal-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-master-trigger' triggers: - timed: '' @@ -402,11 +351,7 @@ triggers: - timed: '' # '0 20 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' # '0 23 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-baremetal-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-baremetal-daily-euphrates-trigger' triggers: - timed: '' # '0 2 * * *' - trigger: @@ -418,14 +363,6 @@ triggers: - timed: '' # '0 8 * * *' - trigger: - name: 'fuel-os-odl_l2-sfc-ha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' # '0 11 * * *' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' # '0 14 * * *' -- trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-euphrates-trigger' triggers: - timed: '' # '0 17 * * *' @@ -447,11 +384,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-baremetal-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-noha-baremetal-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -463,14 +396,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-baremetal-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-euphrates-trigger' triggers: - timed: '' @@ -494,11 +419,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-virtual-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-virtual-daily-master-trigger' + name: 'fuel-os-odl-nofeature-ha-virtual-daily-master-trigger' triggers: - timed: '' - trigger: @@ -510,14 +431,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-virtual-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-virtual-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-virtual-daily-master-trigger' triggers: - timed: '' @@ -539,11 +452,7 @@ triggers: - timed: '5 13 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-virtual-daily-master-trigger' - triggers: - - timed: '35 15 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-virtual-daily-master-trigger' + name: 'fuel-os-odl-nofeature-noha-virtual-daily-master-trigger' triggers: - timed: '5 18 * * *' - trigger: @@ -555,14 +464,6 @@ triggers: - timed: '' # '5 23 * * *' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-virtual-daily-master-trigger' - triggers: - - timed: '' # '35 1 * * *' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-virtual-daily-master-trigger' - triggers: - - timed: '' # '5 4 * * *' -- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-master-trigger' triggers: - timed: '' # '35 6 * * *' @@ -586,11 +487,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-virtual-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-virtual-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -602,14 +499,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-virtual-daily-euphrates-trigger' triggers: - timed: '' @@ -631,11 +520,7 @@ triggers: - timed: '' # '0 13 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' # '30 15 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-virtual-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-noha-virtual-daily-euphrates-trigger' triggers: - timed: '' # '0 18 * * *' - trigger: @@ -647,14 +532,6 @@ triggers: - timed: '' # '0 23 * * *' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' # '30 1 * * *' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-virtual-daily-euphrates-trigger' - triggers: - - timed: '' # '0 4 * * *' -- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-euphrates-trigger' triggers: - timed: '' # '30 6 * * *' @@ -678,11 +555,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-zte-pod1-daily-master-trigger' - triggers: - - timed: '0 10 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-zte-pod1-daily-master-trigger' + name: 'fuel-os-odl-nofeature-ha-zte-pod1-daily-master-trigger' triggers: - timed: '' - trigger: @@ -694,14 +567,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-zte-pod1-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-zte-pod1-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-master-trigger' triggers: - timed: '' @@ -723,11 +588,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-zte-pod1-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-zte-pod1-daily-master-trigger' + name: 'fuel-os-odl-nofeature-noha-zte-pod1-daily-master-trigger' triggers: - timed: '' - trigger: @@ -739,14 +600,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-zte-pod1-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-zte-pod1-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-master-trigger' triggers: - timed: '' @@ -770,11 +623,7 @@ triggers: - timed: '0 10 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-zte-pod3-daily-master-trigger' + name: 'fuel-os-odl-nofeature-ha-zte-pod3-daily-master-trigger' triggers: - timed: '' - trigger: @@ -786,14 +635,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod3-daily-master-trigger' triggers: - timed: '' @@ -815,11 +656,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-zte-pod3-daily-master-trigger' + name: 'fuel-os-odl-nofeature-noha-zte-pod3-daily-master-trigger' triggers: - timed: '' - trigger: @@ -831,14 +668,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-zte-pod3-daily-master-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod3-daily-master-trigger' triggers: - timed: '' @@ -862,11 +691,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' # '0 2 * * *' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-zte-pod1-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-zte-pod1-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -878,14 +703,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-euphrates-trigger' triggers: - timed: '' @@ -907,11 +724,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-zte-pod1-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-noha-zte-pod1-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -923,14 +736,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-zte-pod1-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-euphrates-trigger' triggers: - timed: '' @@ -954,11 +759,7 @@ triggers: - timed: '' # '0 18 * * *' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-zte-pod3-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-ha-zte-pod3-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -970,14 +771,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod3-daily-euphrates-trigger' triggers: - timed: '' # '0 2 * * *' @@ -999,11 +792,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-zte-pod3-daily-euphrates-trigger' + name: 'fuel-os-odl-nofeature-noha-zte-pod3-daily-euphrates-trigger' triggers: - timed: '' - trigger: @@ -1015,14 +804,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-zte-pod3-daily-euphrates-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod3-daily-euphrates-trigger' triggers: - timed: '' @@ -1046,11 +827,7 @@ triggers: - timed: '0 2 * * 6' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-ha-zte-pod1-daily-danube-trigger' + name: 'fuel-os-odl-nofeature-ha-zte-pod1-daily-danube-trigger' triggers: - timed: '' - trigger: @@ -1062,14 +839,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '0 2 * * 1,3,5' -- trigger: - name: 'fuel-os-odl_l2-sfc-ha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-ha-zte-pod1-daily-danube-trigger' triggers: - timed: '' @@ -1091,11 +860,7 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-noha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l3-nofeature-noha-zte-pod1-daily-danube-trigger' + name: 'fuel-os-odl-nofeature-noha-zte-pod1-daily-danube-trigger' triggers: - timed: '' - trigger: @@ -1107,14 +872,6 @@ triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-sfc-noha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '' -- trigger: - name: 'fuel-os-odl_l2-bgpvpn-noha-zte-pod1-daily-danube-trigger' - triggers: - - timed: '' -- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-danube-trigger' triggers: - timed: '' diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index ddaebc94a..18b2fd730 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -12,7 +12,7 @@ set -o pipefail export TERM="vt220" -if [[ "$BRANCH" != 'master' ]]; then +if [[ "$BRANCH" =~ 'danube' ]]; then # source the file so we get OPNFV vars source latest.properties @@ -21,11 +21,11 @@ if [[ "$BRANCH" != 'master' ]]; then fi # shellcheck disable=SC2153 -if [[ "${JOB_NAME}" =~ "merge" ]]; then - # set simplest scenario for virtual deploys to run for merges +if [[ "${JOB_NAME}" =~ 'verify' ]]; then + # set simplest scenario for virtual deploys to run for verify DEPLOY_SCENARIO="os-nosdn-nofeature-ha" -elif [[ "${BRANCH}" != 'master' ]]; then - # for none-merge deployments +elif [[ "${BRANCH}" =~ 'danube' ]]; then + # for Danube deployments (no artifact for current master or newer branches) # checkout the commit that was used for building the downloaded artifact # to make sure the ISO and deployment mechanism uses same versions echo "Checking out ${OPNFV_GIT_SHA1}" diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh index fa0c88b5d..02ca10305 100755 --- a/jjb/fuel/fuel-download-artifact.sh +++ b/jjb/fuel/fuel-download-artifact.sh @@ -16,15 +16,9 @@ set -o pipefail # use proxy url to replace the normal URL, or googleusercontent.com will be blocked randomly [[ "$NODE_NAME" =~ (zte) ]] && GS_URL=${GS_BASE_PROXY%%/*}/$GS_URL -if [[ "$JOB_NAME" =~ "merge" ]]; then - echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" - # get the properties file for the Fuel ISO built for a merged change - curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties -else - # get the latest.properties file in order to get info regarding latest artifact - echo "Downloading http://$GS_URL/latest.properties" - curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties -fi +# get the latest.properties file in order to get info regarding latest artifact +echo "Downloading http://$GS_URL/latest.properties" +curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties # check if we got the file [[ -f $WORKSPACE/latest.properties ]] || exit 1 @@ -36,21 +30,18 @@ source $WORKSPACE/latest.properties OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} echo "Using $OPNFV_ARTIFACT for deployment" -# using ISOs for verify & merge jobs from local storage will be enabled later -if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then - # check if we already have the ISO to avoid redownload - ISOSTORE="/iso_mount/opnfv_ci/${BRANCH##*/}" - if [[ -f "$ISOSTORE/$OPNFV_ARTIFACT" ]]; then - echo "ISO exists locally. Skipping the download and using the file from ISO store" - ln -s $ISOSTORE/$OPNFV_ARTIFACT $WORKSPACE/opnfv.iso - echo "--------------------------------------------------------" - echo - ls -al $WORKSPACE/opnfv.iso - echo - echo "--------------------------------------------------------" - echo "Done!" - exit 0 - fi +# check if we already have the ISO to avoid redownload +ISOSTORE="/iso_mount/opnfv_ci/${BRANCH##*/}" +if [[ -f "$ISOSTORE/$OPNFV_ARTIFACT" ]]; then + echo "ISO exists locally. Skipping the download and using the file from ISO store" + ln -s $ISOSTORE/$OPNFV_ARTIFACT $WORKSPACE/opnfv.iso + echo "--------------------------------------------------------" + echo + ls -al $WORKSPACE/opnfv.iso + echo + echo "--------------------------------------------------------" + echo "Done!" + exit 0 fi [[ "$NODE_NAME" =~ (zte) ]] && OPNFV_ARTIFACT_URL=${GS_BASE_PROXY%%/*}/$OPNFV_ARTIFACT_URL diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml index e850a0acb..cfcbf3695 100644 --- a/jjb/fuel/fuel-project-jobs.yml +++ b/jjb/fuel/fuel-project-jobs.yml @@ -19,84 +19,12 @@ disabled: true jobs: - - 'fuel-merge-deploy-virtual-{stream}' - 'fuel-deploy-generic-daily-{stream}' ######################## # job templates ######################## - job-template: - name: 'fuel-merge-deploy-virtual-{stream}' - - disabled: true - - concurrent: true - - properties: - - logrotate-default - - throttle: - enabled: true - max-total: 2 - max-per-node: 1 - option: 'project' - - build-blocker: - use-build-blocker: true - blocking-jobs: - - 'fuel-os-.*?-virtual-daily-.*' - - 'fuel-merge-deploy-virtual-.*' - block-level: 'NODE' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'fuel-virtual-defaults': - installer: '{installer}' - - '{installer}-defaults' - - fuel-project-parameter: - gs-pathname: '{gs-pathname}' - scm: - - git-scm - - wrappers: - - ssh-agent-wrapper - - triggers: - - gerrit: - server-name: 'gerrit.opnfv.org' - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'ci/**' - - compare-type: ANT - pattern: 'mcp/**' - disable-strict-forbidden-file-verification: 'true' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**' - dependency-jobs: 'fuel-merge-build-{stream}' - - builders: - - shell: - !include-raw-escape: ./fuel-deploy.sh - - shell: - !include-raw-escape: ./fuel-workspace-cleanup.sh - - publishers: - - email: - recipients: fzhadaev@mirantis.com - - email-jenkins-admins-on-failure - -- job-template: name: 'fuel-deploy-generic-daily-{stream}' concurrent: true @@ -120,14 +48,15 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-nofeature-ha' + default: 'os-odl-nofeature-ha' - node: name: SLAVE_NAME description: 'Slave name on Jenkins' @@ -137,8 +66,6 @@ - ericsson-pod1 default-slaves: - ericsson-pod2 - - fuel-project-parameter: - gs-pathname: '{gs-pathname}' scm: - git-scm @@ -150,14 +77,3 @@ builders: - shell: !include-raw-escape: ./fuel-deploy.sh - -######################## -# parameter macros -######################## -- parameter: - name: fuel-project-parameter - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-verify-jobs.yml b/jjb/fuel/fuel-verify-jobs.yml index 899be9a4a..45197fc4e 100644 --- a/jjb/fuel/fuel-verify-jobs.yml +++ b/jjb/fuel/fuel-verify-jobs.yml @@ -21,11 +21,11 @@ ##################################### phase: - 'basic': - slave-label: 'opnfv-build-ubuntu' + slave-label: 'fuel-virtual' - 'deploy-virtual': - slave-label: 'opnfv-build-ubuntu' + slave-label: 'fuel-virtual' - 'smoke-test': - slave-label: 'opnfv-build-ubuntu' + slave-label: 'fuel-virtual' ##################################### # jobs ##################################### @@ -50,6 +50,11 @@ enabled: true max-total: 4 option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-os-.*?-virtual-daily-.*' + block-level: 'NODE' scm: - git-scm-gerrit @@ -94,8 +99,9 @@ - project-parameter: project: '{project}' branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - 'fuel-verify-defaults': + - 'fuel-virtual-defaults': + installer: '{installer}' + - '{installer}-defaults': gs-pathname: '{gs-pathname}' builders: @@ -155,7 +161,8 @@ - logrotate-default - throttle: enabled: true - max-total: 6 + max-total: 2 + max-per-node: 1 option: 'project' - build-blocker: use-build-blocker: true @@ -177,8 +184,9 @@ project: '{project}' branch: '{branch}' - '{slave-label}-defaults' - - '{installer}-defaults' - - 'fuel-verify-defaults': + - 'fuel-virtual-defaults': + installer: '{installer}' + - '{installer}-defaults': gs-pathname: '{gs-pathname}' builders: @@ -199,10 +207,8 @@ - builder: name: 'fuel-verify-deploy-virtual-macro' builders: - - shell: | - #!/bin/bash - - echo "Not activated!" + - shell: + !include-raw: ./fuel-deploy.sh - builder: name: 'fuel-verify-smoke-test-macro' @@ -211,13 +217,3 @@ #!/bin/bash echo "Not activated!" -##################################### -# parameter macros -##################################### -- parameter: - name: 'fuel-verify-defaults' - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-weekly-jobs.yml b/jjb/fuel/fuel-weekly-jobs.yml index c681c6252..e1563ea38 100644 --- a/jjb/fuel/fuel-weekly-jobs.yml +++ b/jjb/fuel/fuel-weekly-jobs.yml @@ -72,6 +72,7 @@ blocking-jobs: - 'fuel-os-.*?-{pod}-daily-.*' - 'fuel-os-.*?-{pod}-weekly-.*' + - 'fuel-verify-.*' block-level: 'NODE' wrappers: @@ -85,14 +86,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO default: '{scenario}' - - fuel-weekly-parameter: - gs-pathname: '{gs-pathname}' builders: - description-setter: @@ -148,14 +148,13 @@ - project-parameter: project: '{project}' branch: '{branch}' - - '{installer}-defaults' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' - '{slave-label}-defaults': installer: '{installer}' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-nofeature-ha' - - fuel-weekly-parameter: - gs-pathname: '{gs-pathname}' + default: 'os-odl-nofeature-ha' - string: name: DEPLOY_TIMEOUT default: '150' @@ -180,16 +179,6 @@ - email-jenkins-admins-on-failure ######################## -# parameter macros -######################## -- parameter: - name: fuel-weekly-parameter - parameters: - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." -######################## # trigger macros ######################## #----------------------------------------------- diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index f0e08e171..cee302699 100644 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -70,7 +70,7 @@ set +e tiers=(healthcheck smoke features vnf) for tier in ${tiers[@]}; do - FUNCTEST_IMAGE=ollivier/functest-${tier} + FUNCTEST_IMAGE=opnfv/functest-${tier} echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." docker pull ${FUNCTEST_IMAGE}>/dev/null cmd="docker run ${envs} ${volumes} ${FUNCTEST_IMAGE}" diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml index 07d5df46e..c25e4ab4a 100644 --- a/jjb/functest/functest-project-jobs.yml +++ b/jjb/functest/functest-project-jobs.yml @@ -9,6 +9,7 @@ jobs: - 'functest-verify-{stream}' + - 'functest-verify-{phase}-{stream}' - 'functest-docs-upload-{stream}' stream: @@ -21,11 +22,21 @@ gs-pathname: '/{stream}' disabled: true + phase: + - 'unit-tests-and-docs': + slave-label: 'opnfv-build-ubuntu' + - 'build-x86_64': + slave-label: 'opnfv-build-ubuntu' + - 'build-aarch64': + slave-label: 'opnfv-build-ubuntu-arm' + - job-template: name: 'functest-verify-{stream}' disabled: '{obj:disabled}' + project-type: 'multijob' + parameters: - project-parameter: project: '{project}' @@ -36,6 +47,109 @@ - git-scm-gerrit triggers: + - 'functest-verify-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - shell: | + #!/bin/bash + # we do nothing here as the main stuff will be done + # in phase jobs + echo "Triggering phase jobs!" + - multijob: + name: 'functest-build-and-unittest' + execution-type: PARALLEL + projects: + - name: 'functest-verify-unit-tests-and-docs-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-verify-build-x86_64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=x86_64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + - name: 'functest-verify-build-aarch64-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + ARCH=aarch64 + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: false + +- job-template: + name: 'functest-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + wrappers: + - ssh-agent-wrapper + - build-timeout: + timeout: 30 + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{slave-label}-defaults' + + scm: + - git-scm-gerrit + + builders: + - 'functest-verify-{phase}-builders-macro' + + publishers: + - 'functest-verify-{phase}-publishers-macro' + +- job-template: + name: 'functest-docs-upload-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + + scm: + - git-scm + + triggers: + - 'functest-docs-upload-triggers-macro': + project: '{project}' + branch: '{branch}' + + builders: + - functest-upload-doc-artifact + +################################ +# job triggers +################################ +- trigger: + name: 'functest-verify-triggers-macro' + triggers: - gerrit: server-name: 'gerrit.opnfv.org' trigger-on: @@ -58,44 +172,8 @@ forbidden-file-paths: - compare-type: ANT pattern: 'docs/**|.gitignore' - - builders: - - functest-unit-tests-and-docs-build - - publishers: - - junit: - results: nosetests.xml - - cobertura: - report-file: "coverage.xml" - only-stable: "true" - health-auto-update: "true" - stability-auto-update: "true" - zoom-coverage-chart: "true" - targets: - - files: - healthy: 10 - unhealthy: 20 - failing: 30 - - method: - healthy: 50 - unhealthy: 40 - failing: 30 - - email-jenkins-admins-on-failure - -- job-template: - name: 'functest-docs-upload-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: '{project}' - branch: '{branch}' - - 'opnfv-build-ubuntu-defaults' - - scm: - - git-scm - +- trigger: + name: 'functest-docs-upload-triggers-macro' triggers: - gerrit: server-name: 'gerrit.opnfv.org' @@ -113,23 +191,60 @@ forbidden-file-paths: - compare-type: ANT pattern: 'docs/**|.gitignore' - - builders: - - functest-upload-doc-artifact - ################################ # job builders ################################ - builder: - name: functest-unit-tests-and-docs-build + name: 'functest-verify-unit-tests-and-docs-builders-macro' builders: - shell: | cd $WORKSPACE && tox - - builder: - name: functest-upload-doc-artifact + name: 'functest-verify-build-x86_64-builders-macro' + builders: + - shell: | + echo "Not activated!" +- builder: + name: 'functest-verify-build-aarch64-builders-macro' + builders: + - shell: | + echo "Not activated!" +- builder: + name: 'functest-upload-doc-artifact' builders: - shell: | cd $WORKSPACE && tox -edocs wget -O - https://git.opnfv.org/releng/plain/utils/upload-artifact.sh | bash -s "api/_build" "docs" +################################ +# job publishers +################################ +- publisher: + name: 'functest-verify-unit-tests-and-docs-publishers-macro' + publishers: + - junit: + results: nosetests.xml + - cobertura: + report-file: "coverage.xml" + only-stable: "true" + health-auto-update: "true" + stability-auto-update: "true" + zoom-coverage-chart: "true" + targets: + - files: + healthy: 10 + unhealthy: 20 + failing: 30 + - method: + healthy: 50 + unhealthy: 40 + failing: 30 + - email-jenkins-admins-on-failure +- publisher: + name: 'functest-verify-build-x86_64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure +- publisher: + name: 'functest-verify-build-aarch64-publishers-macro' + publishers: + - email-jenkins-admins-on-failure diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml index 10afd6f8c..ecef6d3b7 100644 --- a/jjb/global/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -44,7 +44,7 @@ description: 'IP of the salt master (for mcp deployments)' - string: name: SSH_KEY - default: '/tmp/mcp.rsa' + default: "$HOME/opnfv/mcp.rsa" description: 'Path to private SSH key to access environment nodes' - string: name: INSTALLER_TYPE @@ -58,6 +58,10 @@ name: BRIDGE default: 'pxebr' description: 'Bridge(s) to be used by salt master' + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "URL to Google Storage." - parameter: name: 'joid-defaults' diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml index 75c4323e6..a4ee4d9f1 100644 --- a/jjb/global/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -364,21 +364,6 @@ # Parameters for build slaves ##################################################### - parameter: - name: 'opnfv-build-enea-defaults' - parameters: - - label: - name: SLAVE_LABEL - default: 'opnfv-build-enea' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: 'Git URL to use on this Jenkins Slave' - - string: - name: BUILD_DIRECTORY - default: $WORKSPACE/build_output - description: "Directory where the build artifact will be located upon the completion of the build." - -- parameter: name: 'opnfv-build-centos-defaults' parameters: - label: diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index ac1ec07f4..721b5dede 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -5,6 +5,7 @@ jobs: - 'prune-docker-images' - 'archive-repositories' + - 'check-status-of-slaves' ######################## # job templates @@ -62,3 +63,27 @@ builders: - shell: !include-raw-escape: opnfv-repo-archiver.sh + +- job-template: + name: 'check-status-of-slaves' + + disabled: false + + concurrent: true + + parameters: + - node: + name: SLAVE_NAME + description: We don't want workspace wiped. so I just threw the script on the master + default-slaves: + - master + allowed-multiselect: false + ignore-offline-nodes: true + + triggers: + - timed: '@midnight' + + builders: + - shell: | + cd /opt/jenkins-ci/slavemonitor + bash slave-monitor-0.1.sh | sort diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 8d1ee55a4..61d25585c 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -75,7 +75,7 @@ - patchset-created-event: exclude-drafts: 'false' exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' + exclude-no-code-change: 'true' - draft-published-event - comment-added-contains-event: comment-contains-value: 'recheck' @@ -87,19 +87,16 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' - file-paths: - - compare-type: ANT - pattern: 'xci/**' - disable-strict-forbidden-file-verification: 'true' + disable-strict-forbidden-file-verification: 'false' forbidden-file-paths: - compare-type: ANT pattern: 'bifrost/**' - compare-type: ANT - pattern: 'openstack-ansible/**' + pattern: 'prototypes/**' - compare-type: ANT - pattern: 'puppet-infracloud/**' + pattern: 'upstream/**' - compare-type: ANT - pattern: 'README.rst' + pattern: '**/README.rst' readable-message: true parameters: diff --git a/utils/slave-monitor-0.1.sh b/utils/slave-monitor-0.1.sh new file mode 100644 index 000000000..161aaef21 --- /dev/null +++ b/utils/slave-monitor-0.1.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#This will put a bunch of files in the pwd. you have been warned. +#Counts how long slaves have been online or offline + + +#Yes I know about jq +curlcommand() { +curl -s "https://build.opnfv.org/ci/computer/api/json?tree=computer\[displayName,offline\]" \ + | awk -v k=":" '{n=split($0,a,","); for (i=1; i<=n; i++) print a[i]}' \ + | grep -v "_class" \ + | awk 'NR%2{printf "%s ",$0;next;}1' \ + | awk -F":" '{print $2,$3}' \ + | awk '{print $1,$3}' \ + | sed s,\},,g \ + | sed s,],,g \ + | sed s,\",,g +} + +if [ -f podoutput-current ]; then + cp podoutput-current podoutput-lastiteration +fi + +curlcommand > podoutput-current + +declare -A slavescurrent slaveslastiteration + +while read -r name status ; do + slavescurrent["$name"]="$status" +done < <(cat podoutput-current) + +while read -r name status ; do + slaveslastiteration["$name"]=$status +done < <(cat podoutput-lastiteration) + +main () { +for slavename in "${!slavescurrent[@]}"; do + #Slave is online. Mark it down. + if [ "${slavescurrent[$slavename]}" == "false" ]; then + + if [ -f "$slavename"-offline ]; then + echo "removing offline status from $slavename slave was offline for $(cat "$slavename"-offline ) iterations" + rm "$slavename"-offline + fi + + if ! [ -f "$slavename"-online ]; then + echo "1" > "$slavename"-online + elif [ -f "$slavename"-online ]; then + #read and increment slavename + read -r -d $'\x04' var < "$slavename"-online + ((var++)) + echo -n "ONLINE $slavename " + echo "for $var iterations" + echo "$var" > "$slavename"-online + fi + fi + + #went offline since last iteration. + if [ "${slavescurrent[$slavename]}" == "false" ] && [ "${slaveslastiteration[$slavename]}" == "true" ]; then + echo "JUST WENT OFFLINE $slavename " + if [ -f "$slavename"-online ]; then + echo "removing online status from $slavename. slave was online for $(cat "$slavename"-online ) iterations" + rm "$slavename"-online + fi + + fi + + #slave is offline + if [ "${slavescurrent[$slavename]}" == "true" ]; then + if ! [ -f "$slavename"-offline ]; then + echo "1" > "$slavename"-offline + fi + + if [ -f "$slavename"-offline ]; then + #read and increment slavename + read -r -d $'\x04' var < "$slavename"-offline + ((var++)) + echo "$var" > "$slavename"-offline + if [ "$var" -gt "30" ]; then + echo "OFFLINE FOR $var ITERATIONS REMOVE $slavename " + else + echo "OFFLINE $slavename FOR $var ITERATIONS " + fi + fi + fi + +done +} + +main diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html index 7ce36ca7c..e366670a9 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html +++ b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/pods.html @@ -1,8 +1,12 @@ -<h3>{{ctrl.pageHeader}}</h3> -<p>{{ctrl.pageParagraph}}</p> +<h3>Pods</h3> +<p>This page is used to create or query pods.<br> + Querying pods is open to everybody.<br> + But only login users are granted the privilege to create the new pod. +</p> + <div class="row" style="margin-bottom:24px;"></div> -<div class="pod-create"> +<div class="pod-create" ng-class="{ 'hidden': ! auth.isAuthenticated }"> <h4>Create</h4> <div class="row"> <div ng-repeat="require in ctrl.createRequirements"> diff --git a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js index 201258619..894fcc152 100644 --- a/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js +++ b/utils/test/testapi/3rd_party/static/testapi-ui/components/pods/podsController.js @@ -20,7 +20,7 @@ .controller('PodsController', PodsController); PodsController.$inject = [ - '$scope', '$http', '$filter', '$state', 'testapiApiUrl','raiseAlert' + '$rootScope', '$scope', '$http', '$filter', '$state', 'testapiApiUrl','raiseAlert' ]; /** @@ -52,9 +52,6 @@ ctrl.mode = 'metal'; ctrl.details = ''; - ctrl.pageHeader = 'Pods'; - ctrl.pageParagraph = 'This page is used to create or query pods.'; - /** * This is called when the date filter calendar is opened. It * does some event handling, and sets a scope variable so the UI diff --git a/utils/test/testapi/MANIFEST.in b/utils/test/testapi/MANIFEST.in new file mode 100644 index 000000000..0ba1808ba --- /dev/null +++ b/utils/test/testapi/MANIFEST.in @@ -0,0 +1 @@ +recursive-include 3rd_party
\ No newline at end of file diff --git a/utils/test/testapi/docker/prepare-env.sh b/utils/test/testapi/docker/prepare-env.sh index b14bc2448..92a0c9fd7 100755 --- a/utils/test/testapi/docker/prepare-env.sh +++ b/utils/test/testapi/docker/prepare-env.sh @@ -10,5 +10,5 @@ if [ "$base_url" != "" ]; then sudo crudini --set --existing $FILE api url $base_url/api/v1 sudo crudini --set --existing $FILE ui url $base_url sudo echo "{\"testapiApiUrl\": \"$base_url/api/v1\"}" > \ - /usr/local/lib/python2.7/dist-packages/opnfv_testapi/static/testapi-ui/config.json + /usr/local/share/opnfv_testapi/testapi-ui/config.json fi diff --git a/utils/test/testapi/etc/config.ini b/utils/test/testapi/etc/config.ini index db0e191d1..8d0bde20b 100644 --- a/utils/test/testapi/etc/config.ini +++ b/utils/test/testapi/etc/config.ini @@ -21,6 +21,11 @@ authenticate = False [ui] url = http://localhost:8000 +# this path should be the seem with data_files installation path +static_path = /usr/local/share/opnfv_testapi + [lfid] # Linux Foundation cas URL cas_url = https://identity.linuxfoundation.org/cas/ +#service url used to authenticate to cas +signin_return = api/v1/auth/signin_return diff --git a/utils/test/testapi/opnfv_testapi/cmd/server.py b/utils/test/testapi/opnfv_testapi/cmd/server.py index 50ac049a0..b7d3caa20 100644 --- a/utils/test/testapi/opnfv_testapi/cmd/server.py +++ b/utils/test/testapi/opnfv_testapi/cmd/server.py @@ -38,7 +38,7 @@ from opnfv_testapi.tornado_swagger import swagger def make_app(): swagger.docs(base_url=CONF.ui_url, - static_path=CONF.static_path) + static_path=CONF.ui_static_path) return swagger.Application( url_mappings.mappings, debug=CONF.api_debug, diff --git a/utils/test/testapi/opnfv_testapi/common/check.py b/utils/test/testapi/opnfv_testapi/common/check.py index 009d3d46c..24ba876a9 100644 --- a/utils/test/testapi/opnfv_testapi/common/check.py +++ b/utils/test/testapi/opnfv_testapi/common/check.py @@ -8,49 +8,14 @@ ############################################################################## import functools -import cas from tornado import gen from tornado import web -from opnfv_testapi.common import constants from opnfv_testapi.common import message from opnfv_testapi.common import raises -from opnfv_testapi.common.config import CONF from opnfv_testapi.db import api as dbapi -def login(method): - @web.asynchronous - @gen.coroutine - @functools.wraps(method) - def wrapper(self, *args, **kwargs): - ticket = self.get_query_argument('ticket', default=None) - if ticket: - client = cas.CASClient(version='2', - server_url=CONF.lfid_cas_url, - service_url=CONF.ui_url) - (user, attrs, _) = client.verify_ticket(ticket=ticket) - print 'login user: {}'.format(user) - login_user = { - 'user': user, - 'email': attrs.get('mail'), - 'fullname': attrs.get('field_lf_full_name'), - 'groups': constants.TESTAPI_USERS + attrs.get('group', []) - } - q_user = {'user': user} - db_user = yield dbapi.db_find_one(constants.USER_TABLE, q_user) - if not db_user: - dbapi.db_save(constants.USER_TABLE, login_user) - else: - dbapi.db_update(constants.USER_TABLE, q_user, login_user) - - self.clear_cookie(constants.TESTAPI_ID) - self.set_secure_cookie(constants.TESTAPI_ID, user) - ret = yield gen.coroutine(method)(self, *args, **kwargs) - raise gen.Return(ret) - return wrapper - - def authenticate(method): @web.asynchronous @gen.coroutine diff --git a/utils/test/testapi/opnfv_testapi/common/config.py b/utils/test/testapi/opnfv_testapi/common/config.py index 4cd53c619..140e49283 100644 --- a/utils/test/testapi/opnfv_testapi/common/config.py +++ b/utils/test/testapi/opnfv_testapi/common/config.py @@ -16,14 +16,10 @@ import sys class Config(object): def __init__(self): - self.config_file = None + self.config_file = '/etc/opnfv_testapi/config.ini' self._set_config_file() self._parse() self._parse_per_page() - self.static_path = os.path.join( - os.path.dirname(os.path.normpath(__file__)), - os.pardir, - 'static') def _parse(self): if not os.path.exists(self.config_file): @@ -56,23 +52,12 @@ class Config(object): return value def _set_config_file(self): - if not self._set_sys_config_file(): - self._set_default_config_file() - - def _set_sys_config_file(self): parser = argparse.ArgumentParser() parser.add_argument("-c", "--config-file", dest='config_file', help="Config file location", metavar="FILE") args, _ = parser.parse_known_args(sys.argv) - try: + if hasattr(args, 'config_file') and args.config_file: self.config_file = args.config_file - finally: - return self.config_file is not None - - def _set_default_config_file(self): - is_venv = os.getenv('VIRTUAL_ENV') - self.config_file = os.path.join('/' if not is_venv else is_venv, - 'etc/opnfv_testapi/config.ini') CONF = Config() diff --git a/utils/test/testapi/opnfv_testapi/common/constants.py b/utils/test/testapi/opnfv_testapi/common/constants.py index b37ebb3d6..70c922383 100644 --- a/utils/test/testapi/opnfv_testapi/common/constants.py +++ b/utils/test/testapi/opnfv_testapi/common/constants.py @@ -2,4 +2,3 @@ TESTAPI_ID = 'testapi_id' CSRF_TOKEN = 'csrf_token' ROLE = 'role' TESTAPI_USERS = ['opnfv-testapi-users'] -USER_TABLE = 'users' diff --git a/utils/test/testapi/opnfv_testapi/router/url_mappings.py b/utils/test/testapi/opnfv_testapi/router/url_mappings.py index be6240e70..ce0a3eeb3 100644 --- a/utils/test/testapi/opnfv_testapi/router/url_mappings.py +++ b/utils/test/testapi/opnfv_testapi/router/url_mappings.py @@ -72,10 +72,11 @@ mappings = [ # static path (r'/(.*\.(css|png|gif|js|html|json|map|woff2|woff|ttf))', tornado.web.StaticFileHandler, - {'path': CONF.static_path}), + {'path': CONF.ui_static_path}), (r'/', root.RootHandler), (r'/api/v1/auth/signin', sign.SigninHandler), + (r'/{}'.format(CONF.lfid_signin_return), sign.SigninReturnHandler), (r'/api/v1/auth/signout', sign.SignoutHandler), (r'/api/v1/profile', user.UserHandler), diff --git a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py index 01cd0f7c3..318473ea2 100644 --- a/utils/test/testapi/opnfv_testapi/ui/auth/sign.py +++ b/utils/test/testapi/opnfv_testapi/ui/auth/sign.py @@ -1,22 +1,59 @@ from cas import CASClient +from tornado import gen +from tornado import web from opnfv_testapi.common import constants from opnfv_testapi.common.config import CONF +from opnfv_testapi.db import api as dbapi from opnfv_testapi.resources import handlers -class SigninHandler(handlers.GenericApiHandler): +class SignBaseHandler(handlers.GenericApiHandler): + def __init__(self, application, request, **kwargs): + super(SignBaseHandler, self).__init__(application, request, **kwargs) + self.table = 'users' + self.cas_client = CASClient(version='2', + server_url=CONF.lfid_cas_url, + service_url='{}/{}'.format( + CONF.ui_url, + CONF.lfid_signin_return)) + + +class SigninHandler(SignBaseHandler): + def get(self): + self.redirect(url=(self.cas_client.get_login_url())) + + +class SigninReturnHandler(SignBaseHandler): + + @web.asynchronous + @gen.coroutine def get(self): - client = CASClient(version='2', - server_url=CONF.lfid_cas_url, - service_url=CONF.ui_url) - self.redirect(url=(client.get_login_url())) + ticket = self.get_query_argument('ticket', default=None) + if ticket: + (user, attrs, _) = self.cas_client.verify_ticket(ticket=ticket) + login_user = { + 'user': user, + 'email': attrs.get('mail'), + 'fullname': attrs.get('field_lf_full_name'), + 'groups': constants.TESTAPI_USERS + attrs.get('group', []) + } + q_user = {'user': user} + db_user = yield dbapi.db_find_one(self.table, q_user) + if not db_user: + dbapi.db_save(self.table, login_user) + else: + dbapi.db_update(self.table, q_user, login_user) + + self.clear_cookie(constants.TESTAPI_ID) + self.set_secure_cookie(constants.TESTAPI_ID, user) + + self.redirect(url=CONF.ui_url) -class SignoutHandler(handlers.GenericApiHandler): +class SignoutHandler(SignBaseHandler): def get(self): """Handle signout request.""" self.clear_cookie(constants.TESTAPI_ID) - client = CASClient(version='2', - server_url=CONF.lfid_cas_url) - self.redirect(url=(client.get_logout_url(redirect_url=CONF.ui_url))) + logout_url = self.cas_client.get_logout_url(redirect_url=CONF.ui_url) + self.redirect(url=logout_url) diff --git a/utils/test/testapi/opnfv_testapi/ui/root.py b/utils/test/testapi/opnfv_testapi/ui/root.py index 069ad5e93..286a6b097 100644 --- a/utils/test/testapi/opnfv_testapi/ui/root.py +++ b/utils/test/testapi/opnfv_testapi/ui/root.py @@ -1,12 +1,10 @@ -from opnfv_testapi.common import check from opnfv_testapi.common.config import CONF from opnfv_testapi.resources import handlers class RootHandler(handlers.GenericApiHandler): def get_template_path(self): - return CONF.static_path + return CONF.ui_static_path - @check.login def get(self): self.render('testapi-ui/index.html') diff --git a/utils/test/testapi/setup.cfg b/utils/test/testapi/setup.cfg index ab1ef553e..d9aa6762e 100644 --- a/utils/test/testapi/setup.cfg +++ b/utils/test/testapi/setup.cfg @@ -23,18 +23,10 @@ setup-hooks = [files] packages = opnfv_testapi -package_data = - opnfv_testapi = - static/*.* - static/*/*.* - static/*/*/*.* - static/*/*/*/*.* - static/*/*/*/*/*.* - static/*/*/*/*/*/*.* - static/*/*/*/*/*/*/*.* + data_files = - /etc/opnfv_testapi = - etc/config.ini + /etc/opnfv_testapi = etc/config.ini + /usr/local/share/opnfv_testapi = 3rd_party/static/* [entry_points] console_scripts = @@ -44,4 +36,3 @@ console_scripts = tag_build = tag_date = 0 tag_svn_revision = 0 - diff --git a/utils/test/testapi/setup.py b/utils/test/testapi/setup.py index dd52373fd..f9d95a32d 100644 --- a/utils/test/testapi/setup.py +++ b/utils/test/testapi/setup.py @@ -1,6 +1,3 @@ -import os -import subprocess - import setuptools __author__ = 'serena' @@ -10,11 +7,7 @@ try: except ImportError: pass -dirpath = os.path.dirname(os.path.abspath(__file__)) -subprocess.call(['ln', '-s', - '{}/3rd_party/static'.format(dirpath), - '{}/opnfv_testapi/static'.format(dirpath)]) setuptools.setup( - setup_requires=['pbr==2.0.0'], + setup_requires=['pbr>=2.0.0'], pbr=True) diff --git a/utils/test/testapi/tools/watchdog/docker_watch.sh b/utils/test/testapi/tools/watchdog/docker_watch.sh index d67e4b380..786fc10b9 100644 --- a/utils/test/testapi/tools/watchdog/docker_watch.sh +++ b/utils/test/testapi/tools/watchdog/docker_watch.sh @@ -21,33 +21,19 @@ modules=(testapi reporting) declare -A ports=( ["testapi"]="8082" ["reporting"]="8084") ## Urls to check if the modules are deployed or not ? -#declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ -# ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") - -declare -A urls=( ["testapi"]="http://localhost:8082/" \ +declare -A urls=( ["testapi"]="http://testresults.opnfv.org/test/" \ ["reporting"]="http://testresults.opnfv.org/reporting2/reporting/index.html") - ### Functions related to checking. function is_deploying() { - echo -e "Checking job statuses" - for module in "${modules[@]}" - do - if get_status $module; then - exit 0 - fi - done -} - -function get_status() { xml=$(curl -m10 "https://build.opnfv.org/ci/job/${1}-automate-master/lastBuild/api/xml?depth=1") building=$(grep -oPm1 "(?<=<building>)[^<]+" <<< "$xml") if [[ $building == "false" ]] then - return 1 - else return 0 + else + return 1 fi } @@ -78,6 +64,9 @@ function check_modules() { failed_modules=() for module in "${modules[@]}" do + if is_deploying $module; then + continue + fi if ! check_connectivity $module "${urls[$module]}"; then echo -e "$module failed" failed_modules+=($module) @@ -110,27 +99,30 @@ function docker_proxy_fix() { echo $pid if [ ! -z "$pid" ]; then kill $pid - start_containers_fix $module + start_container_fix $module fi done } function start_containers_fix() { - echo "Runnning start_containers_fix" start_modules=("${@}") for module in "${start_modules[@]}" do - echo -e "Starting a container $module" + start_container_fix $module + done +} + +function start_container_fix() { + echo -e "Starting a container $module" + sudo docker stop $module + sudo docker start $module + sleep 5 + if ! check_connectivity $module "${urls[$module]}"; then + echo -e "Starting an old container $module_old" sudo docker stop $module - sudo docker start $module + sudo docker start $module"_old" sleep 5 - if ! check_connectivity $module "${urls[$module]}"; then - echo -e "Starting an old container $module_old" - sudo docker stop $module - sudo docker start $module"_old" - sleep 5 - fi - done + fi } ### Main Flow @@ -141,11 +133,6 @@ echo -e echo -e `date "+%Y-%m-%d %H:%M:%S.%N"` echo -e -if ! is_deploying; then - echo -e "Jenkins Jobs running" - exit -fi - ## If the problem is related to docker daemon if get_docker_status; then @@ -156,16 +143,16 @@ if get_docker_status; then exit fi -## If the problem is related to docker containers +## If the problem is related to docker proxy if ! check_modules; then - start_containers_fix "${failed_modules[@]}" + docker_proxy_fix "${failed_modules[@]}" fi -## If the problem is related to docker proxy +## If any other problem : restart docker if ! check_modules; then - docker_proxy_fix "${failed_modules[@]}" + restart_docker_fix fi ## If nothing works out @@ -175,4 +162,4 @@ if ! check_modules; then fi sudo docker ps -sudo docker images
\ No newline at end of file +sudo docker images |