diff options
| -rw-r--r-- | jjb/apex/apex-verify-jobs.yml | 2 | ||||
| -rw-r--r-- | jjb/apex/apex.yml | 12 | ||||
| -rw-r--r-- | jjb/apex/apex.yml.j2 | 2 | ||||
| -rw-r--r-- | jjb/apex/scenarios.yaml.hidden | 1 | ||||
| -rw-r--r-- | jjb/armband/armband-ci-jobs.yml | 41 | ||||
| -rwxr-xr-x | jjb/daisy4nfv/daisy-deploy.sh | 20 | ||||
| -rwxr-xr-x | jjb/dovetail/dovetail-run.sh | 76 | ||||
| -rw-r--r-- | jjb/fuel/fuel-daily-jobs.yml | 69 | ||||
| -rw-r--r-- | jjb/global/releng-macros.yml | 10 | ||||
| -rw-r--r-- | jjb/releng/compass4nfv-docker.yml | 11 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.sh | 7 | ||||
| -rw-r--r-- | jjb/releng/opnfv-docker.yml | 2 | ||||
| -rw-r--r-- | jjb/xci/osa-periodic-jobs.yml | 194 | ||||
| -rw-r--r-- | jjb/xci/xci-verify-jobs.yml | 20 | ||||
| -rw-r--r-- | modules/opnfv/deployment/daisy/__init__.py | 0 | ||||
| -rw-r--r-- | modules/opnfv/deployment/daisy/adapter.py | 202 | ||||
| -rw-r--r-- | modules/opnfv/deployment/factory.py | 5 | ||||
| -rw-r--r-- | modules/opnfv/utils/ssh_utils.py | 14 | ||||
| -rw-r--r-- | utils/create_pod_file.py | 3 |
19 files changed, 508 insertions, 183 deletions
diff --git a/jjb/apex/apex-verify-jobs.yml b/jjb/apex/apex-verify-jobs.yml index c57ac1d60..88c1b1765 100644 --- a/jjb/apex/apex-verify-jobs.yml +++ b/jjb/apex/apex-verify-jobs.yml @@ -157,6 +157,8 @@ - compare-type: ANT pattern: 'build/**' - compare-type: ANT + pattern: 'ci/**' + - compare-type: ANT pattern: 'lib/**' - compare-type: ANT pattern: 'config/**' diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index eadfbf9ee..6714d6a66 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -35,7 +35,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'euphrates' - disable_daily: false + disable_daily: true - danube: &danube branch: 'stable/danube' gs-pathname: '/danube' @@ -110,6 +110,8 @@ <<: *master - 'os-odl-bgpvpn-ha': <<: *master + - 'os-odl-bgpvpn-noha': + <<: *master - 'os-ovn-nofeature-noha': <<: *master - 'os-nosdn-fdio-noha': @@ -1335,6 +1337,14 @@ kill-phase-on: NEVER abort-all-job: true git-revision: false + - name: 'apex-os-odl-bgpvpn-noha-baremetal-master' + node-parameters: false + current-parameters: false + predefined-parameters: | + OPNFV_CLEAN=yes + kill-phase-on: NEVER + abort-all-job: true + git-revision: false - name: 'apex-os-ovn-nofeature-noha-baremetal-master' node-parameters: false current-parameters: false diff --git a/jjb/apex/apex.yml.j2 b/jjb/apex/apex.yml.j2 index f49c7fa1a..27a854dd7 100644 --- a/jjb/apex/apex.yml.j2 +++ b/jjb/apex/apex.yml.j2 @@ -35,7 +35,7 @@ baremetal-slave: 'apex-baremetal-master' verify-scenario: 'os-odl-nofeature-ha' scenario_stream: 'euphrates' - disable_daily: false + disable_daily: true - danube: &danube branch: 'stable/danube' gs-pathname: '/danube' diff --git a/jjb/apex/scenarios.yaml.hidden b/jjb/apex/scenarios.yaml.hidden index 98b698dd8..789ca7f7e 100644 --- a/jjb/apex/scenarios.yaml.hidden +++ b/jjb/apex/scenarios.yaml.hidden @@ -4,6 +4,7 @@ master: - 'os-odl-nofeature-ha' - 'os-odl-nofeature-noha' - 'os-odl-bgpvpn-ha' + - 'os-odl-bgpvpn-noha' - 'os-ovn-nofeature-noha' - 'os-nosdn-fdio-noha' - 'os-nosdn-fdio-ha' diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 0202ef0e7..b2fa62f87 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -158,34 +158,21 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.dovetail only master, based on D release - # 2.here the stream means the SUT stream, - # dovetail stream is defined in its own job - # 3.only proposed_tests testsuite here(refstack, ha, ipv6, bgpvpn) - # 4.not used for release criteria or compliance, + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) + # 3.not used for release criteria or compliance, # only to debug the dovetail tool bugs with arm pods - # 5.only run against scenario os-(nosdn|odl)-(nofeature-bgpvpn)-ha - - conditional-step: - condition-kind: and - condition-operands: - - condition-kind: regex-match - regex: os-(nosdn|odl)-(nofeature|bgpvpn)-ha - label: '{scenario}' - - condition-kind: regex-match - regex: 'danube' - label: '{stream}' - steps: - - trigger-builds: - - project: 'dovetail-{installer}-{pod}-proposed_tests-master' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' + - trigger-builds: + - project: 'dovetail-{installer}-{pod}-proposed_tests-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' # Armband uses Fuel's log collection project job, no need to duplicate - conditional-step: condition-kind: not diff --git a/jjb/daisy4nfv/daisy-deploy.sh b/jjb/daisy4nfv/daisy-deploy.sh index 803ff5b03..1723fd109 100755 --- a/jjb/daisy4nfv/daisy-deploy.sh +++ b/jjb/daisy4nfv/daisy-deploy.sh @@ -19,14 +19,22 @@ fi # clone the securedlab repo cd $WORKSPACE -SECURELAB_DIR=/var/tmp/opnfv-securedlab -echo "Cloning securedlab repo $BRANCH to $SECURELAB_DIR" -rm -rf $SECURELAB_DIR -git clone ssh://jenkins-zte@gerrit.opnfv.org:29418/securedlab --quiet \ - --branch $BRANCH $SECURELAB_DIR +# There are no PDFs in euphrates branch of pharos repo. +if [[ "$BRANCH" =~ "euphrates" ]]; then + CONFIG_REPO_NAME=securedlab +else + CONFIG_REPO_NAME=pharos +fi + +LABS_DIR=/var/tmp/opnfv-${CONFIG_REPO_NAME} + +echo "Cloning ${CONFIG_REPO_NAME} repo $BRANCH to $LABS_DIR" +rm -rf $LABS_DIR +git clone ssh://jenkins-zte@gerrit.opnfv.org:29418/${CONFIG_REPO_NAME} \ + --quiet --branch $BRANCH $LABS_DIR -DEPLOY_COMMAND="sudo -E ./ci/deploy/deploy.sh -L $SECURELAB_DIR \ +DEPLOY_COMMAND="sudo -E ./ci/deploy/deploy.sh -L $LABS_DIR \ -l $LAB_NAME -p $POD_NAME -B $BRIDGE -s $DEPLOY_SCENARIO" # log info to console diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index e084e4bd0..e50242bd6 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -13,6 +13,9 @@ set -e [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" +DEPLOY_TYPE=baremetal +[[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt + DOVETAIL_HOME=${WORKSPACE}/cvp [ -d ${DOVETAIL_HOME} ] && sudo rm -rf ${DOVETAIL_HOME} @@ -21,6 +24,8 @@ mkdir -p ${DOVETAIL_HOME} DOVETAIL_CONFIG=${DOVETAIL_HOME}/pre_config mkdir -p ${DOVETAIL_CONFIG} +ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + sshkey="" # The path of openrc.sh is defined in fetch_os_creds.sh OPENRC=${DOVETAIL_CONFIG}/env_config.sh @@ -73,13 +78,17 @@ if [[ -f $OPENRC ]]; then exit 1 fi fi - cat $OPENRC else echo "ERROR: cannot find file $OPENRC. Please check if it is existing." sudo ls -al ${DOVETAIL_CONFIG} exit 1 fi +if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "fuel" ]]; then + sed -i "s#/etc/ssl/certs/mcp_os_cacert#${CACERT}#g" ${OPENRC} +fi +cat $OPENRC + if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == "compass" ]]; then cat << EOF >${DOVETAIL_CONFIG}/pod.yaml nodes: @@ -92,6 +101,19 @@ nodes: EOF fi +if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then + fuel_ctl_ssh_options="${ssh_options} -i ${SSH_KEY}" + ssh_user="ubuntu" + fuel_ctl_ip=$(ssh 2>/dev/null ${fuel_ctl_ssh_options} "${ssh_user}@${INSTALLER_IP}" \ + "sudo salt --out yaml 'ctl*' pillar.get _param:openstack_control_address | \ + awk '{print \$2; exit}'") &> /dev/null + cat << EOF >${DOVETAIL_CONFIG}/pod.yaml +nodes: +- {ip: ${fuel_ctl_ip}, name: node1, key_filename: /root/.ssh/id_rsa, role: controller, user: ${ssh_user}} + +EOF +fi + if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then set +e @@ -109,6 +131,8 @@ if [[ ! -f ${DOVETAIL_CONFIG}/pod.yaml ]]; then options="-u root -p r00tme" elif [[ ${INSTALLER_TYPE} == apex ]]; then options="-u stack -k /root/.ssh/id_rsa" + elif [[ ${INSTALLER_TYPE} == daisy ]]; then + options="-u root -p r00tme" else echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently." echo "HA test cases may not run properly." @@ -135,11 +159,13 @@ else echo "HA test cases may not run properly." fi -ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - if [ "$INSTALLER_TYPE" == "fuel" ]; then - echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa + if [[ "${SUT_BRANCH}" =~ "danube" ]]; then + echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." + sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa + else + cp ${SSH_KEY} ${DOVETAIL_CONFIG}/id_rsa + fi fi if [ "$INSTALLER_TYPE" == "apex" ]; then @@ -147,6 +173,12 @@ if [ "$INSTALLER_TYPE" == "apex" ]; then sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${DOVETAIL_CONFIG}/id_rsa fi +if [ "$INSTALLER_TYPE" == "daisy" ]; then + echo "Fetching id_dsa file from jump_server $INSTALLER_IP..." + sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_dsa ${DOVETAIL_CONFIG}/id_rsa +fi + + image_path=${HOME}/opnfv/dovetail/images if [[ ! -d ${image_path} ]]; then mkdir -p ${image_path} @@ -174,20 +206,26 @@ docker_volume="-v /var/run/docker.sock:/var/run/docker.sock" dovetail_home_volume="-v ${DOVETAIL_HOME}:${DOVETAIL_HOME}" # Pull the image with correct tag -echo "Dovetail: Pulling image opnfv/dovetail:${DOCKER_TAG}" -docker pull opnfv/dovetail:$DOCKER_TAG >$redirect +DOCKER_REPO='opnfv/dovetail' +if [ "$(uname -m)" = 'aarch64' ]; then + DOCKER_REPO="${DOCKER_REPO}_$(uname -m)" + DOCKER_TAG="latest" +fi + +echo "Dovetail: Pulling image ${DOCKER_REPO}:${DOCKER_TAG}" +docker pull ${DOCKER_REPO}:$DOCKER_TAG >$redirect env4bgpvpn="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP}" cmd="docker run ${opts} -e DOVETAIL_HOME=${DOVETAIL_HOME} ${docker_volume} ${dovetail_home_volume} \ - ${sshkey} ${env4bgpvpn} opnfv/dovetail:${DOCKER_TAG} /bin/bash" + ${sshkey} ${env4bgpvpn} ${DOCKER_REPO}:${DOCKER_TAG} /bin/bash" echo "Dovetail: running docker run command: ${cmd}" ${cmd} >${redirect} sleep 5 -container_id=$(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | awk '{print $1}' | head -1) +container_id=$(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | awk '{print $1}' | head -1) echo "Container ID=${container_id}" if [ -z ${container_id} ]; then - echo "Cannot find opnfv/dovetail container ID ${container_id}. Please check if it is existing." + echo "Cannot find ${DOCKER_REPO} container ID ${container_id}. Please check if it is existing." docker ps -a exit 1 fi @@ -195,11 +233,25 @@ echo "Container Start: docker start ${container_id}" docker start ${container_id} sleep 5 docker ps >${redirect} -if [ $(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | wc -l) == 0 ]; then - echo "The container opnfv/dovetail with ID=${container_id} has not been properly started. Exiting..." +if [ $(docker ps | grep "${DOCKER_REPO}:${DOCKER_TAG}" | wc -l) == 0 ]; then + echo "The container ${DOCKER_REPO} with ID=${container_id} has not been properly started. Exiting..." exit 1 fi +if [[ ! "${SUT_BRANCH}" =~ "danube" && ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then + source_cmd="source ${OPENRC}" + get_public_url_cmd="openstack --insecure endpoint list --service keystone --interface public | sed -n 4p | awk '{print \$14}'" + public_url=$(sudo docker exec "$container_id" /bin/bash -c "${source_cmd} && ${get_public_url_cmd}") + sed -i 's#OS_AUTH_URL=.*#OS_AUTH_URL='"${public_url}"'#g' ${OPENRC} + sed -i 's/internal/public/g' ${OPENRC} + if [[ ${public_url} =~ 'v2' ]]; then + sed -i "s/OS_IDENTITY_API_VERSION=3/OS_IDENTITY_API_VERSION=2.0/g" ${OPENRC} + sed -i '/OS_PROJECT_DOMAIN_NAME/d' ${OPENRC} + sed -i '/OS_USER_DOMAIN_NAME/d' ${OPENRC} + fi + cat ${OPENRC} +fi + # Modify tempest_conf.yaml file tempest_conf_file=${DOVETAIL_CONFIG}/tempest_conf.yaml if [[ ${INSTALLER_TYPE} == 'compass' || ${INSTALLER_TYPE} == 'apex' ]]; then diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 45b45f387..902e75499 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -85,6 +85,8 @@ auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-onos-nofeature-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' + - 'os-ovn-nofeature-noha': + auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-kvm-noha': auto-trigger-name: 'fuel-{scenario}-{pod}-daily-{stream}-trigger' - 'os-nosdn-ovs-noha': @@ -198,34 +200,21 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - # 1.dovetail only has master, based on D release - # 2.here the stream means the SUT stream, dovetail stream is defined in its own job - # 3.only debug testsuite here(refstack, ha, ipv6, bgpvpn) - # 4.not used for release criteria or compliance, - # only to debug the dovetail tool bugs with bgpvpn and nosdn-nofeature - # 5.only run against scenario os-odl-bgpvpn-ha(regex used here, can extend to more scenarios future) - # 6.ZTE pod1, os-nosdn-nofeature-ha and os-odl-bgpvpn-ha, run against danube - - conditional-step: - condition-kind: and - condition-operands: - - condition-kind: regex-match - regex: os-(nosdn-nofeature|odl_l2-bgpvpn)-ha - label: '{scenario}' - - condition-kind: regex-match - regex: 'danube' - label: '{stream}' - steps: - - trigger-builds: - - project: 'dovetail-fuel-{pod}-proposed_tests-master' - current-parameters: false - predefined-parameters: - DEPLOY_SCENARIO={scenario} - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' + # 1.here the stream means the SUT stream, dovetail stream is defined in its own job + # 2.only debug testsuite here(refstack, ha, vping, ipv6, tempest, bgpvpn) + # 3.not used for release criteria or compliance, only to debug the dovetail tool bugs + # 4.ZTE pod1, os-nosdn-nofeature-ha and os-odl-bgpvpn-ha, run against danube + - trigger-builds: + - project: 'dovetail-fuel-{pod}-proposed_tests-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' - conditional-step: condition-kind: not condition-operand: @@ -400,6 +389,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-noha-baremetal-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-master-trigger' triggers: - timed: '' @@ -469,6 +462,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-noha-baremetal-daily-euphrates-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-noha-baremetal-daily-euphrates-trigger' triggers: - timed: '' @@ -537,6 +534,10 @@ triggers: - timed: '' # '5 23 * * *' - trigger: + name: 'fuel-os-ovn-nofeature-noha-virtual-daily-master-trigger' + triggers: + - timed: '5 23 * * *' +- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-master-trigger' triggers: - timed: '' # '35 6 * * *' @@ -605,6 +606,10 @@ triggers: - timed: '' # '0 23 * * *' - trigger: + name: 'fuel-os-ovn-nofeature-noha-virtual-daily-euphrates-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-noha-virtual-daily-euphrates-trigger' triggers: - timed: '' # '30 6 * * *' @@ -673,6 +678,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-noha-zte-pod1-daily-master-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-master-trigger' triggers: - timed: '' @@ -741,6 +750,10 @@ triggers: - timed: '' - trigger: + name: 'fuel-os-ovn-nofeature-noha-zte-pod1-daily-euphrates-trigger' + triggers: + - timed: '' +- trigger: name: 'fuel-os-nosdn-kvm-noha-zte-pod1-daily-euphrates-trigger' triggers: - timed: '' diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index 86f4c4265..aeeb24610 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -76,6 +76,16 @@ submodule: recursive: true timeout: 20 + +- scm: + name: git-scm-openstack + scm: + - git: &git-scm-openstack-defaults + url: '$GIT_BASE' + branches: + - 'origin/$BRANCH' + timeout: 15 + - trigger: name: 'daily-trigger-disabled' triggers: diff --git a/jjb/releng/compass4nfv-docker.yml b/jjb/releng/compass4nfv-docker.yml index 299908d66..db2e4273b 100644 --- a/jjb/releng/compass4nfv-docker.yml +++ b/jjb/releng/compass4nfv-docker.yml @@ -26,8 +26,7 @@ - 'cobbler' - 'db' - 'deck' - - 'tasks-k8s' - - 'tasks-osa' + - 'tasks-base' # settings for jobs run in multijob phases build-job-settings: &build-job-settings @@ -78,10 +77,10 @@ builders: - multijob: - name: 'build compass-tasks images' + name: 'build compass-tasks-base images' execution-type: PARALLEL projects: - - name: 'compass-tasks-build-amd64-{stream}' + - name: 'compass-tasks-base-build-amd64-{stream}' <<: *build-job-settings - multijob: name: 'build all compass images' @@ -94,9 +93,7 @@ <<: *build-job-settings - name: 'compass-deck-build-amd64-{stream}' <<: *build-job-settings - - name: 'compass-tasks-k8s-build-amd64-{stream}' - <<: *build-job-settings - - name: 'compass-tasks-osa-build-amd64-{stream}' + - name: 'compass-tasks-build-amd64-{stream}' <<: *build-job-settings publishers: diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index ec7b3fda3..7f646f1d9 100644 --- a/jjb/releng/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -61,7 +61,9 @@ done # Remove the existing containers and images before building remove_containers_images -cd "$WORKSPACE/$DOCKER_DIR" || exit 1 +DOCKER_PATH=$WORKSPACE/$DOCKER_DIR + +cd $DOCKER_PATH || exit 1 HOST_ARCH="$(uname -m)" #If there is a patch for other arch then x86, apply the patch and #replace Dockerfile file @@ -107,7 +109,8 @@ echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" echo "--------------------------------------------------------" echo cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$BUILD_BRANCH - -f $DOCKERFILE ." + $ARCH_BUILD_ARG + -f $DOCKERFILE $DOCKER_PATH" echo ${cmd} ${cmd} diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml index 49b3fde2c..3351a5e94 100644 --- a/jjb/releng/opnfv-docker.yml +++ b/jjb/releng/opnfv-docker.yml @@ -61,6 +61,8 @@ <<: *other-receivers - 'qtip': project: 'qtip' + dockerdir: '.' + dockerfile: 'docker/Dockerfile.local' <<: *master <<: *other-receivers - 'storperf-master': diff --git a/jjb/xci/osa-periodic-jobs.yml b/jjb/xci/osa-periodic-jobs.yml index 26c1575f4..64daa696b 100644 --- a/jjb/xci/osa-periodic-jobs.yml +++ b/jjb/xci/osa-periodic-jobs.yml @@ -2,7 +2,7 @@ - project: name: 'opnfv-osa-periodic' - project: 'releng-xci' + project: openstack-ansible # ------------------------------- # branches # ------------------------------- @@ -13,19 +13,19 @@ # distros # ------------------------------- distro: - - 'xenial': + - ubuntu: + disabled: false + - centos: + disabled: false + - opensuse: disabled: false - - 'centos7': - disabled: true - - 'suse': - disabled: true # ------------------------------- # type # ------------------------------- type: - virtual # ------------------------------- - # phases + # periodic deploy & test phases # ------------------------------- phase: - 'deploy' @@ -34,32 +34,37 @@ # jobs # ------------------------------- jobs: - - 'osa-periodic-{distro}-{type}-{stream}' - - 'osa-periodic-{phase}-{type}-{stream}' - + - 'xci-osa-periodic-{distro}-{type}-{stream}' + - 'xci-osa-periodic-{distro}-{phase}-{type}-{stream}' # ------------------------------- # job templates # ------------------------------- - job-template: - name: 'osa-periodic-{distro}-{type}-{stream}' + name: 'xci-osa-periodic-{distro}-{type}-{stream}' project-type: multijob disabled: '{obj:disabled}' - concurrent: false + concurrent: true properties: - logrotate-default - build-blocker: use-build-blocker: true blocking-jobs: - - 'xci-verify-.*' - - 'bifrost-verify-.*' - - 'bifrost-periodic-.*' - - 'osa-verify-.*' - - 'osa-periodic-.*' + - 'xci-verify-{distro}-.*' + - 'bifrost-verify-{distro}-.*' + - 'bifrost-periodic-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' block-level: 'NODE' + - throttle: + max-per-node: 2 + max-total: 10 + categories: + - xci-verify-virtual + option: category wrappers: - ssh-agent-wrapper @@ -68,12 +73,7 @@ - fix-workspace-permissions scm: - - git-scm-osa - - triggers: - - pollscm: - cron: "@midnight" - ignore-post-commit-hooks: true + - git-scm-openstack parameters: - project-parameter: @@ -81,7 +81,17 @@ branch: '{branch}' - label: name: SLAVE_LABEL - default: 'xci-virtual-{distro}' + default: 'xci-virtual' + - string: + name: OPENSTACK_OSA_VERSION + default: 'master' + - string: + name: CLEAN_DIB_IMAGES + default: 'true' + - string: + name: GIT_BASE + default: 'https://git.openstack.org/openstack/$PROJECT' + description: 'Git URL to use on this Jenkins Slave' builders: - description-setter: @@ -90,32 +100,38 @@ name: deploy condition: SUCCESSFUL projects: - - name: 'osa-periodic-deploy-{type}-{stream}' + - name: 'xci-osa-periodic-{distro}-deploy-{type}-{stream}' current-parameters: true predefined-parameters: | DISTRO={distro} DEPLOY_SCENARIO=os-nosdn-nofeature-noha - git-revision: true + OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + BRANCH=$BRANCH node-parameters: true kill-phase-on: FAILURE abort-all-job: true + git-revision: true - multijob: name: healthcheck condition: SUCCESSFUL projects: - - name: 'osa-periodic-healthcheck-{type}-{stream}' + - name: 'xci-osa-periodic-{distro}-healthcheck-{type}-{stream}' current-parameters: true predefined-parameters: | DISTRO={distro} DEPLOY_SCENARIO=os-nosdn-nofeature-noha + OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES FUNCTEST_MODE=tier FUNCTEST_TIER=healthcheck + BRANCH=$BRANCH node-parameters: true kill-phase-on: NEVER - abort-all-job: false + abort-all-job: true - job-template: - name: 'osa-periodic-{phase}-{type}-{stream}' + name: 'xci-osa-periodic-{distro}-{phase}-{type}-{stream}' disabled: false @@ -126,41 +142,23 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'xci-verify-deploy-.*' - - 'xci-verify-healthcheck-.*' - - 'bifrost-verify-.*' - - 'bifrost-periodic-.*' - - 'osa-verify-deploy-.*' - - 'osa-verify-halthcheck-.*' - - 'osa-periodic-deploy-.*' - - 'osa-periodic-healthcheck-.*' + - '.*-bifrost-verify-.*' + - '.*-bifrost-periodic-.*' block-level: 'NODE' parameters: - project-parameter: project: '{project}' branch: '{branch}' - - label: - name: SLAVE_LABEL - default: 'xci-virtual-{distro}' - - string: - name: OPENSTACK_OSA_VERSION - default: 'master' - string: name: DISTRO - default: 'xenial' + default: 'ubuntu' - string: name: DEPLOY_SCENARIO default: 'os-nosdn-nofeature-noha' - string: - name: XCI_FLAVOR - default: 'mini' - - string: - name: XCI_LOOP - default: 'periodic' - - string: - name: OPNFV_RELENG_DEV_PATH - default: $WORKSPACE/releng-xci + name: OPENSTACK_OSA_VERSION + default: 'master' - string: name: FUNCTEST_MODE default: 'tier' @@ -168,14 +166,18 @@ name: FUNCTEST_SUITE_NAME default: 'healthcheck' - string: - name: FORCE_MASTER + name: XCI_FLAVOR + default: 'mini' + - string: + name: CLEAN_DIB_IMAGES default: 'true' - string: + name: INSTALLER_TYPE + default: 'osa' + - string: name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - - scm: - - git-scm-osa + default: 'https://git.openstack.org/openstack/$PROJECT' + description: 'Git URL to use on this Jenkins Slave' wrappers: - ssh-agent-wrapper @@ -183,48 +185,76 @@ timeout: 240 - fix-workspace-permissions + scm: + - git-scm-openstack + builders: - description-setter: description: "Built on $NODE_NAME" - - 'osa-periodic-{phase}-macro' + - 'xci-osa-periodic-{phase}-macro' # ------------------------------- # builder macros # ------------------------------- - builder: - name: 'osa-periodic-deploy-macro' + name: 'xci-osa-periodic-deploy-macro' builders: - shell: | #!/bin/bash - # here we will - # - clone releng-xci repo as the jobs are running against openstack gerrit - # and we need to clone releng-xci ourselves to $OPNFV_RELENG_DEV_PATH - # - run sources-branch-updater.sh from osa to update/pin the role versions - # at the time this job gets triggered against osa master in case if the - # deployment succeeds and we decide to bump version used by xci - # - copy generated role versions into $OPNFV_RELENG_DEV_PATH/xci/file - # - start the deployment by executing xci-deploy.sh as usual - # - # we might also need to pin versions of openstack services as well. + cd $WORKSPACE + + # The start-new-vm.sh script will copy the entire releng-xci directory + # so lets prepare the test script now so it can be copied by the script. + # Please do not move it elsewhere or you would have to move it to the VM + # yourself. + cat > xci_test.sh<<EOF + #!/bin/bash + export DISTRO=$DISTRO + export DEPLOY_SCENARIO=$DEPLOY_SCENARIO + export OPENSTACK_OSA_VERSION=$OPENSTACK_OSA_VERSION + export FUNCTEST_MODE=$FUNCTEST_MODE + export FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + export XCI_FLAVOR=$XCI_FLAVOR + export CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES + export OPNFV_RELENG_DEV_PATH=/home/devuser/releng-xci/ + export INSTALLER_TYPE=$INSTALLER_TYPE + export GIT_BASE=$GIT_BASE + export JENKINS_HOME=$JENKINS_HOME + + cd xci + ./xci-deploy.sh + EOF + chmod a+x xci_test.sh + + export XCI_BUILD_CLEAN_VM_OS=false + export XCI_UPDATE_CLEAN_VM_OS=true + + ./xci/scripts/vm/start-new-vm.sh $DISTRO + - shell: | + #!/bin/bash + + ssh -F $HOME/.ssh/xci-vm-config ${DISTRO}_xci_vm "cd releng-xci && ./xci_test.sh" - echo "Hello World!" - builder: - name: 'osa-periodic-healthcheck-macro' + name: 'xci-osa-periodic-healthcheck-macro' builders: - shell: | #!/bin/bash echo "Hello World!" -# ------------------------------- -# scm macro -# ------------------------------- -- scm: - name: git-scm-osa - scm: - - git: - url: https://review.openstack.org/p/openstack/openstack-ansible.git - branches: - - master - timeout: 15 + - shell: | + #!/bin/bash + + sudo virsh destroy ${DISTRO}_xci_vm + sudo virsh undefine ${DISTRO}_xci_vm + +# this will be enabled once the xci is prepared +# - builder: +# name: 'xci-verify-healthcheck-macro' +# builders: +# - shell: +# !include-raw: ../../utils/fetch_os_creds.sh +# - shell: +# !include-raw: ../functest/functest-alpine.sh diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 9fa5633a0..fffd5c2b4 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -68,8 +68,8 @@ - 'xci-verify-{distro}-.*' - 'bifrost-verify-{distro}-.*' - 'bifrost-periodic-{distro}-.*' - - 'osa-verify-{distro}-.*' - - 'osa-periodic-{distro}-.*' + - 'xci-osa-verify-{distro}-.*' + - 'xci-osa-periodic-{distro}-.*' block-level: 'NODE' - throttle: max-per-node: 2 @@ -106,17 +106,9 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' - disable-strict-forbidden-file-verification: 'true' + disable-strict-forbidden-file-verification: 'false' forbidden-file-paths: - compare-type: ANT - pattern: 'prototypes/**' - - compare-type: ANT - pattern: 'upstream/**' - - compare-type: ANT - pattern: '**/README.rst' - - compare-type: ANT - pattern: 'docs/**' - - compare-type: ANT pattern: 'xci/scripts/vm/**' readable-message: true custom-url: '* $JOB_NAME $BUILD_URL' @@ -152,7 +144,7 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature-noha + DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES GERRIT_BRANCH=$GERRIT_BRANCH GERRIT_REFSPEC=$GERRIT_REFSPEC @@ -169,7 +161,7 @@ current-parameters: true predefined-parameters: | DISTRO={distro} - DEPLOY_SCENARIO=os-nosdn-nofeature-noha + DEPLOY_SCENARIO=os-nosdn-nofeature CLEAN_DIB_IMAGES=$CLEAN_DIB_IMAGES FUNCTEST_MODE=tier FUNCTEST_TIER=healthcheck @@ -208,7 +200,7 @@ default: 'ubuntu' - string: name: DEPLOY_SCENARIO - default: 'os-nosdn-nofeature-noha' + default: 'os-nosdn-nofeature' - string: name: FUNCTEST_MODE default: 'tier' diff --git a/modules/opnfv/deployment/daisy/__init__.py b/modules/opnfv/deployment/daisy/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/modules/opnfv/deployment/daisy/__init__.py diff --git a/modules/opnfv/deployment/daisy/adapter.py b/modules/opnfv/deployment/daisy/adapter.py new file mode 100644 index 000000000..5634e242b --- /dev/null +++ b/modules/opnfv/deployment/daisy/adapter.py @@ -0,0 +1,202 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from opnfv.deployment import manager +from opnfv.utils import opnfv_logger as logger +from opnfv.utils import ssh_utils + +logger = logger.Logger(__name__).getLogger() + + +class DaisyAdapter(manager.DeploymentHandler): + + def __init__(self, installer_ip, installer_user, installer_pwd): + super(DaisyAdapter, self).__init__(installer='daisy', + installer_ip=installer_ip, + installer_user=installer_user, + installer_pwd=installer_pwd, + pkey_file=None) + + def _get_clusters(self): + clusters = [] + cmd = 'source /root/daisyrc_admin; daisy cluster-list | grep -v "+--"' + output = self.installer_node.run_cmd(cmd) + lines = output.rsplit('\n') + if len(lines) < 2: + logger.info("No environments found in the deployment.") + return None + else: + fields = lines[0].rsplit('|') + + index_id = -1 + index_status = -1 + index_name = -1 + index_nodes = -1 + + for i in range(len(fields)): + if "ID" in fields[i]: + index_id = i + elif "Status" in fields[i]: + index_status = i + elif "Name" in fields[i]: + index_name = i + elif "Nodes" in fields[i]: + index_nodes = i + + # order env info + for i in range(1, len(lines)): + fields = lines[i].rsplit('|') + dict = {"id": fields[index_id].strip(), + "status": fields[index_status].strip(), + "name": fields[index_name].strip(), + "nodes": fields[index_nodes].strip()} + clusters.append(dict) + + return clusters + + def get_nodes(self, options=None): + if hasattr(self, 'nodes') and len(self.nodes) > 0: + if options and 'cluster' in options and options['cluster']: + nodes = [] + for node in self.nodes: + if str(node.info['cluster']) == str(options['cluster']): + nodes.append(node) + return nodes + else: + return self.nodes + + clusters = self._get_clusters() + nodes = [] + for cluster in clusters: + if options and 'cluster' in options and options['cluster']: + if cluster["id"] != options['cluster']: + continue + cmd = 'source /root/daisyrc_admin; daisy host-list ' \ + '--cluster-id {} | grep -v "+--"'.format(cluster["id"]) + output = self.installer_node.run_cmd(cmd) + lines = output.rsplit('\n') + if len(lines) < 2: + logger.info("No nodes found in the cluster {}".format( + cluster["id"])) + continue + + fields = lines[0].rsplit('|') + index_id = -1 + index_status = -1 + index_name = -1 + + for i in range(len(fields)): + if "ID" in fields[i]: + index_id = i + elif "Role_status" in fields[i]: + index_status = i + elif "Name" in fields[i]: + index_name = i + + for i in range(1, len(lines)): + fields = lines[i].rsplit('|') + id = fields[index_id].strip().encode() + status_node = fields[index_status].strip().encode().lower() + name = fields[index_name].strip().encode() + ip = ".".join(name.split("-")[1:]) + + cmd_role = 'source /root/daisyrc_admin; ' \ + 'daisy host-detail {} | grep "^| role"'.format(id) + output_role = self.installer_node.run_cmd(cmd_role) + role_all = output_role.rsplit('|')[2].strip().encode() + roles = [] + if 'COMPUTER' in role_all: + roles.append(manager.Role.COMPUTE) + if 'CONTROLLER_LB' in role_all or 'CONTROLLER_HA' in role_all: + roles.append(manager.Role.CONTROLLER) + + ssh_client = None + if status_node == 'active': + status = manager.NodeStatus.STATUS_OK + proxy = {'ip': self.installer_ip, + 'username': self.installer_user, + 'password': self.installer_pwd, + 'pkey_file': '/root/.ssh/id_dsa'} + ssh_client = ssh_utils.get_ssh_client(hostname=ip, + username='root', + proxy=proxy) + else: + status = manager.NodeStatus.STATUS_INACTIVE + + node = DaisyNode(id, ip, name, status, roles, ssh_client) + nodes.append(node) + return nodes + + def get_openstack_version(self): + cmd = 'docker exec nova_api nova-manage version 2>/dev/null' + version = None + for node in self.nodes: + if node.is_controller() and node.is_active(): + version = node.run_cmd(cmd) + break + return version + + def get_sdn_version(self): + version = None + for node in self.nodes: + if manager.Role.CONTROLLER in node.roles and node.is_active(): + cmd = 'docker inspect --format=\'{{.Name}}\' `docker ps -q`' + output = node.run_cmd(cmd) + if '/opendaylight' in output.rsplit('\n'): + cmd2 = 'docker exec opendaylight ' \ + 'sudo yum info opendaylight 2>/dev/null ' \ + '| grep Version | tail -1' + odl_ver = node.run_cmd(cmd2) + if odl_ver: + version = 'OpenDaylight: ' + odl_ver.split(' ')[-1] + break + return version + + def get_deployment_status(self): + clusters = self._get_clusters() + if clusters is None or len(clusters) == 0: + return 'unknown' + else: + return clusters[0]['status'] + + +class DaisyNode(manager.Node): + + def __init__(self, + id, + ip, + name, + status, + roles=None, + ssh_client=None, + info=None): + super(DaisyNode, self).__init__(id, ip, name, status, + roles, ssh_client, info) + + def is_odl(self): + ''' + Returns if the node is an opendaylight + ''' + if manager.Role.CONTROLLER in self.roles and self.is_active(): + cmd = 'docker inspect --format=\'{{.Name}}\' `docker ps -q`' + output = self.run_cmd(cmd) + if '/opendaylight' in output.rsplit('\n'): + return True + return False + + def get_ovs_info(self): + ''' + Returns the ovs version installed + ''' + if self.is_active(): + cmd = 'docker exec openvswitch_vswitchd ' \ + 'ovs-vsctl --version | head -1 | awk \'{print $NF}\'' + return self.run_cmd(cmd) + return None diff --git a/modules/opnfv/deployment/factory.py b/modules/opnfv/deployment/factory.py index e14783fe2..2788e5eaa 100644 --- a/modules/opnfv/deployment/factory.py +++ b/modules/opnfv/deployment/factory.py @@ -12,6 +12,7 @@ from opnfv.deployment.apex import adapter as apex_adapter from opnfv.deployment.compass import adapter as compass_adapter from opnfv.deployment.fuel import adapter as fuel_adapter from opnfv.deployment.osa import adapter as osa_adapter +from opnfv.deployment.daisy import adapter as daisy_adapter from opnfv.utils import opnfv_logger as logger logger = logger.Logger(__name__).getLogger() @@ -51,6 +52,10 @@ class Factory(object): return osa_adapter.OSAAdapter(installer_ip=installer_ip, installer_user=installer_user, pkey_file=pkey_file) + elif installer.lower() == "daisy": + return daisy_adapter.DaisyAdapter(installer_ip=installer_ip, + installer_user=installer_user, + installer_pwd=installer_pwd) else: raise Exception("Installer adapter is not implemented for " "the given installer.") diff --git a/modules/opnfv/utils/ssh_utils.py b/modules/opnfv/utils/ssh_utils.py index 4c5ff5c1b..175a38078 100644 --- a/modules/opnfv/utils/ssh_utils.py +++ b/modules/opnfv/utils/ssh_utils.py @@ -49,9 +49,11 @@ def get_ssh_client(hostname, client = paramiko.SSHClient() else: client = ProxyHopClient() + proxy_pkey_file = proxy.get('pkey_file', '/root/.ssh/id_rsa') client.configure_jump_host(proxy['ip'], proxy['username'], - proxy['password']) + proxy['password'], + proxy_pkey_file) if client is None: raise Exception('Could not connect to client') @@ -115,6 +117,8 @@ class ProxyHopClient(paramiko.SSHClient): jh_ssh_key='/root/.ssh/id_rsa'): self.proxy_ip = jh_ip self.proxy_ssh_key = jh_ssh_key + self.local_ssh_key = os.path.join(os.getcwd(), + jh_ssh_key.split('/')[-1]) self.proxy_ssh = paramiko.SSHClient() self.proxy_ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.proxy_ssh.connect(jh_ip, @@ -138,8 +142,12 @@ class ProxyHopClient(paramiko.SSHClient): self.local_ssh_key) if get_file_res is None: raise Exception('Could\'t fetch SSH key from jump host') - proxy_key = (paramiko.RSAKey - .from_private_key_file(self.local_ssh_key)) + if self.proxy_ssh_key.split('/')[-1] == 'id_dsa': + proxy_key = (paramiko.DSSKey + .from_private_key_file(self.local_ssh_key)) + else: + proxy_key = (paramiko.RSAKey + .from_private_key_file(self.local_ssh_key)) self.proxy_channel = self.proxy_transport.open_channel( "direct-tcpip", diff --git a/utils/create_pod_file.py b/utils/create_pod_file.py index def5ecca8..a60ece475 100644 --- a/utils/create_pod_file.py +++ b/utils/create_pod_file.py @@ -92,6 +92,9 @@ def create_file(handler, INSTALLER_TYPE): if args.INSTALLER_TYPE == 'compass': for item in node_list: item['password'] = 'root' + elif args.INSTALLER_TYPE == 'daisy': + for item in node_list: + item['key_filename'] = '/root/.ssh/id_dsa' else: for item in node_list: item['key_filename'] = args.sshkey |