Generate Docker builds via Xtesting CI

Change-Id: I883d2c088849ba293ebed9541b42d63d3de4567a
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

2 files changed, 112 insertions, 376 deletions

diff --git a/jjb/functest/xtesting-docker.yaml b/jjb/functest/xtesting-docker.yaml
deleted file mode 100644
index a2dec355d..000000000
--- a/jjb/functest/xtesting-docker.yaml
+++ /dev/null
@@ -1,259 +0,0 @@
----
-##############################################
-# job configuration for docker build and push
-##############################################
-- project:
-
-    name: xtesting-docker
-
-    project: functest-xtesting
-
-    stream:
-      - master:
-          branch: '{stream}'
-          disabled: false
-      - leguer:
-          branch: 'stable/{stream}'
-          disabled: false
-      - kali:
-          branch: 'stable/{stream}'
-          disabled: false
-      - jerma:
-          branch: 'stable/{stream}'
-          disabled: false
-      - iruya:
-          branch: 'stable/{stream}'
-          disabled: false
-      - hunter:
-          branch: 'stable/{stream}'
-          disabled: false
-
-    arch_tag:
-      - 'amd64':
-          slave_label: 'lf-build2'
-
-    image:
-      - 'core'
-      - 'mts'
-
-    exclude:
-      - stream: 'kali'
-        image: 'mts'
-      - stream: 'jerma'
-        image: 'mts'
-      - stream: 'iruya'
-        image: 'mts'
-      - stream: 'hunter'
-        image: 'mts'
-
-    # settings for jobs run in multijob phases
-    build-job-settings: &build-job-settings
-      current-parameters: false
-      git-revision: true
-      node-parameters: false
-      predefined-parameters: |
-        PUSH_IMAGE=$PUSH_IMAGE
-        COMMIT_ID=$COMMIT_ID
-        GERRIT_REFNAME=$GERRIT_REFNAME
-        DOCKERFILE=$DOCKERFILE
-      kill-phase-on: FAILURE
-      abort-all-jobs: false
-
-    manifest-job-settings: &manifest-job-settings
-      current-parameters: false
-      git-revision: true
-      node-parameters: false
-      predefined-parameters:
-        GERRIT_REFNAME=$GERRIT_REFNAME
-      kill-phase-on: FAILURE
-      abort-all-jobs: false
-
-    # yamllint enable rule:key-duplicates
-    jobs:
-      - "xtesting-docker-{stream}"
-      - "xtesting-{image}-docker-build-{arch_tag}-{stream}"
-      - "xtesting-{image}-docker-manifest-{stream}"
-
-########################
-# job templates
-########################
-- job-template:
-    name: 'xtesting-docker-{stream}'
-
-    project-type: multijob
-
-    disabled: '{obj:disabled}'
-
-    parameters:
-      - xtesting-job-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          slave_label: 'lf-build2'
-          arch_tag: 'amd64'
-
-    properties:
-      - throttle:
-          max-per-node: 1
-          option: 'project'
-
-    scm:
-      - git-scm
-
-    triggers:
-      - pollscm:
-          cron: "*/30 * * * *"
-      - gerrit-trigger-tag-created:
-          project: '{project}'
-
-    builders:
-      - multijob:
-          name: 'build xtesting images'
-          execution-type: PARALLEL
-          projects:
-            - name: 'xtesting-core-docker-build-amd64-{stream}'
-              <<: *build-job-settings
-      - multijob:
-          name: 'publish xtesting manifests'
-          execution-type: PARALLEL
-          projects:
-            - name: 'xtesting-core-docker-manifest-{stream}'
-              <<: *manifest-job-settings
-      - multijob:
-          name: 'build xtesting-mts images'
-          execution-type: PARALLEL
-          projects:
-            - name: 'xtesting-mts-docker-build-amd64-{stream}'
-              <<: *build-job-settings
-      - multijob:
-          name: 'publish xtesting-mts manifests'
-          execution-type: PARALLEL
-          projects:
-            - name: 'xtesting-mts-docker-manifest-{stream}'
-              <<: *manifest-job-settings
-
-
-    publishers:
-      - 'xtesting-amd64-recipients'
-
-- job-template:
-    name: 'xtesting-{image}-docker-build-{arch_tag}-{stream}'
-    disabled: '{obj:disabled}'
-    parameters:
-      - xtesting-job-parameters:
-          project: '{project}'
-          branch: '{branch}'
-          slave_label: '{slave_label}'
-          arch_tag: '{arch_tag}'
-    scm:
-      - git-scm
-    builders:
-      - shell: |
-          #!/bin/bash -ex
-          sudo arch=amd64 amd64_dirs=docker/{image} arm64_dirs= arm_dirs= bash ./build.sh
-          exit $?
-
-- job-template:
-    name: 'xtesting-{image}-docker-manifest-{stream}'
-
-    parameters:
-      - project-parameter:
-          project: '{project}'
-          branch: '{branch}'
-      - label:
-          name: SLAVE_LABEL
-          default: 'lf-build2'
-          description: 'Slave label on Jenkins'
-          all-nodes: false
-          node-eligibility: 'ignore-offline'
-      - string:
-          name: PROJECT
-          default: "{project}"
-          description: "Project name used to enable job conditions"
-      - string:
-          name: GIT_BASE
-          default: https://gerrit.opnfv.org/gerrit/$PROJECT
-          description: 'Git URL to use on this Jenkins Slave'
-      - string:
-          name: REPO
-          default: "opnfv"
-          description: "Repository name for xtesting images"
-
-
-    disabled: '{obj:disabled}'
-
-    builders:
-      - shell: |
-          #!/bin/bash -ex
-          case "{stream}" in
-          "master")
-              tag="latest" ;;
-          *)
-              tag="{stream}" ;;
-          esac
-          case "{image}" in
-          "core")
-              img="" ;;
-          *)
-              img="-{image}" ;;
-          esac
-          sudo manifest-tool push from-args \
-              --platforms linux/amd64 \
-              --template $REPO/xtesting$img:ARCH-$tag \
-              --target $REPO/xtesting$img:$tag
-          exit $?
-
-- parameter:
-    name: xtesting-job-parameters
-    parameters:
-      - project-parameter:
-          project: '{project}'
-          branch: '{branch}'
-      - label:
-          name: SLAVE_LABEL
-          default: '{slave_label}'
-          description: 'Slave label on Jenkins'
-          all-nodes: false
-          node-eligibility: 'ignore-offline'
-      - string:
-          name: GIT_BASE
-          default: https://gerrit.opnfv.org/gerrit/$PROJECT
-          description: 'Git URL to use on this Jenkins Slave'
-      - string:
-          name: PUSH_IMAGE
-          default: "true"
-          description: "To enable/disable pushing the image to Dockerhub."
-      - string:
-          name: COMMIT_ID
-          default: ""
-          description: "commit id to make a snapshot docker image"
-      - string:
-          name: GERRIT_REFNAME
-          default: ""
-          description: "Docker tag to be built, e.g. refs/tags/5.0.0, refs/tags/opnfv-5.0.0, refs/tags/5.0.RC1"
-      - string:
-          name: DOCKERFILE
-          default: "Dockerfile"
-          description: "Dockerfile to use for creating the image."
-      - string:
-          name: ARCH_TAG
-          default: "{arch_tag}"
-          description: "If set, this value will be added to the docker image tag as a prefix"
-      - string:
-          name: PROJECT
-          default: "{project}"
-          description: "Project name used to enable job conditions"
-      - string:
-          name: REPO
-          default: "opnfv"
-          description: "Repository name for xtesting images"
-
-# publisher macros
-- publisher:
-    name: 'xtesting-amd64-recipients'
-    publishers:
-      - email:
-          recipients: >
-            jalausuch@suse.com morgan.richomme@orange.com
-            cedric.ollivier@orange.com feng.xiaowei@zte.com.cn
-            juha.kosonen@nokia.com wangwulin@huawei.com
-            valentin.boucher@kontron.com
diff --git a/jjb/functest/xtesting.yaml b/jjb/functest/xtesting.yaml
index 7a41b6818..d4b6f9261 100644
--- a/jjb/functest/xtesting.yaml
+++ b/jjb/functest/xtesting.yaml
@@ -41,18 +41,11 @@
           name: build_tag
 
 - parameter:
-    name: xtesting-branch
-    parameters:
-      - string:
-          name: branch
-          default: '{branch}'
-
-- parameter:
     name: xtesting-DEBUG
     parameters:
       - string:
           name: DEBUG
-          default: 'true'
+          default: true
 
 - xtesting-containers: &xtesting-containers
     name: 'xtesting-containers'
@@ -87,7 +80,7 @@
     builders:
       - shell: |
           set +x
-          [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/* || true
+          [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/results || true
           if [ "{repo}" = "_" ]; then
             image={container}:{tag}
           elif [ "{port}" = "None" ]; then
@@ -101,13 +94,13 @@
             -e S3_ENDPOINT_URL=https://storage.googleapis.com \
             -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \
             -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \
+            -v /home/opnfv/xtesting/.boto:/root/.boto \
             -e TEST_DB_URL=http://testresults.opnfv.org/test/api/v1/results \
             -e TEST_DB_EXT_URL=http://testresults.opnfv.org/test/api/v1/results \
             -e NODE_NAME=$slave \
             -e BUILD_TAG=$BUILD_TAG \
             -v $WORKSPACE/../$JOB_NAME/results:/var/lib/xtesting/results \
             -e DEBUG=$DEBUG \
-            -v /home/opnfv/xtesting/.boto:/root/.boto \
             $image run_tests -t {test} -p -r
 
 - builder:
@@ -124,6 +117,13 @@
           fi
           sudo docker rmi $image || true
 
+- parameter:
+    name: xtesting-branch
+    parameters:
+      - string:
+          name: branch
+          default: '{branch}'
+
 - xtesting-build-containers: &xtesting-build-containers
     name: 'xtesting-build-containers'
     <<: *xtesting-containers
@@ -230,7 +230,7 @@
       - xtesting-build_tag:
           build_tag: ''
       - xtesting-DEBUG:
-          DEBUG: 'true'
+          DEBUG: true
     builders:
       - xtesting-run-containers:
           <<: *xtesting-run-containers
@@ -246,13 +246,13 @@
       - fourth
       - fifth
       - sixth
+    privileged: 'false'
+    network: bridge
     exclude:
       - tag: hunter
         test: sixth
       - tag: iruya
         test: sixth
-    privileged: 'false'
-    network: bridge
     jobs:
       - 'xtesting-{repo}-{container}-{tag}-{test}-run'
 
@@ -262,6 +262,8 @@
     container: 'xtesting-mts'
     test:
       - seventh
+    privileged: 'false'
+    network: bridge
     exclude:
       - tag: hunter
         test: seventh
@@ -271,17 +273,15 @@
         test: seventh
       - tag: kali
         test: seventh
-    privileged: 'false'
-    network: bridge
     jobs:
       - 'xtesting-{repo}-{container}-{tag}-{test}-run'
 
+
 - builder:
     name: xtesting-zip
     builders:
       - shell: |
           set +x
-          [ ! -z "$WORKSPACE" ] && sudo rm -rf $WORKSPACE/* || true
           if [ "{repo}" = "_" ]; then
             image={container}:{tag}
           elif [ "{port}" = "None" ]; then
@@ -293,12 +293,12 @@
             -e S3_ENDPOINT_URL=https://storage.googleapis.com \
             -e S3_DST_URL=s3://artifacts.opnfv.org/xtesting \
             -e HTTP_DST_URL=http://artifacts.opnfv.org/xtesting \
+            -v /home/opnfv/xtesting/.boto:/root/.boto \
             -e TEST_DB_URL=http://testresults.opnfv.org/test/api/v1/results \
             -e TEST_DB_EXT_URL=http://testresults.opnfv.org/test/api/v1/results \
             -e BUILD_TAG=$BUILD_TAG \
             -v $WORKSPACE/../$JOB_NAME/results:/var/lib/xtesting/results \
             -e DEBUG=$DEBUG \
-            -v /home/opnfv/xtesting/.boto:/root/.boto \
             $image zip_campaign
 
 - job-template:
@@ -309,7 +309,7 @@
       - xtesting-build_tag:
           build_tag: ''
       - xtesting-DEBUG:
-          DEBUG: 'true'
+          DEBUG: true
     builders:
       - xtesting-zip:
           <<: *xtesting-containers
@@ -332,13 +332,13 @@
       - xtesting-build_tag:
           build_tag: ''
       - xtesting-DEBUG:
-          DEBUG: 'true'
+          DEBUG: true
     properties:
       - build-blocker:
           use-build-blocker: true
           blocking-level: 'NODE'
           blocking-jobs:
-            - '^xtesting-{tag}-(daily|check|gate)$'
+            - '^xtesting-{tag}-(daily|gate)$'
     builders:
       - multijob:
           name: remove former images
@@ -393,21 +393,6 @@
           <<: *xtesting-build-containers
           ref: $GERRIT_REFSPEC
 
-- job-template:
-    name: 'xtesting-{repo}-{container}-{tag}-check'
-    parameters:
-      - xtesting-slave:
-          slave: '{slave}'
-      - xtesting-branch:
-          branch: '{branch}'
-    scm:
-      - xtesting-scm:
-          ref: $branch
-    builders:
-      - xtesting-build-containers:
-          <<: *xtesting-build-containers
-          ref: $branch
-
 - project:
     name: 'xtesting-_-alpine-3.13-rmi'
     repo: _
@@ -429,17 +414,16 @@
       - 'xtesting-{repo}-{container}-{tag}-pull'
 
 - project:
-    name: xtesting-opnfv-xtesting-{tag}-build
+    name: xtesting-opnfv-xtesting-{tag}-gate
     <<: *xtesting-params
     container: xtesting
     ref_arg: BRANCH
     path: docker/core
     jobs:
       - 'xtesting-{repo}-{container}-{tag}-gate'
-      - 'xtesting-{repo}-{container}-{tag}-check'
 
 - project:
-    name: xtesting-opnfv-xtesting-mts-{tag}-build
+    name: xtesting-opnfv-xtesting-mts-{tag}-gate
     <<: *xtesting-params
     container: xtesting-mts
     ref_arg: BRANCH
@@ -451,74 +435,7 @@
       - tag: kali
     jobs:
       - 'xtesting-{repo}-{container}-{tag}-gate'
-      - 'xtesting-{repo}-{container}-{tag}-check'
 
-- job-template:
-    name: 'xtesting-{tag}-check'
-    project-type: multijob
-    parameters:
-      - xtesting-slave:
-          slave: '{slave}'
-      - xtesting-build_tag:
-          build_tag: ''
-      - xtesting-branch:
-          branch: '{branch}'
-      - xtesting-DEBUG:
-          DEBUG: 'true'
-    properties:
-      - build-blocker:
-          use-build-blocker: true
-          blocking-level: 'NODE'
-          blocking-jobs:
-            - '^xtesting-{tag}-(daily|check|gate)$'
-    builders:
-      - multijob:
-          name: remove former images
-          projects:
-            - name: 'xtesting-opnfv-xtesting-{tag}-rmi'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-mts-{tag}-rmi'
-              <<: *xtesting-jobs
-      - multijob:
-          name: remove dependencies
-          projects:
-            - name: 'xtesting-_-alpine-3.13-rmi'
-              <<: *xtesting-jobs
-      - multijob:
-          name: pull dependencies
-          projects:
-            - name: 'xtesting-_-alpine-3.13-pull'
-              <<: *xtesting-jobs
-      - multijob:
-          name: opnfv/xtesting
-          projects:
-            - name: 'xtesting-opnfv-xtesting-{tag}-check'
-              <<: *xtesting-jobs
-      - multijob:
-          name: opnfv/xtesting-mts
-          projects:
-            - name: 'xtesting-opnfv-xtesting-mts-{tag}-check'
-              <<: *xtesting-jobs
-      - multijob:
-          name: opnfv/xtesting:{tag}
-          projects:
-            - name: 'xtesting-opnfv-xtesting-{tag}-first-run'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-{tag}-second-run'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-{tag}-third-run'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-{tag}-fourth-run'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-{tag}-fifth-run'
-              <<: *xtesting-jobs
-            - name: 'xtesting-opnfv-xtesting-{tag}-sixth-run'
-              <<: *xtesting-jobs
-      - multijob:
-          name: opnfv/xtesting-mts:{tag}
-          projects:
-            - name: 'xtesting-opnfv-xtesting-mts-{tag}-seventh-run'
-              <<: *xtesting-jobs
 
 - trigger:
     name: xtesting-patchset-created
@@ -544,7 +461,7 @@
             notbuilt: false
 
 - job-template:
-    name: 'xtesting-{tag}-gate'
+    name: 'xtesting-{tag}-review'
     project-type: multijob
     triggers:
       - xtesting-patchset-created:
@@ -555,13 +472,13 @@
       - xtesting-build_tag:
           build_tag: ''
       - xtesting-DEBUG:
-          DEBUG: 'true'
+          DEBUG: true
     properties:
       - build-blocker:
           use-build-blocker: true
           blocking-level: 'NODE'
           blocking-jobs:
-            - '^xtesting-{tag}-(daily|check|gate)$'
+            - '^xtesting-{tag}-(daily|gate)$'
     builders:
       - multijob:
           name: remove former images
@@ -611,21 +528,99 @@
             - name: 'xtesting-opnfv-xtesting-mts-{tag}-seventh-run'
               <<: *xtesting-jobs
 
+- job-template:
+    name: 'xtesting-{repo}-{container}-{tag}-build'
+    parameters:
+      - xtesting-slave:
+          slave: '{slave}'
+      - xtesting-branch:
+          branch: '{branch}'
+    scm:
+      - xtesting-scm:
+          ref: $branch
+    builders:
+      - xtesting-build-containers:
+          <<: *xtesting-build-containers
+          ref: $branch
+
 - project:
-    name: 'xtesting'
+    name: xtesting-opnfv-xtesting-{tag}-build
     <<: *xtesting-params
+    container: xtesting
+    ref_arg: BRANCH
+    path: docker/core
     jobs:
-      - 'xtesting-{tag}-daily'
+      - 'xtesting-{repo}-{container}-{tag}-build'
 
 - project:
-    name: 'xtesting-gate'
+    name: xtesting-opnfv-xtesting-mts-{tag}-build
     <<: *xtesting-params
+    container: xtesting-mts
+    ref_arg: BRANCH
+    path: docker/mts
+    exclude:
+      - tag: hunter
+      - tag: iruya
+      - tag: jerma
+      - tag: kali
     jobs:
-      - 'xtesting-{tag}-check'
-      - 'xtesting-{tag}-gate'
+      - 'xtesting-{repo}-{container}-{tag}-build'
+
+
+- job-template:
+    name: 'xtesting-{tag}-docker'
+    project-type: multijob
+    parameters:
+      - xtesting-slave:
+          slave: '{slave}'
+      - xtesting-branch:
+          branch: '{branch}'
+      - xtesting-DEBUG:
+          DEBUG: true
+    builders:
+      - multijob:
+          name: remove dependencies
+          projects:
+            - name: 'xtesting-_-alpine-3.13-rmi'
+              <<: *xtesting-jobs
+      - multijob:
+          name: pull dependencies
+          projects:
+            - name: 'xtesting-_-alpine-3.13-pull'
+              <<: *xtesting-jobs
+      - multijob:
+          name: opnfv/xtesting
+          projects:
+            - name: 'xtesting-opnfv-xtesting-{tag}-build'
+              <<: *xtesting-jobs
+      - multijob:
+          name: opnfv/xtesting-mts
+          projects:
+            - name: 'xtesting-opnfv-xtesting-mts-{tag}-build'
+              <<: *xtesting-jobs
+
+- project:
+    name: 'xtesting'
+    <<: *xtesting-params
+    jobs:
+      - 'xtesting-{tag}-daily'
+      - 'xtesting-{tag}-review'
+      - 'xtesting-{tag}-docker'
+
+- view:
+    name: xtesting-daily
+    view-type: list
+    columns:
+      - status
+      - weather
+      - job
+      - last-success
+      - last-failure
+      - last-duration
+    regex: ^xtesting-[a-z-]+-daily$
 
 - view:
-    name: xtesting
+    name: xtesting-review
     view-type: list
     columns:
       - status
@@ -634,10 +629,10 @@
       - last-success
       - last-failure
       - last-duration
-    regex: ^xtesting-(amd64-)*[a-z]+-daily$
+    regex: ^xtesting-[a-z-]+-review$
 
 - view:
-    name: xtesting-gate
+    name: xtesting-docker
     view-type: list
     columns:
       - status
@@ -646,4 +641,4 @@
       - last-success
       - last-failure
       - last-duration
-    regex: ^xtesting-(amd64-)*[a-z]+-gate$
+    regex: ^xtesting-[a-z-]+-docker$