kolla-build-vm.sh is ready for Queens

This PS does:

1. Set default build target to stable/queens, but still fix to pike
for OPNFV master branch, please see line:83

2. Create mappings between branch name and version number in code,
since upstream kolla does not do it automatically using git tag
for us start from Queens relese. Therefore, we do not need to let
the user to define the version code and then check it with git tag
command return value any more. So -t argument is deleted.

3. Support building Queens images.

4. Arrange patches in sub-directories name by version code so that
user can use this one script to patch and build any version they
want. But for OPNFV Jenkins, only the version for the current branch
can be built according to the -j argument.

5. Upgrade DPDK version from 16.11 to 17.02.

6. Call cleanup_registry_server before pack_registry_data, this is
resonable for keeping registry data stable.

Change-Id: Ic5a7e2da261d456a2a1481ad0e765a7ce28b8121
Signed-off-by: Zhijiang Hu <hu.zhijiang@zte.com.cn>

3 files changed, 129 insertions, 27 deletions

diff --git a/ci/kolla-build-vm.sh b/ci/kolla-build-vm.sh
index 48a84894..30113cfe 100755
--- a/ci/kolla-build-vm.sh
+++ b/ci/kolla-build-vm.sh
@@ -13,12 +13,11 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-KOLLA_BRANCH="stable/ocata"
+KOLLA_BRANCH="stable/queens"
 OPNFV_JOB_NAME=
-KOLLA_TAG=
 EXT_TAG=
 KOLLA_GIT_VERSION=
-KOLLA_IMAGE_VERSION=
+KOLLA_IMAGE_VERSION=6.0.0
 
 SCRIPT_PATH=$(readlink -f $(dirname $0))
 WORKSPACE=$(cd ${SCRIPT_PATH}/..; pwd)
@@ -38,21 +37,19 @@ usage: `basename $0` [options]
 OPTIONS:
   -b  Kolla git repo branch
   -j  OPNFV job name
-  -t  Kolla git repo code tag(base version of image)
   -e  user defined tag extension(extended version)
   -w  working directroy
 
 Examples:
 sudo `basename $0` -b stable/ocata
                    -j daisy-docker-build-euphrates
-                   -t 4.0.2
                    -e .1
                    -w /path/to/the/working/dir
 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
 EOF
 }
 
-while getopts "b:j:t:e:w:h" OPTION
+while getopts "b:j:e:w:h" OPTION
 do
     case $OPTION in
         b)
@@ -61,9 +58,6 @@ do
         j)
             OPNFV_JOB_NAME=${OPTARG}
             ;;
-        t)
-            KOLLA_TAG=${OPTARG}
-            ;;
         e)
             EXT_TAG=${OPTARG}
             ;;
@@ -101,12 +95,14 @@ fi
 function pre_check {
     echo "Pre setup"
     if [ $KOLLA_BRANCH == "stable/mitaka" ] ; then
+        KOLLA_IMAGE_VERSION=2.0.0
         yum install -y epel-release centos-release-openstack-mitaka
         RPM_REQUIRES="python-docker-py:1.6 python-pbr:1.6 python-jinja2:2.8 \
             python-gitdb:0.6.4 GitPython:1.0.1 python-six:1.9.0 \
             python2-oslo-config:3.7.0 python-beautifulsoup4:4.4.1 \
             python2-setuptools:16.0.0 python2-crypto:2.6 docker-engine:1.12"
     elif [ $KOLLA_BRANCH == "stable/newton" ] ; then
+        KOLLA_IMAGE_VERSION=3.0.0
         yum install -y epel-release centos-release-openstack-newton
         RPM_REQUIRES="python-docker-py:1.6 python-pbr:1.6 python-jinja2:2.8 \
             python-gitdb:0.6.4 GitPython:1.0.1 python-six:1.9.0 \
@@ -114,6 +110,7 @@ function pre_check {
             python2-setuptools:16.0.0 python2-crypto:2.6 docker-engine:1.12 \
             centos-release-openstack-newton:1 epel-release:7"
     elif [ $KOLLA_BRANCH == "stable/ocata" ] ; then
+        KOLLA_IMAGE_VERSION=4.0.0
         yum install -y epel-release centos-release-openstack-ocata
         yum update -y
         yum install -y python-docker-py python2-pbr python-jinja2 \
@@ -126,6 +123,7 @@ function pre_check {
             python2-setuptools:22.0.0 python2-crypto:2.6 docker:1.12 \
             centos-release-openstack-ocata:1 epel-release:7"
     elif [ $KOLLA_BRANCH == "stable/pike" ] ; then
+        KOLLA_IMAGE_VERSION=5.0.0
         yum install -y epel-release centos-release-openstack-pike
         yum update -y
         yum install -y python2-docker python2-pbr python2-jinja2 \
@@ -137,6 +135,19 @@ function pre_check {
             python2-oslo-config:3.22.0 python-netaddr:0.7.18 \
             python2-setuptools:22.0.0 python2-crypto:2.6 docker:1.12 \
             centos-release-openstack-pike:1 epel-release:7"
+    elif [ $KOLLA_BRANCH == "stable/queens" ] ; then
+        KOLLA_IMAGE_VERSION=6.0.0
+        yum install -y epel-release centos-release-openstack-queens
+        yum update -y
+        yum install -y python2-docker python2-pbr python2-jinja2 \
+            python-gitdb GitPython python2-six \
+            python2-oslo-config python2-netaddr \
+            python2-setuptools python2-crypto docker
+        RPM_REQUIRES="python2-docker:2.4.2 python2-pbr:3.1.1 python2-jinja2:2.8 \
+            python-gitdb:0.6.4 GitPython:1.0.1 python2-six:1.10.0 \
+            python2-oslo-config:4.6.0 python2-netaddr:0.7.18 \
+            python2-setuptools:22.0.0 python2-crypto:2.6 docker-engine:1.12 \
+            centos-release-openstack-queens:1 epel-release:7"
     else
         exit 1
     fi
@@ -176,7 +187,7 @@ function pre_check {
     pip install tox
 
     # Just make sure docker is working.
-    service docker restart
+    /usr/bin/systemctl restart docker
 }
 
 function cleanup_registry_server {
@@ -257,25 +268,12 @@ function update_kolla_code {
     pushd $KOLLA_GIT_DIR/kolla
     git checkout $KOLLA_BRANCH
 
-    if [[ ! -z "$KOLLA_TAG" ]]; then
-        git checkout $KOLLA_TAG
-    fi
-
     # Apply patches for openstack/kolla project
-    cp $WORKSPACE/ci/kolla_patches/*.patch ./
+    cp $WORKSPACE/ci/kolla_patches/$KOLLA_IMAGE_VERSION/*.patch ./
     git apply *.patch
 
     KOLLA_GIT_VERSION=$(git log -1 --pretty="%H")
     tox -e genconfig
-    KOLLA_IMAGE_VERSION=$(cat $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf \
-        | grep "#tag" | gawk -F' = ' '{print $2}')
-
-    if [[ ! -z "$KOLLA_TAG" ]]; then
-        if ["$KOLLA_TAG" != $KOLLA_IMAGE_VERSION] ; then
-            echo "tag in git: $KOLLA_TAG, while tag in code: $KOLLA_IMAGE_VERSION"
-            exit 1
-        fi
-    fi
 
     popd
     popd
@@ -300,8 +298,7 @@ function config_kolla_with_dpdksource {
 
     crudini --set $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf  ovsdpdk-plugin-dpdk type git
     crudini --set $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf  ovsdpdk-plugin-dpdk location http://dpdk.org/git/dpdk
-    #crudini --set $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf  ovsdpdk-plugin-dpdk reference v17.02
-    crudini --set $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf  ovsdpdk-plugin-dpdk reference v16.11
+    crudini --set $KOLLA_GIT_DIR/kolla/etc/kolla/kolla-build.conf  ovsdpdk-plugin-dpdk reference v17.02
 
     mkdir -p /etc/kolla/
     rm -rf /etc/kolla/kolla-build.conf
@@ -360,5 +357,6 @@ cleanup_registry_data
 start_registry_server
 
 start_build
-cleanup_kolla_image
+cleanup_registry_server
 pack_registry_data
+cleanup_kolla_image
diff --git a/ci/kolla_patches/0000-kolla-odl-ovsdpdk.patch b/ci/kolla_patches/5.0.0/0000-kolla-odl-ovsdpdk.patch
index 3bf3f85a..3bf3f85a 100644
--- a/ci/kolla_patches/0000-kolla-odl-ovsdpdk.patch
+++ b/ci/kolla_patches/5.0.0/0000-kolla-odl-ovsdpdk.patch
diff --git a/ci/kolla_patches/6.0.0/0000-kolla.patch b/ci/kolla_patches/6.0.0/0000-kolla.patch
new file mode 100644
index 00000000..c4b51a3e
--- /dev/null
+++ b/ci/kolla_patches/6.0.0/0000-kolla.patch
@@ -0,0 +1,104 @@
+diff --git a/contrib/template-override/ovs-dpdk.j2 b/contrib/template-override/ovs-dpdk.j2
+index 4252002..3a2a20a 100644
+--- a/contrib/template-override/ovs-dpdk.j2
++++ b/contrib/template-override/ovs-dpdk.j2
+@@ -69,7 +69,7 @@ WORKDIR "/dpdk/build"
+                            "s/CONFIG_RTE_BUILD_SHARED_LIB=.*$/CONFIG_RTE_BUILD_SHARED_LIB=n/"])
+ }}
+ 
+-RUN make -j $(nproc) EXTRA_CFLAGS='-O2 -fPIC' \
++RUN make -j $(nproc) EXTRA_CFLAGS='-O2 -fPIC -Wno-maybe-uninitialized' \
+     && make install
+ WORKDIR /ovs
+ RUN ./boot.sh \
+diff --git a/docker/base/grafana.repo b/docker/base/grafana.repo
+index dc8d30b..58fee69 100644
+--- a/docker/base/grafana.repo
++++ b/docker/base/grafana.repo
+@@ -2,6 +2,6 @@
+ name=grafana
+ baseurl=https://packagecloud.io/grafana/stable/el/7/$basearch
+ repo_gpgcheck=1
+-enabled=1
++enabled=0
+ gpgcheck=1
+ gpgkey=https://packagecloud.io/gpg.key https://grafanarel.s3.amazonaws.com/RPM-GPG-KEY-grafana
+diff --git a/docker/neutron/neutron-bgp-dragent/Dockerfile.j2 b/docker/neutron/neutron-bgp-dragent/Dockerfile.j2
+deleted file mode 100644
+index a43d3d7..0000000
+--- a/docker/neutron/neutron-bgp-dragent/Dockerfile.j2
++++ /dev/null
+@@ -1,36 +0,0 @@
+-FROM {{ namespace }}/{{ image_prefix }}neutron-base:{{ tag }}
+-LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build_date }}"
+-
+-{% block neutron_bgp_dragent_header %}{% endblock %}
+-
+-{% import "macros.j2" as macros with context %}
+-
+-{% if install_type == 'binary' %}
+-    {% if base_distro in ['centos', 'oraclelinux', 'rhel']  %}
+-RUN echo '{{ install_type }} not yet available for {{ base_distro }}' \
+-    && /bin/false
+-    {% elif base_distro in ['ubuntu', 'debian'] %}
+-        {% set neutron_bgp_dragent_packages = [
+-            'neutron-bgp-dragent'
+-        ] %}
+-    {% endif %}
+-
+-{{ macros.install_packages(neutron_bgp_dragent_packages | customizable("packages")) }}
+-
+-{% elif install_type == 'source'  %}
+-
+-ADD neutron-bgp-dragent-archive /neutron-bgp-dragent-source
+-
+-    {% set neutron_bgp_dragent_pip_packages = [
+-        '/neutron_dynamic_routing'
+-    ] %}
+-
+-RUN ln -s neutron-bgp-dragent-source/* neutron_dynamic_routing \
+-    && {{ macros.install_pip(neutron_bgp_dragent_pip_packages | customizable("pip_packages")) }}
+-
+-{% endif %}
+-
+-{% block neutron_bgp_dragent_footer %}{% endblock %}
+-{% block footer %}{% endblock %}
+-
+-USER neutron
+diff --git a/docker/opendaylight/Dockerfile.j2 b/docker/opendaylight/Dockerfile.j2
+index 8a005b3..29d2d2f 100644
+--- a/docker/opendaylight/Dockerfile.j2
++++ b/docker/opendaylight/Dockerfile.j2
+@@ -8,19 +8,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
+ {{ macros.configure_user(name='odl') }}
+ 
+ {% block opendaylight_install %}
+-
+-{% set opendaylight_packages = ['opendaylight'] %}
+-{% if base_distro in ['debian', 'ubuntu'] %}
+-# NOTE(egonzalez): ODL fails to install in debian family images with
+-# existing odl user. First install odl and then allow usage of
+-# install_packages macro for custom configs.
+-    {% block debian_opendaylight_install_fix_block %}
+-        {{ macros.debian_opendaylight_existing_user_fix() }}
+-    {% endblock %}
+-{% endif %}
+-
+-{{ macros.install_packages(opendaylight_packages | customizable("packages")) }}
+-
++RUN yum install -y https://nexus.opendaylight.org/content/repositories/opendaylight-carbon-epel-7-x86_64-devel/org/opendaylight/integration-packaging/opendaylight/6.4.0-0.1.20180330snap193.el7.noarch/opendaylight-6.4.0-0.1.20180330snap193.el7.noarch.rpm
+ {% endblock %}
+ 
+ COPY extend_start.sh /usr/local/bin/kolla_extend_start
+diff --git a/kolla/image/build.py b/kolla/image/build.py
+index 5ba551f..2a8a3d3 100755
+--- a/kolla/image/build.py
++++ b/kolla/image/build.py
+@@ -99,7 +99,6 @@ SKIPPED_IMAGES = {
+         "zun-base",
+     ],
+     'centos+source': [
+-        "ovsdpdk",
+         # TODO(jeffrey4l): remove tripleo-ui when following bug is fixed
+         # https://bugs.launchpad.net/tripleo/+bug/1744215
+         "tripleo-ui"