replace build.conf with yaml

JIRA: -

use build.yaml as config file
from Euphrates openstack deb/rpm/pip packages download from compass
repo containers, not from compass.iso. so delete these packages from
build.cfg

Change-Id: Id254852ece0d738b3ec543efe68216206bd7daf2
Signed-off-by: chigang <chigang@huawei.com>

4 files changed, 148 insertions, 126 deletions

diff --git a/build.sh b/build.sh
index c4f73aee..3665f6e2 100755
--- a/build.sh
+++ b/build.sh
@@ -8,28 +8,19 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 set -ex
-#COMPASS_PATH=$(cd "$(dirname "$0")"/..; pwd)
-BUILD_IMAGES=${BUILD_IMAGES:-"false"}
 
 COMPASS_PATH=`cd ${BASH_SOURCE[0]%/*};pwd`
 WORK_DIR=$COMPASS_PATH/work/building
-CACHE_DIR=$WORK_DIR/cache
+export CACHE_DIR=$WORK_DIR/cache
 
 echo $COMPASS_PATH
 
-# REPO related setting
-REPO_PATH=$COMPASS_PATH/repo
-WORK_PATH=$COMPASS_PATH
-
 REDHAT_REL=${REDHAT_REL:-"false"}
 
 PACKAGES="curl"
 
 mkdir -p $WORK_DIR $CACHE_DIR
 
-source $COMPASS_PATH/build/build.conf
-#cd $WORK_DIR
-
 function install_docker_ubuntu()
 {
     sudo apt-get install -y linux-image-extra-$(uname -r) linux-image-extra-virtual
@@ -83,77 +74,9 @@ function prepare_env()
     set -e
 }
 
-function download_git()
-{
-    file_dir=$CACHE_DIR/${1%.*}
-    if [[ -d $file_dir/.git ]]; then
-        cd $file_dir
-        source=`git remote -v | head -n 1  | awk '{print $2}'`
-        if [[ $2 == $source ]]; then
-            git pull origin master
-            if [[ $? -eq 0 ]]; then
-                cd -
-                return
-            fi
-        fi
-        cd -
-    fi
-    rm -rf $CACHE_DIR/${1%.*}
-    git clone $2 $file_dir
-}
-
-function download_url()
-{
-    rm -f $CACHE_DIR/$1.md5
-    curl --connect-timeout 10 -o $CACHE_DIR/$1.md5 $2.md5 2>/dev/null || true
-    if [[ -f $CACHE_DIR/$1 ]]; then
-        local_md5=`md5sum $CACHE_DIR/$1 | cut -d ' ' -f 1`
-        repo_md5=`cat $CACHE_DIR/$1.md5 | cut -d ' ' -f 1`
-        if [[ $local_md5 == $repo_md5 ]]; then
-            return
-        fi
-    fi
-
-    curl --connect-timeout 10 -o $CACHE_DIR/$1 $2
-}
-
-function download_local()
-{
-    if [[ $2 != $CACHE_DIR/$1 ]]; then
-       cp $2 $CACHE_DIR/ -rf
-    fi
-}
-
-function download_docker_images()
-{
-    for i in $COMPASS_DECK $COMPASS_TASKS $COMPASS_COBBLER \
-             $COMPASS_DB $COMPASS_MQ; do
-        basename=`basename $i`
-        sudo docker pull $i
-        sudo docker save $i -o $CACHE_DIR/${basename%:*}.tar
-    done
-}
-
 function download_packages()
 {
-    for i in $PIP_OPENSTACK_REPO $APP_PACKAGE $COMPASS_COMPOSE \
-             $UBUNTU_ISO $CENTOS_ISO $UBUNTU_PPA $CENTOS_PPA; do
-
-         if [[ ! $i ]]; then
-             continue
-         fi
-         name=`basename $i`
-
-         if [[ ${name##*.} == git ]]; then
-             download_git  $name $i
-         elif [[ "https?" =~ ${i%%:*} || "file://" =~ ${i%%:*} ]]; then
-             download_url  $name $i
-         else
-             download_local $name $i
-         fi
-     done
-
-    download_docker_images
+    python $COMPASS_PATH/build/parser.py $COMPASS_PATH/build/build.yaml
 }
 
 function build_tar()
@@ -161,11 +84,7 @@ function build_tar()
     cd $CACHE_DIR
     sudo rm -rf compass_dists
     mkdir -p compass_dists
-    sudo cp -f `basename $PIP_OPENSTACK_REPO` `basename $APP_PACKAGE` \
-    `basename $UBUNTU_ISO` `basename $CENTOS_ISO` \
-    `basename $UBUNTU_PPA` `basename $CENTOS_PPA` \
-    compass-deck.tar compass-tasks-osa.tar compass-cobbler.tar \
-    compass-db.tar compass-mq.tar compass_dists
+    sudo cp -f *.tar *.iso compass_dists
     sudo tar -zcf compass.tar.gz compass-docker-compose compass_dists
     sudo mv compass.tar.gz $TAR_DIR/$TAR_NAME
     cd -
diff --git a/build/build.conf b/build/build.conf
deleted file mode 100644
index fea50c80..00000000
--- a/build/build.conf
+++ /dev/null
@@ -1,42 +0,0 @@
-TIMEOUT=10
-
-# PACKAGE_URL will be reset in Jenkins for different branch
-export PACKAGE_URL=${PACKAGE_URL:-http://artifacts.opnfv.org/compass4nfv/package/master}
-
-# Jumphost OS version
-# export CENTOS_BASE=${CENTOS_BASE:-$PACKAGE_URL/CentOS-7-x86_64-Minimal-1511.iso}
-
-# Compass git repository
-# export COMPASS_CORE=${COMPASS_CORE:-https://github.com/openstack/compass-core.git}
-# export COMPASS_WEB=${COMPASS_WEB:-https://github.com/openstack/compass-web.git}
-# export COMPASS_INSTALL=${COMPASS_INSTALL:-http://github.com/baigk/compass-install.git}
-
-# Compass core packages
-# export COMPASS_PKG=${COMPASS_PKG:-$PACKAGE_URL/centos7-compass-core.tar.gz}
-# export PIP_REPO=${PIP_REPO:-$PACKAGE_URL/pip.tar.gz}
-# export PIP_OPENSTACK_REPO=${PIP_OPENSTACK_REPO:-$PACKAGE_URL/pip-openstack.tar.gz}
-
-# OS ISO for provisioning
-export CENTOS_ISO=${CENTOS_ISO:-$PACKAGE_URL/CentOS-7-x86_64-Minimal-1611.iso} # centos 7.3
-export UBUNTU_ISO=${UBUNTU_ISO:-$PACKAGE_URL/ubuntu-16.04-server-amd64.iso} # ubuntu 16.04
-
-# OpenStack Packages for deployment
-# export UBUNTU_PPA=${UBUNTU_PPA:-$PACKAGE_URL/xenial-ocata-ppa.tar.gz}
-# export CENTOS_PPA=${CENTOS_PPA:-$PACKAGE_URL/centos7-ocata-ppa.tar.gz}
-
-# SDN Packages for integration
-# export APP_PACKAGE=${APP_PACKAGE:-$PACKAGE_URL/packages.tar.gz}
-
-# Other Packages
-# export LOADERS=${LOADERS:-$PACKAGE_URL/loaders.tar.gz}
-# export CIRROS=${CIRROS:-$PACKAGE_URL/cirros-0.3.3-x86_64-disk.img}
-
-# Containerized compass-core docker images
-export COMPASS_DECK=${COMPASS_DECK:-huangxiangyu/compass-deck:v0.2}
-export COMPASS_TASKS=${COMPASS_TASKS:-wtwde/compass-tasks-osa:v0.2}
-export COMPASS_COBBLER=${COMPASS_COBBLER:-huangxiangyu/compass-cobbler:v0.1}
-export COMPASS_DB=${COMPASS_DB:-huangxiangyu/compass-db:v0.1}
-export COMPASS_MQ=${COMPASS_MQ:-huangxiangyu/compass-mq:v0.1}
-
-# Containerized compass-core ansible
-export COMPASS_COMPOSE=${COMPASS_COMPOSE:-https://github.com/hexhxy/compass-docker-compose.git}
diff --git a/build/build.yaml b/build/build.yaml
new file mode 100644
index 00000000..3f684613
--- /dev/null
+++ b/build/build.yaml
@@ -0,0 +1,41 @@
+---
+packages:
+  - name: CentOS-7-x86_64-Minimal-1611.iso
+    description: "Centos ISO of each host for provisioning"
+    get_method: cached
+    url: http://artifacts.opnfv.org/compass4nfv/package/master/CentOS-7-x86_64-Minimal-1611.iso
+
+  - name: ubuntu-16.04-server-amd64.iso
+    description: "Ubuntu ISO of each host for provisioning"
+    get_method: cached
+    url: http://artifacts.opnfv.org/compass4nfv/package/master/ubuntu-16.04-server-amd64.iso
+
+  - name: compass-deck
+    description: "RESTful API and DB Handlers for Compass"
+    get_method: docker
+    url: huangxiangyu/compass-deck:v0.2
+
+  - name: compass-tasks-osa
+    description: "compass task container for openstack deployment via openstack-ansible"
+    get_method: docker
+    url: wtwde/compass-tasks-osa:v0.2
+
+  - name: compass-cobbler
+    description: "cobbler container for compass"
+    get_method: docker
+    url: huangxiangyu/compass-cobbler:v0.1
+
+  - name: compass-db
+    description: "datebase container for compass"
+    get_method: docker
+    url: huangxiangyu/compass-db:v0.1
+
+  - name: compass-mq
+    description: "message queue container for compass"
+    get_method: docker
+    url: huangxiangyu/compass-mq:v0.1
+
+  - name: compass-docker-compose
+    description: "containerized compass compose ansible"
+    get_method: git
+    url: https://github.com/hexhxy/compass-docker-compose.git
diff --git a/build/parser.py b/build/parser.py
new file mode 100644
index 00000000..602d7c21
--- /dev/null
+++ b/build/parser.py
@@ -0,0 +1,104 @@
+##############################################################################
+# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import os
+import sys
+
+
+def load_env():
+    cache_dir = os.environ.get('CACHE_DIR')
+    print "Read local Cache dir is %s" % cache_dir
+    return cache_dir
+
+
+def get_from_cache(cache, package):
+    filename = package.get("name")
+    remotefile = package.get("url")
+    localfile = cache + "/" + filename
+    localmd5file = localfile + ".md5"
+    remotemd5file = remotefile + ".md5"
+    print "removing local md5 file...."
+    cmd = "rm -f " + localmd5file
+    os.system(cmd)
+    print "downloading remote md5 file to local...."
+    cmd = "curl --connect-timeout 10 -o " + localmd5file + " " + remotemd5file
+    os.system(cmd)
+    if os.path.exists(localmd5file):
+        print "calculate md5sum of local file"
+        cmd = "md5sum " + localfile + "|cut -d ' ' -f 1"
+        localmd5sum = os.popen(cmd).readlines()
+        cmd = "cat " + localmd5file + "|cut -d ' ' -f 1"
+        remotemd5sum = os.popen(cmd).readlines()
+        print "md5 local %s remote %s" % (localmd5sum, remotemd5sum)
+        if (remotemd5sum == localmd5sum):
+            print "Same with remote, no need to download...."
+            return
+    print "downloading remote file to local...."
+    cmd = "curl --connect-timeout 10 -o " + localfile + " " + remotefile
+    print cmd
+    os.system(cmd)
+
+
+def get_from_git(cache, package):
+    localfile = cache + "/" + package.get("name")
+    cmd = "rm -rf " + localfile
+    print cmd
+    os.system(cmd)
+    cmd = "git clone " + package.get("url") + " " + localfile
+    print cmd
+    os.system(cmd)
+
+
+def get_from_docker(cache, package):
+    cmd = "sudo docker pull "+package.get("url")
+    os.system(cmd)
+    cmd = "sudo docker save "+package.get("url")+" -o "+cache+"/"
+    cmd += package.get("name")+".tar"
+    os.system(cmd)
+
+
+def get_from_curl(cache, package):
+    cmd = "curl --connect-timeout 10 -o " + cache + "/"
+    cmd += package.get("name") + " " + package.get("url")
+    print cmd
+    os.system(cmd)
+
+
+def usage():
+    print "cached : Download from a cached server"
+    print "git    : Download from git url"
+    print "curl   : Download from a url link by curl"
+    print "docker : Download from docker hub"
+
+
+def build_parser(build_file_name):
+    cache = load_env()
+    cfg = yaml.load(file(build_file_name, 'r'))
+
+    print "Starting building...."
+    for pkg in cfg.get("packages"):
+        print "processing %s" % pkg.get("description")
+
+        if pkg.get("get_method") == "cached":
+            get_from_cache(cache, pkg)
+        elif pkg.get("get_method") == "git":
+            get_from_git(cache, pkg)
+        elif pkg.get("get_method") == "docker":
+            get_from_docker(cache, pkg)
+        elif pkg.get("get_method") == "curl":
+            get_from_curl(cache, pkg)
+        else:
+            usage
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("parameter wrong%d %s" % (len(sys.argv), sys.argv))
+        sys.exit(1)
+    build_parser(sys.argv[1])