From cf202278ba12b249bc643af2b468606c087b1108 Mon Sep 17 00:00:00 2001 From: Harry Huang Date: Fri, 7 Sep 2018 09:31:11 +0800 Subject: Support multiple url when building compass tar ball JIRA: COMPASS-617 1. Use aria2 instead of curl to download package from different sites 2. Add an internal file server to accelerate building process for CI Change-Id: I2dca7a9a3fccf51165d0239126571fa16b94ebee Signed-off-by: Harry Huang --- build.sh | 2 +- build/build.yaml | 12 +++++++++--- build/parser.py | 35 ++++++++++++++++++++--------------- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/build.sh b/build.sh index a6276621..d7118432 100755 --- a/build.sh +++ b/build.sh @@ -22,7 +22,7 @@ echo $COMPASS_PATH REDHAT_REL=${REDHAT_REL:-"false"} -PACKAGES="curl python-pip" +PACKAGES="curl python-pip aria2" mkdir -p $WORK_DIR $CACHE_DIR diff --git a/build/build.yaml b/build/build.yaml index 50602dbe..ef42ba9c 100644 --- a/build/build.yaml +++ b/build/build.yaml @@ -3,17 +3,23 @@ packages: - name: CentOS-7-x86_64-Minimal-1708.iso description: "Centos ISO of each host for provisioning" get_method: cached - url: http://artifacts.opnfv.org/compass4nfv/package/master/CentOS-7-x86_64-Minimal-1708.iso + url: + - http://192.168.137.222/download/CentOS-7-x86_64-Minimal-1708.iso + - http://artifacts.opnfv.org/compass4nfv/package/master/CentOS-7-x86_64-Minimal-1708.iso - name: ubuntu-16.04.3-server-amd64.iso description: "Ubuntu ISO of each host for provisioning" get_method: cached - url: http://artifacts.opnfv.org/compass4nfv/package/master/ubuntu-16.04.3-server-amd64.iso + url: + - http://192.168.137.222/download/ubuntu-16.04.3-server-amd64.iso + - http://artifacts.opnfv.org/compass4nfv/package/master/ubuntu-16.04.3-server-amd64.iso - name: harbor-offline-installer-v1.5.0.tgz description: "The package of harbor v1.5.5" get_method: cached - url: http://artifacts.opnfv.org/compass4nfv/package/master/harbor-offline-installer-v1.5.0.tgz + url: + - http://192.168.137.222/download/harbor-offline-installer-v1.5.0.tgz + - http://artifacts.opnfv.org/compass4nfv/package/master/harbor-offline-installer-v1.5.0.tgz - name: compass-deck description: "RESTful API and DB Handlers for Compass" diff --git a/build/parser.py b/build/parser.py index b80709cb..b0a87f93 100644 --- a/build/parser.py +++ b/build/parser.py @@ -20,28 +20,33 @@ def load_env(): def get_from_cache(cache, package): filename = package.get("name") - remotefile = package.get("url") + remotefile = list(package.get("url")) localfile = cache + "/" + filename localmd5file = localfile + ".md5" - remotemd5file = remotefile + ".md5" print "removing local md5 file...." cmd = "rm -f " + localmd5file os.system(cmd) print "downloading remote md5 file to local...." - cmd = "curl --connect-timeout 10 -o " + localmd5file + " " + remotemd5file - os.system(cmd) - if os.path.exists(localmd5file): - print "calculate md5sum of local file" - cmd = "md5sum " + localfile + "|cut -d ' ' -f 1" - localmd5sum = os.popen(cmd).readlines() - cmd = "cat " + localmd5file + "|cut -d ' ' -f 1" - remotemd5sum = os.popen(cmd).readlines() - print "md5 local %s remote %s" % (localmd5sum, remotemd5sum) - if (remotemd5sum == localmd5sum): - print "Same with remote, no need to download...." - return + for file in remotefile: + remotemd5file = file + ".md5" + cmd = "curl --connect-timeout 10 -o {0} {1}".format( + localmd5file, remotemd5file) + rc = os.system(cmd) + if os.path.exists(localfile): + print "calculate md5sum of local file" + cmd = "md5sum " + localfile + "|cut -d ' ' -f 1" + localmd5sum = os.popen(cmd).readlines() + cmd = "cat " + localmd5file + "|cut -d ' ' -f 1" + remotemd5sum = os.popen(cmd).readlines() + print "md5 local %s remote %s" % (localmd5sum, remotemd5sum) + if (remotemd5sum == localmd5sum): + print "Same with remote, no need to download...." + return + if rc == 0: + break print "downloading remote file to local...." - cmd = "curl --connect-timeout 10 -o " + localfile + " " + remotefile + cmd = "aria2c --max-connection-per-server=4 --allow-overwrite=true --dir={0} \ + --out={1} {2}".format(cache, filename, " ".join(remotefile)) print cmd rc = os.system(cmd) if rc != 0: -- cgit 1.2.3-korg