Support multiple url when building compass tar ball

JIRA: COMPASS-617

1. Use aria2 instead of curl to download package from
different sites
2. Add an internal file server to accelerate building
process for CI

Change-Id: I2dca7a9a3fccf51165d0239126571fa16b94ebee
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

3 files changed, 30 insertions, 19 deletions

diff --git a/build.sh b/build.sh
index a6276621..d7118432 100755
--- a/build.sh
+++ b/build.sh
@@ -22,7 +22,7 @@ echo $COMPASS_PATH
 
 REDHAT_REL=${REDHAT_REL:-"false"}
 
-PACKAGES="curl python-pip"
+PACKAGES="curl python-pip aria2"
 
 mkdir -p $WORK_DIR $CACHE_DIR
 
diff --git a/build/build.yaml b/build/build.yaml
index 50602dbe..ef42ba9c 100644
--- a/build/build.yaml
+++ b/build/build.yaml
@@ -3,17 +3,23 @@ packages:
   - name: CentOS-7-x86_64-Minimal-1708.iso
     description: "Centos ISO of each host for provisioning"
     get_method: cached
-    url: http://artifacts.opnfv.org/compass4nfv/package/master/CentOS-7-x86_64-Minimal-1708.iso
+    url:
+      - http://192.168.137.222/download/CentOS-7-x86_64-Minimal-1708.iso
+      - http://artifacts.opnfv.org/compass4nfv/package/master/CentOS-7-x86_64-Minimal-1708.iso
 
   - name: ubuntu-16.04.3-server-amd64.iso
     description: "Ubuntu ISO of each host for provisioning"
     get_method: cached
-    url: http://artifacts.opnfv.org/compass4nfv/package/master/ubuntu-16.04.3-server-amd64.iso
+    url:
+      - http://192.168.137.222/download/ubuntu-16.04.3-server-amd64.iso
+      - http://artifacts.opnfv.org/compass4nfv/package/master/ubuntu-16.04.3-server-amd64.iso
 
   - name: harbor-offline-installer-v1.5.0.tgz
     description: "The package of harbor v1.5.5"
     get_method: cached
-    url: http://artifacts.opnfv.org/compass4nfv/package/master/harbor-offline-installer-v1.5.0.tgz
+    url:
+      - http://192.168.137.222/download/harbor-offline-installer-v1.5.0.tgz
+      - http://artifacts.opnfv.org/compass4nfv/package/master/harbor-offline-installer-v1.5.0.tgz
 
   - name: compass-deck
     description: "RESTful API and DB Handlers for Compass"
diff --git a/build/parser.py b/build/parser.py
index b80709cb..b0a87f93 100644
--- a/build/parser.py
+++ b/build/parser.py
@@ -20,28 +20,33 @@ def load_env():
 
 def get_from_cache(cache, package):
     filename = package.get("name")
-    remotefile = package.get("url")
+    remotefile = list(package.get("url"))
     localfile = cache + "/" + filename
     localmd5file = localfile + ".md5"
-    remotemd5file = remotefile + ".md5"
     print "removing local md5 file...."
     cmd = "rm -f " + localmd5file
     os.system(cmd)
     print "downloading remote md5 file to local...."
-    cmd = "curl --connect-timeout 10 -o " + localmd5file + " " + remotemd5file
-    os.system(cmd)
-    if os.path.exists(localmd5file):
-        print "calculate md5sum of local file"
-        cmd = "md5sum " + localfile + "|cut -d ' ' -f 1"
-        localmd5sum = os.popen(cmd).readlines()
-        cmd = "cat " + localmd5file + "|cut -d ' ' -f 1"
-        remotemd5sum = os.popen(cmd).readlines()
-        print "md5 local %s remote %s" % (localmd5sum, remotemd5sum)
-        if (remotemd5sum == localmd5sum):
-            print "Same with remote, no need to download...."
-            return
+    for file in remotefile:
+        remotemd5file = file + ".md5"
+        cmd = "curl --connect-timeout 10 -o {0} {1}".format(
+            localmd5file, remotemd5file)
+        rc = os.system(cmd)
+        if os.path.exists(localfile):
+            print "calculate md5sum of local file"
+            cmd = "md5sum " + localfile + "|cut -d ' ' -f 1"
+            localmd5sum = os.popen(cmd).readlines()
+            cmd = "cat " + localmd5file + "|cut -d ' ' -f 1"
+            remotemd5sum = os.popen(cmd).readlines()
+            print "md5 local %s remote %s" % (localmd5sum, remotemd5sum)
+            if (remotemd5sum == localmd5sum):
+                print "Same with remote, no need to download...."
+                return
+        if rc == 0:
+            break
     print "downloading remote file to local...."
-    cmd = "curl --connect-timeout 10 -o " + localfile + " " + remotefile
+    cmd = "aria2c --max-connection-per-server=4 --allow-overwrite=true --dir={0} \
+          --out={1} {2}".format(cache, filename, " ".join(remotefile))
     print cmd
     rc = os.system(cmd)
     if rc != 0: