local db support offline mode

JIRA: DOVETAIL-415

The local db also need to support offline, move all the git clone/apt/yum/pip/wget/curl into the Dockerfile

Change-Id: I2392dc9f4a6bd6f6a5a3f4849625a576c51a44f8
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>

6 files changed, 93 insertions, 38 deletions

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7b3d99ed..499624f8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,3 +40,8 @@ RUN \
     pip install -e .
 
 WORKDIR ${REPOS_DIR}/dovetail
+
+# get db schema from opnfv sites
+RUN mkdir -p ${REPOS_DIR}/dovetail/utils/local_db
+ADD get_db_schema.py ${REPOS_DIR}/dovetail/utils/local_db
+RUN cd ${REPOS_DIR}/dovetail/utils/local_db && python get_db_schema.py
diff --git a/docker/get_db_schema.py b/docker/get_db_schema.py
new file mode 100644
index 00000000..9a9d10d4
--- /dev/null
+++ b/docker/get_db_schema.py
@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import requests
+import json
+
+
+source_url = 'http://testresults.opnfv.org/test/api/v1'
+
+
+def get(url):
+    try:
+        ret = requests.get(url)
+        return ret.json()
+    except:
+        return None
+
+
+def pod():
+    source = '{}/pods'.format(source_url)
+    try:
+        pods = get(source)['pods']
+        with open("pods.json", "w") as f:
+            f.write(json.dumps(pods, indent=4))
+    except:
+        return
+
+
+def project():
+    source = '{}/projects'.format(source_url)
+
+    try:
+        projects = get(source)['projects']
+        with open("projects.json", "w") as f:
+            f.write(json.dumps(projects, indent=4))
+    except:
+        return
+
+    for p in projects:
+        source = '{}/projects/{}/cases'.format(source_url, p['name'])
+        print(p['name'])
+        print(source)
+        try:
+            cases = get(source)
+            with open("cases.json", "a+") as f:
+                f.write(json.dumps(cases))
+                f.write('\n')
+                f.close()
+        except:
+            print("useless data")
+
+
+if __name__ == '__main__':
+    pod()
+    project()
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 7b613488..83390e9d 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -169,11 +169,11 @@ def show_progress_bar(length):
 
 
 def check_docker_version(logger=None):
-    ret, server_ver = exec_cmd("docker version -f'{{.Server.Version}}'",
+    ret, server_ver = exec_cmd("sudo docker version -f'{{.Server.Version}}'",
                                logger=logger)
-    ret, client_ver = exec_cmd("docker version -f'{{.Client.Version}}'",
+    ret, client_ver = exec_cmd("sudo docker version -f'{{.Client.Version}}'",
                                logger=logger)
-    logger.info("\ndocker version: \nclient:%s\nservr:%s", client_ver,
+    logger.info("\ndocker version: \nclient:%s\nserver:%s", client_ver,
                 server_ver)
     if(LooseVersion(client_ver) <= LooseVersion('1.8.0') or
        LooseVersion(server_ver) <= LooseVersion('1.8.0')):
diff --git a/utils/init_db.py b/dovetail/utils/local_db/init_db.py
index 129c61f8..246139c4 100644
--- a/utils/init_db.py
+++ b/dovetail/utils/local_db/init_db.py
@@ -14,7 +14,6 @@ import sys
 db_host_ip = sys.argv[1]
 testapi_port = sys.argv[2]
 
-source_url = 'http://testresults.opnfv.org/test/api/v1'
 target_url = 'http://{}:{}/api/v1'.format(db_host_ip, testapi_port)
 print(target_url)
 
@@ -30,10 +29,10 @@ def post(url, data):
 
 
 def pod():
-    source = '{}/pods'.format(source_url)
     target = '{}/pods'.format(target_url)
 
-    pods = get(source)['pods']
+    with open('pods.json', 'r') as f:
+        pods = json.load(f)
     for p in pods:
         post(target, p)
 
@@ -42,24 +41,25 @@ def pod():
 
 
 def project():
-    source = '{}/projects'.format(source_url)
     target = '{}/projects'.format(target_url)
-
-    projects = get(source)['projects']
+    with open('projects.json', 'r') as f:
+        projects = json.load(f)
     for p in projects:
         post(target, p)
 
 
 def cases():
-    project_list = ['yardstick', 'functest', 'dovetail']
-
-    for p in project_list:
-        source = '{}/projects/{}/cases'.format(source_url, p)
-        target = '{}/projects/{}/cases'.format(target_url, p)
-
-        cases = get(source)['testcases']
-        for c in cases:
-            post(target, c)
+    with open('cases.json', 'r') as f:
+        for line in f:
+            try:
+                cases = json.loads(line)
+                for c in cases["testcases"]:
+                    target = '{}/projects/{}/cases'.format(target_url,
+                                                           c['project_name'])
+                    print(target)
+                    post(target, c)
+            except:
+                print("useless data")
 
 
 def add_pod(name, mode):
diff --git a/utils/launch_db.sh b/dovetail/utils/local_db/launch_db.sh
index f3681665..77646713 100755
--- a/utils/launch_db.sh
+++ b/dovetail/utils/local_db/launch_db.sh
@@ -29,10 +29,12 @@ echo "==================="
 echo "Create the mongodb."
 echo "==================="
 
+set +e
 # pull image kkltcjk/mongodb:reporting
 mongodb_img="kkltcjk/mongodb:reporting"
 echo "Step1: pull the image $mongodb_img."
-sudo docker pull $mongodb_img > /dev/null
+sudo docker pull $mongodb_img
+set -e
 
 container_name='mongodb'
 
@@ -56,10 +58,12 @@ echo "=========================="
 echo "Create the testapi service."
 echo "=========================="
 
+set +e
 # pull image kkltcjk/testapi:reporting
 testapi_img="kkltcjk/testapi:reporting"
 echo "Step1: pull the image $testapi_img."
-sudo docker pull $testapi_img > /dev/null
+sudo docker pull $testapi_img
+set -e
 
 container_name='testapi'
 
@@ -76,32 +80,17 @@ cmd="sudo docker run -itd -p ${testapi_port}:8000 --name ${container_name} -e mo
 echo $cmd
 ${cmd}
 
-echo "Successfully create the testapi service."
+echo "Wait for testapi to work..."
+sleep 10
 
 echo "================================="
 echo "Upload default project info to DB"
 echo "================================="
 
-# For Ubuntu, there is file /etc/lsb-release
-# For Centos and redhat, there is file /etc/redhat-release
-if [ -f /etc/lsb-release ]; then
-    sudo apt-get update > /dev/null
-    sudo apt-get install -y python-pip > /dev/null
-elif [ -f /etc/redhat-release ]; then
-    sudo yum -y update > /dev/null
-    sudo yum -y install epel-release > /dev/null
-    sudo yum -y install python-pip > /dev/null
-else
-    echo "This operating system is not currently supported."
-    exit 1
-fi
-
-pip install requests > /dev/null
-
 echo "Init DB info..."
 cmd="python ./init_db.py ${db_host_ip} ${testapi_port}"
 echo ${cmd}
-${cmd} > /dev/null
+${cmd}
 
 echo "Successfully load DB info."
 
diff --git a/utils/restart_db.sh b/dovetail/utils/local_db/restart_db.sh
index 39b60e05..39b60e05 100755
--- a/utils/restart_db.sh
+++ b/dovetail/utils/local_db/restart_db.sh