From 03ade964103d88af4d9a4f729d3bb00c12bff489 Mon Sep 17 00:00:00 2001
From: Leo Wang <grakiss.wanglei@huawei.com>
Date: Tue, 2 May 2017 16:50:02 -0700
Subject: local db support offline mode

JIRA: DOVETAIL-415

The local db also need to support offline, move all the git clone/apt/yum/pip/wget/curl into the Dockerfile

Change-Id: I2392dc9f4a6bd6f6a5a3f4849625a576c51a44f8
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>
---
 docker/Dockerfile                     |   5 ++
 docker/get_db_schema.py               |  61 +++++++++++++++++++
 dovetail/utils/dovetail_utils.py      |   6 +-
 dovetail/utils/local_db/init_db.py    |  80 +++++++++++++++++++++++++
 dovetail/utils/local_db/launch_db.sh  |  96 ++++++++++++++++++++++++++++++
 dovetail/utils/local_db/restart_db.sh |  29 +++++++++
 utils/init_db.py                      |  80 -------------------------
 utils/launch_db.sh                    | 107 ----------------------------------
 utils/restart_db.sh                   |  29 ---------
 9 files changed, 274 insertions(+), 219 deletions(-)
 create mode 100644 docker/get_db_schema.py
 create mode 100644 dovetail/utils/local_db/init_db.py
 create mode 100755 dovetail/utils/local_db/launch_db.sh
 create mode 100755 dovetail/utils/local_db/restart_db.sh
 delete mode 100644 utils/init_db.py
 delete mode 100755 utils/launch_db.sh
 delete mode 100755 utils/restart_db.sh

diff --git a/docker/Dockerfile b/docker/Dockerfile
index 7b3d99ed..499624f8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,3 +40,8 @@ RUN \
     pip install -e .
 
 WORKDIR ${REPOS_DIR}/dovetail
+
+# get db schema from opnfv sites
+RUN mkdir -p ${REPOS_DIR}/dovetail/utils/local_db
+ADD get_db_schema.py ${REPOS_DIR}/dovetail/utils/local_db
+RUN cd ${REPOS_DIR}/dovetail/utils/local_db && python get_db_schema.py
diff --git a/docker/get_db_schema.py b/docker/get_db_schema.py
new file mode 100644
index 00000000..9a9d10d4
--- /dev/null
+++ b/docker/get_db_schema.py
@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import requests
+import json
+
+
+source_url = 'http://testresults.opnfv.org/test/api/v1'
+
+
+def get(url):
+    try:
+        ret = requests.get(url)
+        return ret.json()
+    except:
+        return None
+
+
+def pod():
+    source = '{}/pods'.format(source_url)
+    try:
+        pods = get(source)['pods']
+        with open("pods.json", "w") as f:
+            f.write(json.dumps(pods, indent=4))
+    except:
+        return
+
+
+def project():
+    source = '{}/projects'.format(source_url)
+
+    try:
+        projects = get(source)['projects']
+        with open("projects.json", "w") as f:
+            f.write(json.dumps(projects, indent=4))
+    except:
+        return
+
+    for p in projects:
+        source = '{}/projects/{}/cases'.format(source_url, p['name'])
+        print(p['name'])
+        print(source)
+        try:
+            cases = get(source)
+            with open("cases.json", "a+") as f:
+                f.write(json.dumps(cases))
+                f.write('\n')
+                f.close()
+        except:
+            print("useless data")
+
+
+if __name__ == '__main__':
+    pod()
+    project()
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 7b613488..83390e9d 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -169,11 +169,11 @@ def show_progress_bar(length):
 
 
 def check_docker_version(logger=None):
-    ret, server_ver = exec_cmd("docker version -f'{{.Server.Version}}'",
+    ret, server_ver = exec_cmd("sudo docker version -f'{{.Server.Version}}'",
                                logger=logger)
-    ret, client_ver = exec_cmd("docker version -f'{{.Client.Version}}'",
+    ret, client_ver = exec_cmd("sudo docker version -f'{{.Client.Version}}'",
                                logger=logger)
-    logger.info("\ndocker version: \nclient:%s\nservr:%s", client_ver,
+    logger.info("\ndocker version: \nclient:%s\nserver:%s", client_ver,
                 server_ver)
     if(LooseVersion(client_ver) <= LooseVersion('1.8.0') or
        LooseVersion(server_ver) <= LooseVersion('1.8.0')):
diff --git a/dovetail/utils/local_db/init_db.py b/dovetail/utils/local_db/init_db.py
new file mode 100644
index 00000000..246139c4
--- /dev/null
+++ b/dovetail/utils/local_db/init_db.py
@@ -0,0 +1,80 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import requests
+import json
+import sys
+
+db_host_ip = sys.argv[1]
+testapi_port = sys.argv[2]
+
+target_url = 'http://{}:{}/api/v1'.format(db_host_ip, testapi_port)
+print(target_url)
+
+
+def get(url):
+    return requests.get(url).json()
+
+
+def post(url, data):
+    headers = {'Content-Type': 'application/json'}
+    res = requests.post(url, data=json.dumps(data), headers=headers)
+    print(res.text)
+
+
+def pod():
+    target = '{}/pods'.format(target_url)
+
+    with open('pods.json', 'r') as f:
+        pods = json.load(f)
+    for p in pods:
+        post(target, p)
+
+    add_pod('master', 'metal')
+    add_pod('virtual_136_2', 'virtual')
+
+
+def project():
+    target = '{}/projects'.format(target_url)
+    with open('projects.json', 'r') as f:
+        projects = json.load(f)
+    for p in projects:
+        post(target, p)
+
+
+def cases():
+    with open('cases.json', 'r') as f:
+        for line in f:
+            try:
+                cases = json.loads(line)
+                for c in cases["testcases"]:
+                    target = '{}/projects/{}/cases'.format(target_url,
+                                                           c['project_name'])
+                    print(target)
+                    post(target, c)
+            except:
+                print("useless data")
+
+
+def add_pod(name, mode):
+    data = {
+        "role": "",
+        "name": name,
+        "details": '',
+        "mode": mode,
+        "creation_date": "2017-2-23 11:23:03.765581"
+    }
+    pod_url = '{}/pods'.format(target_url)
+    post(pod_url, data)
+
+
+if __name__ == '__main__':
+    pod()
+    project()
+    cases()
diff --git a/dovetail/utils/local_db/launch_db.sh b/dovetail/utils/local_db/launch_db.sh
new file mode 100755
index 00000000..77646713
--- /dev/null
+++ b/dovetail/utils/local_db/launch_db.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+if [ "$#" -ne 1 ]; then
+    echo "Error: missing parameter! try again like this:"
+    echo ""
+    echo "./launch_db.sh 192.168.115.2"
+    echo ""
+    echo "parameters:"
+    echo "  db_host_ip: your localhost ip address "
+    echo ""
+    exit 1
+fi
+
+export mongodb_port=${mongodb_port:-"27017"}
+export testapi_port=${testapi_port:-"8000"}
+export db_host_ip=${db_host_ip:-"$1"}
+
+set -e
+
+echo "==================="
+echo "Create the mongodb."
+echo "==================="
+
+set +e
+# pull image kkltcjk/mongodb:reporting
+mongodb_img="kkltcjk/mongodb:reporting"
+echo "Step1: pull the image $mongodb_img."
+sudo docker pull $mongodb_img
+set -e
+
+container_name='mongodb'
+
+echo "Step2: remove the exist container with the same name '$container_name' if exists."
+sudo docker ps -a -f "name=${container_name}"
+
+if [[ ! -z $(sudo docker ps -aq -f "name=${container_name}") ]]; then
+    sudo docker ps -aq -f "name=${container_name}" | xargs sudo docker rm -f
+fi
+
+# run mongodb container
+echo "Step3: run ${container_name} container."
+cmd="sudo docker run -itd -p ${mongodb_port}:27017 --name ${container_name} ${mongodb_img}"
+echo $cmd
+${cmd}
+
+echo "Successfully create mongo DB."
+
+
+echo "=========================="
+echo "Create the testapi service."
+echo "=========================="
+
+set +e
+# pull image kkltcjk/testapi:reporting
+testapi_img="kkltcjk/testapi:reporting"
+echo "Step1: pull the image $testapi_img."
+sudo docker pull $testapi_img
+set -e
+
+container_name='testapi'
+
+echo "Step2: remove the exist container with the same name '$container_name' if exists."
+sudo docker ps -a -f "name=${container_name}"
+
+if [[ ! -z $(sudo docker ps -aq -f "name=${container_name}") ]]; then
+    sudo docker ps -aq -f "name=${container_name}" | xargs sudo docker rm -f
+fi
+
+# run testapi container
+echo "Step3: run ${container_name} container."
+cmd="sudo docker run -itd -p ${testapi_port}:8000 --name ${container_name} -e mongodb_url=mongodb://${db_host_ip}:${mongodb_port}/ ${testapi_img}"
+echo $cmd
+${cmd}
+
+echo "Wait for testapi to work..."
+sleep 10
+
+echo "================================="
+echo "Upload default project info to DB"
+echo "================================="
+
+echo "Init DB info..."
+cmd="python ./init_db.py ${db_host_ip} ${testapi_port}"
+echo ${cmd}
+${cmd}
+
+echo "Successfully load DB info."
+
diff --git a/dovetail/utils/local_db/restart_db.sh b/dovetail/utils/local_db/restart_db.sh
new file mode 100755
index 00000000..39b60e05
--- /dev/null
+++ b/dovetail/utils/local_db/restart_db.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+if [ "$#" -ne 1 ]; then
+    echo "Error: missing parameter! try again like this:"
+    echo ""
+    echo "./restart_db.sh 192.168.115.2"
+    echo ""
+    echo "parameters:"
+    echo "  db_host_ip: your localhost ip address"
+    echo ""
+    exit 1
+fi
+
+export mongodb_port=${mongodb_port:-"27017"}
+export testapi_port=${testapi_port:-"8000"}
+export db_host_ip=${db_host_ip:-"$1"}
+
+sudo docker rm -f testapi
+sudo docker run -itd -p $testapi_port:8000 --name testapi \
+    -e mongodb_url=mongodb://$db_host_ip:$mongodb_port/ kkltcjk/testapi:reporting
diff --git a/utils/init_db.py b/utils/init_db.py
deleted file mode 100644
index 129c61f8..00000000
--- a/utils/init_db.py
+++ /dev/null
@@ -1,80 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-import requests
-import json
-import sys
-
-db_host_ip = sys.argv[1]
-testapi_port = sys.argv[2]
-
-source_url = 'http://testresults.opnfv.org/test/api/v1'
-target_url = 'http://{}:{}/api/v1'.format(db_host_ip, testapi_port)
-print(target_url)
-
-
-def get(url):
-    return requests.get(url).json()
-
-
-def post(url, data):
-    headers = {'Content-Type': 'application/json'}
-    res = requests.post(url, data=json.dumps(data), headers=headers)
-    print(res.text)
-
-
-def pod():
-    source = '{}/pods'.format(source_url)
-    target = '{}/pods'.format(target_url)
-
-    pods = get(source)['pods']
-    for p in pods:
-        post(target, p)
-
-    add_pod('master', 'metal')
-    add_pod('virtual_136_2', 'virtual')
-
-
-def project():
-    source = '{}/projects'.format(source_url)
-    target = '{}/projects'.format(target_url)
-
-    projects = get(source)['projects']
-    for p in projects:
-        post(target, p)
-
-
-def cases():
-    project_list = ['yardstick', 'functest', 'dovetail']
-
-    for p in project_list:
-        source = '{}/projects/{}/cases'.format(source_url, p)
-        target = '{}/projects/{}/cases'.format(target_url, p)
-
-        cases = get(source)['testcases']
-        for c in cases:
-            post(target, c)
-
-
-def add_pod(name, mode):
-    data = {
-        "role": "",
-        "name": name,
-        "details": '',
-        "mode": mode,
-        "creation_date": "2017-2-23 11:23:03.765581"
-    }
-    pod_url = '{}/pods'.format(target_url)
-    post(pod_url, data)
-
-
-if __name__ == '__main__':
-    pod()
-    project()
-    cases()
diff --git a/utils/launch_db.sh b/utils/launch_db.sh
deleted file mode 100755
index f3681665..00000000
--- a/utils/launch_db.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/bin/bash
-##############################################################################
-# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-if [ "$#" -ne 1 ]; then
-    echo "Error: missing parameter! try again like this:"
-    echo ""
-    echo "./launch_db.sh 192.168.115.2"
-    echo ""
-    echo "parameters:"
-    echo "  db_host_ip: your localhost ip address "
-    echo ""
-    exit 1
-fi
-
-export mongodb_port=${mongodb_port:-"27017"}
-export testapi_port=${testapi_port:-"8000"}
-export db_host_ip=${db_host_ip:-"$1"}
-
-set -e
-
-echo "==================="
-echo "Create the mongodb."
-echo "==================="
-
-# pull image kkltcjk/mongodb:reporting
-mongodb_img="kkltcjk/mongodb:reporting"
-echo "Step1: pull the image $mongodb_img."
-sudo docker pull $mongodb_img > /dev/null
-
-container_name='mongodb'
-
-echo "Step2: remove the exist container with the same name '$container_name' if exists."
-sudo docker ps -a -f "name=${container_name}"
-
-if [[ ! -z $(sudo docker ps -aq -f "name=${container_name}") ]]; then
-    sudo docker ps -aq -f "name=${container_name}" | xargs sudo docker rm -f
-fi
-
-# run mongodb container
-echo "Step3: run ${container_name} container."
-cmd="sudo docker run -itd -p ${mongodb_port}:27017 --name ${container_name} ${mongodb_img}"
-echo $cmd
-${cmd}
-
-echo "Successfully create mongo DB."
-
-
-echo "=========================="
-echo "Create the testapi service."
-echo "=========================="
-
-# pull image kkltcjk/testapi:reporting
-testapi_img="kkltcjk/testapi:reporting"
-echo "Step1: pull the image $testapi_img."
-sudo docker pull $testapi_img > /dev/null
-
-container_name='testapi'
-
-echo "Step2: remove the exist container with the same name '$container_name' if exists."
-sudo docker ps -a -f "name=${container_name}"
-
-if [[ ! -z $(sudo docker ps -aq -f "name=${container_name}") ]]; then
-    sudo docker ps -aq -f "name=${container_name}" | xargs sudo docker rm -f
-fi
-
-# run testapi container
-echo "Step3: run ${container_name} container."
-cmd="sudo docker run -itd -p ${testapi_port}:8000 --name ${container_name} -e mongodb_url=mongodb://${db_host_ip}:${mongodb_port}/ ${testapi_img}"
-echo $cmd
-${cmd}
-
-echo "Successfully create the testapi service."
-
-echo "================================="
-echo "Upload default project info to DB"
-echo "================================="
-
-# For Ubuntu, there is file /etc/lsb-release
-# For Centos and redhat, there is file /etc/redhat-release
-if [ -f /etc/lsb-release ]; then
-    sudo apt-get update > /dev/null
-    sudo apt-get install -y python-pip > /dev/null
-elif [ -f /etc/redhat-release ]; then
-    sudo yum -y update > /dev/null
-    sudo yum -y install epel-release > /dev/null
-    sudo yum -y install python-pip > /dev/null
-else
-    echo "This operating system is not currently supported."
-    exit 1
-fi
-
-pip install requests > /dev/null
-
-echo "Init DB info..."
-cmd="python ./init_db.py ${db_host_ip} ${testapi_port}"
-echo ${cmd}
-${cmd} > /dev/null
-
-echo "Successfully load DB info."
-
diff --git a/utils/restart_db.sh b/utils/restart_db.sh
deleted file mode 100755
index 39b60e05..00000000
--- a/utils/restart_db.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-##############################################################################
-# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-if [ "$#" -ne 1 ]; then
-    echo "Error: missing parameter! try again like this:"
-    echo ""
-    echo "./restart_db.sh 192.168.115.2"
-    echo ""
-    echo "parameters:"
-    echo "  db_host_ip: your localhost ip address"
-    echo ""
-    exit 1
-fi
-
-export mongodb_port=${mongodb_port:-"27017"}
-export testapi_port=${testapi_port:-"8000"}
-export db_host_ip=${db_host_ip:-"$1"}
-
-sudo docker rm -f testapi
-sudo docker run -itd -p $testapi_port:8000 --name testapi \
-    -e mongodb_url=mongodb://$db_host_ip:$mongodb_port/ kkltcjk/testapi:reporting
-- 
cgit 1.2.3-korg