Move all web portal code to the new repo dovetail-webportal

This is only the first step to simply copy the file here.
There still need some more work to make sure all work well.
All the changes will be submitted with other patches to make it easily to review.

JIRA: DOVETAIL-671

Change-Id: I64d32a9df562184166b6199e2719f298687d1a0a
Signed-off-by: xudan <xudan16@huawei.com>

14 files changed, 610 insertions, 0 deletions

diff --git a/update/README.md b/update/README.md
new file mode 100644
index 0000000..cb0e67b
--- /dev/null
+++ b/update/README.md
@@ -0,0 +1,100 @@
+Welcome to TESTAPI Update!
+========================
+
+
+This file is used to describe how testapi update works
+
+----------
+How to use
+---------------
+
+#### <i class="icon-file"></i> backup mongodb
+
+arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+the backup output will be put under dir/db__XXXX_XX_XX_XXXXXX/db
+-d/--db: database for the backup, default = test_results_collection
+
+usage:
+```
+python backup_mongodb.py
+```
+
+#### <i class="icon-file"></i> restore mongodb
+
+arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+  -i/--input_dir: Input directory for the Restore, must be specified,
+  the restore input must be specified to dir/db__XXXX_XX_XX_XXXXXX/db
+  -d/--db: database name after the restore, default = basename of input_dir
+
+usage:
+```
+python restore_mongodb.py
+```
+#### <i class="icon-file"></i> update mongodb
+
+ arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+ -d/--db: database name to be updated, default = test_results_collection
+
+changes need to be done:
+change collection name, modify changes.collections_old2New
+ > collections_old2New = {
+     'old_collection': 'new_collection',
+ }
+
+ change field name, modify changes.fields_old2New
+ > fields_old2New = {
+     'collection': [(query, {'old_field': 'new_field'})]
+ }
+
+ change the doc, modify changes.docs_old2New
+ > docs_old2New = {
+     'test_results': [
+         ({'field': 'old_value'}, {'field': 'new_value'}),
+         (query, {'field': 'new_value'}),
+     ]
+ }
+
+#### <i class="icon-file"></i> update opnfv-testapi process
+This script must be run right in this directory and remember to
+change ../etc/config.ini before running this script.
+
+operations includes:
+: kill running test_collection_api & opnfv-testapi
+install or update dependencies according to ../requirements.txt
+install opnfv-testapi
+run opnfv-testapi
+
+usage:
+```
+python update_api.py
+```
+#### <i class="icon-file"></i> update opnfv/testapi container
+Here ansible-playbook is used to implement auto update.
+Please make sure that the remote server is accessible via ssh.
+
+install ansible, please refer:
+```
+http://docs.ansible.com/ansible/intro_installation.html
+```
+
+playbook-update.sh
+
+arguments:
+: -h|--help           show this help text
+-r|--remote         remote server
+-u|--user           ssh username used to access to remote server
+-i|--identity       ssh PublicKey file used to access to remote server
+-e|--execute        execute update, if not set just check the ansible connectivity
+
+usage:
+```
+ssh-agent ./playbook-update.sh -r testresults.opnfv.org -u serena -i ~/.ssh/id_rsa -e
+```
+
+> **Note:**
+
+> - If documents need to be changed, please modify file
+templates/changes_in_mongodb.py, and refer section **update mongodb**
diff --git a/update/__init__.py b/update/__init__.py
new file mode 100644
index 0000000..363bc38
--- /dev/null
+++ b/update/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/update/playbook-update.sh b/update/playbook-update.sh
new file mode 100755
index 0000000..86d30e4
--- /dev/null
+++ b/update/playbook-update.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+#
+# Author: Serena Feng (feng.xiaoewi@zte.com.cn)
+# Update testapi on remote server using ansible playbook automatically
+#
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+set -e
+
+usage="Script to trigger update automatically.
+
+usage:
+    bash $(basename "$0") [-h|--help] [-h <host>] [-u username] [-i identityfile] [-e|--execute]
+
+where:
+    -h|--help           show this help text
+    -r|--remote         remote server
+    -u|--user           ssh username used to access to remote server
+    -i|--identity       ssh PublicKey file used to access to remote server
+    -e|--execute        execute update, if not set just check the ansible connectivity"
+
+remote=testresults.opnfv.org
+user=root
+identity=~/.ssh/id_rsa
+hosts=./hosts
+execute=false
+
+# Parse parameters
+while [[ $# > 0 ]]
+    do
+    key="$1"
+    case $key in
+        -h|--help)
+            echo "$usage"
+            exit 0
+            shift
+        ;;
+        -r|--remote)
+            remote="$2"
+            shift
+        ;;
+        -u|--user)
+            user="$2"
+            shift
+        ;;
+        -i|--identity)
+            identity="$2"
+            shift
+        ;;
+        -e|--execute)
+            execute=true
+        ;;
+        *)
+            echo "unknown option"
+            exit 1
+        ;;
+    esac
+    shift # past argument or value
+done
+
+echo $remote > $hosts
+
+echo "add authentication"
+ssh-add $identity
+
+echo "test ansible connectivity"
+ansible -i ./hosts $remote -m ping -u $user
+
+echo "test playbook connectivity"
+ansible-playbook -i $hosts test.yml -e "host=$remote user=$user"
+
+if [ $execute == true ]; then
+    echo "do update"
+    ansible-playbook -i $hosts update.yml -e "host=$remote \
+    user=$user \
+    port=8082 \
+    image=opnfv/testapi \
+    update_path=/home/$user/testapi \
+    mongodb_url=mongodb://172.17.0.1:27017 \
+    swagger_url=http://testresults.opnfv.org/test"
+fi
+
+rm -fr $hosts
+ssh-agent -k
diff --git a/update/templates/__init__.py b/update/templates/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/update/templates/__init__.py
diff --git a/update/templates/backup_mongodb.py b/update/templates/backup_mongodb.py
new file mode 100644
index 0000000..9c24377
--- /dev/null
+++ b/update/templates/backup_mongodb.py
@@ -0,0 +1,45 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+import datetime
+import os
+
+from utils import execute, main, get_abspath
+
+parser = argparse.ArgumentParser(description='Backup MongoDBs')
+
+parser.add_argument('-u', '--url',
+                    type=str,
+                    required=False,
+                    default='mongodb://127.0.0.1:27017/',
+                    help='Mongo DB URL for Backups')
+parser.add_argument('-o', '--output_dir',
+                    type=str,
+                    required=False,
+                    default='./',
+                    help='Output directory for the backup.')
+
+parser.add_argument('-d', '--db',
+                    type=str,
+                    required=False,
+                    default='test_results_collection',
+                    help='database for the backup.')
+
+
+def backup(args):
+    db = args.db
+    out = get_abspath(args.output_dir)
+    now = datetime.datetime.now()
+    out = os.path.join(out, '%s__%s' % (db, now.strftime('%Y_%m_%d_%H%M%S')))
+    cmd = ['mongodump', '-o', '%s' % out]
+    execute(cmd, args)
+
+
+if __name__ == '__main__':
+    main(backup, parser)
diff --git a/update/templates/changes_in_mongodb.py b/update/templates/changes_in_mongodb.py
new file mode 100644
index 0000000..1a4d5a1
--- /dev/null
+++ b/update/templates/changes_in_mongodb.py
@@ -0,0 +1,51 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# 09/06/2016: change for migration after refactoring
+# 16/06/2016: Alignment of test name (JIRA: FUNCTEST-304)
+##############################################################################
+collections_old2New = {
+    # 'pod': 'pods',
+    # 'test_projects': 'projects',
+    # 'test_testcases': 'testcases',
+    # 'test_results': 'results'
+}
+
+fields_old2New = {
+    # 'test_results': [({}, {'creation_date': 'start_date'})]
+}
+
+docs_old2New = {
+    # 'test_results': [
+    #    ({'criteria': 'failed'}, {'criteria': 'FAILED'}),
+    #    ({'criteria': 'passed'}, {'criteria': 'PASS'})
+    # ]
+    # 'testcases': [
+    #     ({'name': 'vPing'}, {'name': 'vping_ssh'}),
+    #     ({'name': 'Tempest'}, {'name': 'tempest_smoke_serial'}),
+    #     ({'name': 'Rally'}, {'name': 'rally_sanity'}),
+    #     ({'name': 'ODL'}, {'name': 'odl'}),
+    #     ({'name': 'vIMS'}, {'name': 'vims'}),
+    #     ({'name': 'ONOS'}, {'name': 'onos'}),
+    #     ({'name': 'vPing_userdata'}, {'name': 'vping_userdata'}),
+    #     ({'name': 'ovno'}, {'name': 'ocl'})
+    # ],
+    # 'results': [
+    #     ({'case_name': 'vPing'}, {'case_name': 'vping_ssh'}),
+    #     ({'case_name': 'Tempest'}, {'case_name': 'tempest_smoke_serial'}),
+    #     ({'case_name': 'Rally'}, {'case_name': 'rally_sanity'}),
+    #     ({'case_name': 'ODL'}, {'case_name': 'odl'}),
+    #     ({'case_name': 'vIMS'}, {'case_name': 'vims'}),
+    #     ({'case_name': 'ONOS'}, {'case_name': 'onos'}),
+    #     ({'case_name': 'vPing_userdata'}, {'case_name': 'vping_userdata'}),
+    #     ({'case_name': 'ovno'}, {'case_name': 'ocl'})
+    # ]
+    'results': [
+        ({'trust_indicator': 0},
+         {'trust_indicator': {'current': 0, 'histories': []}})
+    ]
+}
diff --git a/update/templates/restore_mongodb.py b/update/templates/restore_mongodb.py
new file mode 100644
index 0000000..c45a0e6
--- /dev/null
+++ b/update/templates/restore_mongodb.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+
+from utils import execute, main, get_abspath
+
+parser = argparse.ArgumentParser(description='Restore MongoDBs')
+
+parser.add_argument('-u', '--url',
+                    type=str,
+                    required=False,
+                    default='mongodb://127.0.0.1:27017/',
+                    help='Mongo DB URL for Backup')
+parser.add_argument('-i', '--input_dir',
+                    type=str,
+                    required=True,
+                    help='Input directory for the Restore.')
+parser.add_argument('-d', '--db',
+                    type=str,
+                    required=False,
+                    default='test_results_collection',
+                    help='database name after the restore.')
+
+
+def restore(args):
+    input_dir = get_abspath(args.input_dir)
+    cmd = ['mongorestore', '%s' % input_dir]
+    execute(cmd, args)
+
+
+if __name__ == '__main__':
+    main(restore, parser)
diff --git a/update/templates/rm_images.sh b/update/templates/rm_images.sh
new file mode 100755
index 0000000..6722573
--- /dev/null
+++ b/update/templates/rm_images.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+number=`docker images | awk 'NR != 1' | grep testapi | wc -l`
+if [ $number -gt 0 ]; then
+    images=`docker images -a | awk 'NR != 1' | grep testapi | awk '{print $1}'`
+    echo "begin to rm images $images"
+    docker images | awk 'NR != 1' | grep testapi | awk '{print $3}' | xargs docker rmi -f &>/dev/null
+fi
diff --git a/update/templates/rm_olds.sh b/update/templates/rm_olds.sh
new file mode 100644
index 0000000..c6bca18
--- /dev/null
+++ b/update/templates/rm_olds.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+proc_number=`ps -ef | grep opnfv-testapi | grep -v grep | wc -l`
+if [ $proc_number -gt 0 ]; then
+    procs=`ps -ef | grep opnfv-testapi | grep -v grep`
+    echo "begin to kill opnfv-testapi $procs"
+    ps -ef | grep opnfv-testapi | grep -v grep | awk '{print $2}' | xargs kill -kill &>/dev/null
+fi
+
+number=`docker ps -a | awk 'NR != 1' | grep testapi | wc -l`
+if [ $number -gt 0 ]; then
+    containers=number=`docker ps -a | awk 'NR != 1' | grep testapi`
+    echo "begin to rm containers $containers"
+    docker ps -a | awk 'NR != 1' | grep testapi | awk '{print $1}' | xargs docker rm -f &>/dev/null
+fi
diff --git a/update/templates/update_mongodb.py b/update/templates/update_mongodb.py
new file mode 100644
index 0000000..f759592
--- /dev/null
+++ b/update/templates/update_mongodb.py
@@ -0,0 +1,90 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+
+from pymongo import MongoClient
+
+from changes_in_mongodb import collections_old2New, \
+    fields_old2New, docs_old2New
+from utils import main, parse_mongodb_url
+
+parser = argparse.ArgumentParser(description='Update MongoDBs')
+
+parser.add_argument('-u', '--url',
+                    type=str,
+                    required=False,
+                    default='mongodb://127.0.0.1:27017/',
+                    help='Mongo DB URL for Backups')
+
+parser.add_argument('-d', '--db',
+                    type=str,
+                    required=False,
+                    default='test_results_collection',
+                    help='database for the update.')
+
+
+def assert_collections(a_dict):
+    if a_dict is not None:
+        collections = eval_db('collection_names')
+        no_collections = []
+        for collection in a_dict.keys():
+            if collection not in collections:
+                no_collections.append(collection)
+        assert len(no_collections) == 0, \
+            'collections {} not exist'.format(no_collections)
+
+
+def rename_collections(a_dict):
+    if a_dict is not None:
+        for collection, new_name in a_dict.iteritems():
+            eval_collection(collection, 'rename', new_name)
+
+
+def rename_fields(a_dict):
+    collection_update(a_dict, '$rename')
+
+
+def change_docs(a_dict):
+    collection_update(a_dict, '$set')
+
+
+def eval_db(method, *args, **kwargs):
+    exec_db = db.__getattribute__(method)
+    return exec_db(*args, **kwargs)
+
+
+def eval_collection(collection, method, *args, **kwargs):
+    exec_collection = db.__getattr__(collection)
+    return exec_collection.__getattribute__(method)(*args, **kwargs)
+
+
+def collection_update(a_dict, operator):
+    if a_dict is not None:
+        for collection, updates in a_dict.iteritems():
+            for (query, doc) in updates:
+                doc_dict = {operator: doc}
+                eval_collection(collection, 'update', query,
+                                doc_dict, upsert=False, multi=True)
+
+
+def update(args):
+    parse_mongodb_url(args.url)
+    client = MongoClient(args.url)
+    global db
+    db = client[args.db]
+    assert_collections(docs_old2New)
+    assert_collections(fields_old2New)
+    assert_collections(collections_old2New)
+    change_docs(docs_old2New)
+    rename_fields(fields_old2New)
+    rename_collections(collections_old2New)
+
+
+if __name__ == '__main__':
+    main(update, parser)
diff --git a/update/templates/utils.py b/update/templates/utils.py
new file mode 100644
index 0000000..4254fee
--- /dev/null
+++ b/update/templates/utils.py
@@ -0,0 +1,48 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import urlparse
+import subprocess
+
+
+def get_abspath(path):
+    assert os.path.isdir(path), 'Path %s can\'t be found.' % path
+    return os.path.abspath(path)
+
+
+def parse_mongodb_url(url):
+    url = urlparse.urlparse(url)
+    assert url.scheme == 'mongodb', 'URL must be a MongoDB URL'
+    return url
+
+
+def url_parse(url):
+    url = parse_mongodb_url(url)
+    return url.username, url.password, url.hostname, url.port
+
+
+def execute(cmd, args):
+    (username, password, hostname, port) = url_parse(args.url)
+    cmd.extend(['--host', '%s' % hostname, '--port', '%s' % port])
+    db = args.db
+    if db is not None:
+        cmd.extend(['--db', '%s' % db])
+    if username is not None:
+        cmd.extend(['-u', '%s' % username, '-p', '%s' % password])
+    print('execute: %s' % cmd)
+    execute_output = subprocess.check_output(cmd)
+    print(execute_output)
+
+
+def main(method, parser):
+    args = parser.parse_args()
+    try:
+        method(args)
+    except AssertionError as msg:
+        print(msg)
diff --git a/update/test.yml b/update/test.yml
new file mode 100644
index 0000000..943105c
--- /dev/null
+++ b/update/test.yml
@@ -0,0 +1,12 @@
+---
+- hosts: "{{ host }}"
+  remote_user: "{{ user }}"
+  become: "yes"
+  become_method: sudo
+  vars:
+    user: "root"
+  tasks:
+    - name: test connectivity
+      command: "echo hello {{ host }}"
+      register: result
+    - debug: msg="{{ result }}"
diff --git a/update/update.yml b/update/update.yml
new file mode 100644
index 0000000..18b75b6
--- /dev/null
+++ b/update/update.yml
@@ -0,0 +1,50 @@
+---
+- hosts: "{{ host }}"
+  remote_user: "{{ user }}"
+  become: "yes"
+  become_method: sudo
+  vars:
+    user: "root"
+    port: "8000"
+    update_path: "/tmp/testapi"
+    image: "opnfv/testapi"
+    mode: "pull"
+    mongodb_url: "mongodb://172.17.0.1:27017"
+    swagger_url: "http://{{ host }}:{{ port }}"
+  tasks:
+    - name: create temporary update directory
+      file:
+        path: "{{ update_path }}"
+        state: directory
+    - name: transfer files in templates
+      copy:
+        src: templates/
+        dest: "{{ update_path }}"
+    - name: transfer Dockerfile
+      copy:
+        src: ../docker/Dockerfile
+        dest: "{{ update_path }}"
+      when: mode == "build"
+    - name: backup mongodb database
+      command: "python {{ update_path }}/backup_mongodb.py -u {{ mongodb_url }} -o {{ update_path }}"
+    - name: stop and remove old versions
+      command: bash "{{ update_path }}/rm_olds.sh"
+      register: rm_result
+    - debug: msg="{{ rm_result.stderr }}"
+    - name: delete old docker images
+      command: bash "{{ update_path }}/rm_images.sh"
+      ignore_errors: true
+    - name: update mongodb
+      command: "python {{ update_path }}/update_mongodb.py -u {{ mongodb_url }}"
+    - name: docker build image
+      command: "docker build -t {{ image }} {{ update_path }}"
+      when: mode == "build"
+    - name: docker start testapi server
+      command: docker run -dti -p "{{ port }}:8000"
+               -e "mongodb_url={{ mongodb_url }}"
+               -e "swagger_url={{ swagger_url }}"
+               "{{ image }}"
+    - name: remove temporary update directory
+      file:
+        path: "{{ update_path }}"
+        state: absent
diff --git a/update/update_api.py b/update/update_api.py
new file mode 100644
index 0000000..db8ad2d
--- /dev/null
+++ b/update/update_api.py
@@ -0,0 +1,55 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import subprocess
+
+possible_processes = [
+    'result_collection_api',
+    'opnfv-testapi'
+]
+
+
+def kill_olds():
+    for proc in possible_processes:
+        query = 'ps -ef | grep {} | grep -v grep'.format(proc)
+        runnings = execute_with_output(query)
+        if runnings:
+            for running in runnings:
+                kill = 'kill -kill ' + running.split()[1]
+                execute_with_output(kill)
+            runnings = execute_with_output(query)
+        assert len(runnings) == 0, 'kill %s failed'.format(proc)
+
+
+def install_dependencies():
+    execute_with_assert('pip install -r ../requirements.txt')
+
+
+def install_new():
+    execute_with_assert('cd ../ && python setup.py install')
+
+
+def run_new():
+    execute_with_assert('opnfv-testapi &')
+
+
+def execute_with_output(cmd):
+    return subprocess.Popen(cmd, shell=True,
+                            stdout=subprocess.PIPE).stdout.readlines()
+
+
+def execute_with_assert(cmd):
+    execute_output = subprocess.call(cmd, shell=True)
+    assert execute_output == 0
+
+
+if __name__ == '__main__':
+    kill_olds()
+    install_dependencies()
+    install_new()
+    run_new()