summaryrefslogtreecommitdiffstats
path: root/cvp/update
diff options
context:
space:
mode:
Diffstat (limited to 'cvp/update')
-rw-r--r--cvp/update/README.md100
-rw-r--r--cvp/update/__init__.py8
-rwxr-xr-xcvp/update/playbook-update.sh90
-rw-r--r--cvp/update/templates/__init__.py0
-rw-r--r--cvp/update/templates/backup_mongodb.py45
-rw-r--r--cvp/update/templates/changes_in_mongodb.py51
-rw-r--r--cvp/update/templates/restore_mongodb.py38
-rwxr-xr-xcvp/update/templates/rm_images.sh8
-rw-r--r--cvp/update/templates/rm_olds.sh15
-rw-r--r--cvp/update/templates/update_mongodb.py90
-rw-r--r--cvp/update/templates/utils.py48
-rw-r--r--cvp/update/test.yml12
-rw-r--r--cvp/update/update.yml50
-rw-r--r--cvp/update/update_api.py55
14 files changed, 610 insertions, 0 deletions
diff --git a/cvp/update/README.md b/cvp/update/README.md
new file mode 100644
index 00000000..cb0e67b3
--- /dev/null
+++ b/cvp/update/README.md
@@ -0,0 +1,100 @@
+Welcome to TESTAPI Update!
+========================
+
+
+This file is used to describe how testapi update works
+
+----------
+How to use
+---------------
+
+#### <i class="icon-file"></i> backup mongodb
+
+arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+the backup output will be put under dir/db__XXXX_XX_XX_XXXXXX/db
+-d/--db: database for the backup, default = test_results_collection
+
+usage:
+```
+python backup_mongodb.py
+```
+
+#### <i class="icon-file"></i> restore mongodb
+
+arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+ -i/--input_dir: Input directory for the Restore, must be specified,
+ the restore input must be specified to dir/db__XXXX_XX_XX_XXXXXX/db
+ -d/--db: database name after the restore, default = basename of input_dir
+
+usage:
+```
+python restore_mongodb.py
+```
+#### <i class="icon-file"></i> update mongodb
+
+ arguments:
+: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/
+ -d/--db: database name to be updated, default = test_results_collection
+
+changes need to be done:
+change collection name, modify changes.collections_old2New
+ > collections_old2New = {
+ 'old_collection': 'new_collection',
+ }
+
+ change field name, modify changes.fields_old2New
+ > fields_old2New = {
+ 'collection': [(query, {'old_field': 'new_field'})]
+ }
+
+ change the doc, modify changes.docs_old2New
+ > docs_old2New = {
+ 'test_results': [
+ ({'field': 'old_value'}, {'field': 'new_value'}),
+ (query, {'field': 'new_value'}),
+ ]
+ }
+
+#### <i class="icon-file"></i> update opnfv-testapi process
+This script must be run right in this directory and remember to
+change ../etc/config.ini before running this script.
+
+operations includes:
+: kill running test_collection_api & opnfv-testapi
+install or update dependencies according to ../requirements.txt
+install opnfv-testapi
+run opnfv-testapi
+
+usage:
+```
+python update_api.py
+```
+#### <i class="icon-file"></i> update opnfv/testapi container
+Here ansible-playbook is used to implement auto update.
+Please make sure that the remote server is accessible via ssh.
+
+install ansible, please refer:
+```
+http://docs.ansible.com/ansible/intro_installation.html
+```
+
+playbook-update.sh
+
+arguments:
+: -h|--help show this help text
+-r|--remote remote server
+-u|--user ssh username used to access to remote server
+-i|--identity ssh PublicKey file used to access to remote server
+-e|--execute execute update, if not set just check the ansible connectivity
+
+usage:
+```
+ssh-agent ./playbook-update.sh -r testresults.opnfv.org -u serena -i ~/.ssh/id_rsa -e
+```
+
+> **Note:**
+
+> - If documents need to be changed, please modify file
+templates/changes_in_mongodb.py, and refer section **update mongodb**
diff --git a/cvp/update/__init__.py b/cvp/update/__init__.py
new file mode 100644
index 00000000..363bc388
--- /dev/null
+++ b/cvp/update/__init__.py
@@ -0,0 +1,8 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
diff --git a/cvp/update/playbook-update.sh b/cvp/update/playbook-update.sh
new file mode 100755
index 00000000..86d30e4b
--- /dev/null
+++ b/cvp/update/playbook-update.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+#
+# Author: Serena Feng (feng.xiaoewi@zte.com.cn)
+# Update testapi on remote server using ansible playbook automatically
+#
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+set -e
+
+usage="Script to trigger update automatically.
+
+usage:
+ bash $(basename "$0") [-h|--help] [-h <host>] [-u username] [-i identityfile] [-e|--execute]
+
+where:
+ -h|--help show this help text
+ -r|--remote remote server
+ -u|--user ssh username used to access to remote server
+ -i|--identity ssh PublicKey file used to access to remote server
+ -e|--execute execute update, if not set just check the ansible connectivity"
+
+remote=testresults.opnfv.org
+user=root
+identity=~/.ssh/id_rsa
+hosts=./hosts
+execute=false
+
+# Parse parameters
+while [[ $# > 0 ]]
+ do
+ key="$1"
+ case $key in
+ -h|--help)
+ echo "$usage"
+ exit 0
+ shift
+ ;;
+ -r|--remote)
+ remote="$2"
+ shift
+ ;;
+ -u|--user)
+ user="$2"
+ shift
+ ;;
+ -i|--identity)
+ identity="$2"
+ shift
+ ;;
+ -e|--execute)
+ execute=true
+ ;;
+ *)
+ echo "unknown option"
+ exit 1
+ ;;
+ esac
+ shift # past argument or value
+done
+
+echo $remote > $hosts
+
+echo "add authentication"
+ssh-add $identity
+
+echo "test ansible connectivity"
+ansible -i ./hosts $remote -m ping -u $user
+
+echo "test playbook connectivity"
+ansible-playbook -i $hosts test.yml -e "host=$remote user=$user"
+
+if [ $execute == true ]; then
+ echo "do update"
+ ansible-playbook -i $hosts update.yml -e "host=$remote \
+ user=$user \
+ port=8082 \
+ image=opnfv/testapi \
+ update_path=/home/$user/testapi \
+ mongodb_url=mongodb://172.17.0.1:27017 \
+ swagger_url=http://testresults.opnfv.org/test"
+fi
+
+rm -fr $hosts
+ssh-agent -k
diff --git a/cvp/update/templates/__init__.py b/cvp/update/templates/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/cvp/update/templates/__init__.py
diff --git a/cvp/update/templates/backup_mongodb.py b/cvp/update/templates/backup_mongodb.py
new file mode 100644
index 00000000..9c243776
--- /dev/null
+++ b/cvp/update/templates/backup_mongodb.py
@@ -0,0 +1,45 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+import datetime
+import os
+
+from utils import execute, main, get_abspath
+
+parser = argparse.ArgumentParser(description='Backup MongoDBs')
+
+parser.add_argument('-u', '--url',
+ type=str,
+ required=False,
+ default='mongodb://127.0.0.1:27017/',
+ help='Mongo DB URL for Backups')
+parser.add_argument('-o', '--output_dir',
+ type=str,
+ required=False,
+ default='./',
+ help='Output directory for the backup.')
+
+parser.add_argument('-d', '--db',
+ type=str,
+ required=False,
+ default='test_results_collection',
+ help='database for the backup.')
+
+
+def backup(args):
+ db = args.db
+ out = get_abspath(args.output_dir)
+ now = datetime.datetime.now()
+ out = os.path.join(out, '%s__%s' % (db, now.strftime('%Y_%m_%d_%H%M%S')))
+ cmd = ['mongodump', '-o', '%s' % out]
+ execute(cmd, args)
+
+
+if __name__ == '__main__':
+ main(backup, parser)
diff --git a/cvp/update/templates/changes_in_mongodb.py b/cvp/update/templates/changes_in_mongodb.py
new file mode 100644
index 00000000..1a4d5a16
--- /dev/null
+++ b/cvp/update/templates/changes_in_mongodb.py
@@ -0,0 +1,51 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# 09/06/2016: change for migration after refactoring
+# 16/06/2016: Alignment of test name (JIRA: FUNCTEST-304)
+##############################################################################
+collections_old2New = {
+ # 'pod': 'pods',
+ # 'test_projects': 'projects',
+ # 'test_testcases': 'testcases',
+ # 'test_results': 'results'
+}
+
+fields_old2New = {
+ # 'test_results': [({}, {'creation_date': 'start_date'})]
+}
+
+docs_old2New = {
+ # 'test_results': [
+ # ({'criteria': 'failed'}, {'criteria': 'FAILED'}),
+ # ({'criteria': 'passed'}, {'criteria': 'PASS'})
+ # ]
+ # 'testcases': [
+ # ({'name': 'vPing'}, {'name': 'vping_ssh'}),
+ # ({'name': 'Tempest'}, {'name': 'tempest_smoke_serial'}),
+ # ({'name': 'Rally'}, {'name': 'rally_sanity'}),
+ # ({'name': 'ODL'}, {'name': 'odl'}),
+ # ({'name': 'vIMS'}, {'name': 'vims'}),
+ # ({'name': 'ONOS'}, {'name': 'onos'}),
+ # ({'name': 'vPing_userdata'}, {'name': 'vping_userdata'}),
+ # ({'name': 'ovno'}, {'name': 'ocl'})
+ # ],
+ # 'results': [
+ # ({'case_name': 'vPing'}, {'case_name': 'vping_ssh'}),
+ # ({'case_name': 'Tempest'}, {'case_name': 'tempest_smoke_serial'}),
+ # ({'case_name': 'Rally'}, {'case_name': 'rally_sanity'}),
+ # ({'case_name': 'ODL'}, {'case_name': 'odl'}),
+ # ({'case_name': 'vIMS'}, {'case_name': 'vims'}),
+ # ({'case_name': 'ONOS'}, {'case_name': 'onos'}),
+ # ({'case_name': 'vPing_userdata'}, {'case_name': 'vping_userdata'}),
+ # ({'case_name': 'ovno'}, {'case_name': 'ocl'})
+ # ]
+ 'results': [
+ ({'trust_indicator': 0},
+ {'trust_indicator': {'current': 0, 'histories': []}})
+ ]
+}
diff --git a/cvp/update/templates/restore_mongodb.py b/cvp/update/templates/restore_mongodb.py
new file mode 100644
index 00000000..c45a0e62
--- /dev/null
+++ b/cvp/update/templates/restore_mongodb.py
@@ -0,0 +1,38 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+
+from utils import execute, main, get_abspath
+
+parser = argparse.ArgumentParser(description='Restore MongoDBs')
+
+parser.add_argument('-u', '--url',
+ type=str,
+ required=False,
+ default='mongodb://127.0.0.1:27017/',
+ help='Mongo DB URL for Backup')
+parser.add_argument('-i', '--input_dir',
+ type=str,
+ required=True,
+ help='Input directory for the Restore.')
+parser.add_argument('-d', '--db',
+ type=str,
+ required=False,
+ default='test_results_collection',
+ help='database name after the restore.')
+
+
+def restore(args):
+ input_dir = get_abspath(args.input_dir)
+ cmd = ['mongorestore', '%s' % input_dir]
+ execute(cmd, args)
+
+
+if __name__ == '__main__':
+ main(restore, parser)
diff --git a/cvp/update/templates/rm_images.sh b/cvp/update/templates/rm_images.sh
new file mode 100755
index 00000000..6722573b
--- /dev/null
+++ b/cvp/update/templates/rm_images.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+number=`docker images | awk 'NR != 1' | grep testapi | wc -l`
+if [ $number -gt 0 ]; then
+ images=`docker images -a | awk 'NR != 1' | grep testapi | awk '{print $1}'`
+ echo "begin to rm images $images"
+ docker images | awk 'NR != 1' | grep testapi | awk '{print $3}' | xargs docker rmi -f &>/dev/null
+fi
diff --git a/cvp/update/templates/rm_olds.sh b/cvp/update/templates/rm_olds.sh
new file mode 100644
index 00000000..c6bca186
--- /dev/null
+++ b/cvp/update/templates/rm_olds.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+proc_number=`ps -ef | grep opnfv-testapi | grep -v grep | wc -l`
+if [ $proc_number -gt 0 ]; then
+ procs=`ps -ef | grep opnfv-testapi | grep -v grep`
+ echo "begin to kill opnfv-testapi $procs"
+ ps -ef | grep opnfv-testapi | grep -v grep | awk '{print $2}' | xargs kill -kill &>/dev/null
+fi
+
+number=`docker ps -a | awk 'NR != 1' | grep testapi | wc -l`
+if [ $number -gt 0 ]; then
+ containers=number=`docker ps -a | awk 'NR != 1' | grep testapi`
+ echo "begin to rm containers $containers"
+ docker ps -a | awk 'NR != 1' | grep testapi | awk '{print $1}' | xargs docker rm -f &>/dev/null
+fi
diff --git a/cvp/update/templates/update_mongodb.py b/cvp/update/templates/update_mongodb.py
new file mode 100644
index 00000000..f7595928
--- /dev/null
+++ b/cvp/update/templates/update_mongodb.py
@@ -0,0 +1,90 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import argparse
+
+from pymongo import MongoClient
+
+from changes_in_mongodb import collections_old2New, \
+ fields_old2New, docs_old2New
+from utils import main, parse_mongodb_url
+
+parser = argparse.ArgumentParser(description='Update MongoDBs')
+
+parser.add_argument('-u', '--url',
+ type=str,
+ required=False,
+ default='mongodb://127.0.0.1:27017/',
+ help='Mongo DB URL for Backups')
+
+parser.add_argument('-d', '--db',
+ type=str,
+ required=False,
+ default='test_results_collection',
+ help='database for the update.')
+
+
+def assert_collections(a_dict):
+ if a_dict is not None:
+ collections = eval_db('collection_names')
+ no_collections = []
+ for collection in a_dict.keys():
+ if collection not in collections:
+ no_collections.append(collection)
+ assert len(no_collections) == 0, \
+ 'collections {} not exist'.format(no_collections)
+
+
+def rename_collections(a_dict):
+ if a_dict is not None:
+ for collection, new_name in a_dict.iteritems():
+ eval_collection(collection, 'rename', new_name)
+
+
+def rename_fields(a_dict):
+ collection_update(a_dict, '$rename')
+
+
+def change_docs(a_dict):
+ collection_update(a_dict, '$set')
+
+
+def eval_db(method, *args, **kwargs):
+ exec_db = db.__getattribute__(method)
+ return exec_db(*args, **kwargs)
+
+
+def eval_collection(collection, method, *args, **kwargs):
+ exec_collection = db.__getattr__(collection)
+ return exec_collection.__getattribute__(method)(*args, **kwargs)
+
+
+def collection_update(a_dict, operator):
+ if a_dict is not None:
+ for collection, updates in a_dict.iteritems():
+ for (query, doc) in updates:
+ doc_dict = {operator: doc}
+ eval_collection(collection, 'update', query,
+ doc_dict, upsert=False, multi=True)
+
+
+def update(args):
+ parse_mongodb_url(args.url)
+ client = MongoClient(args.url)
+ global db
+ db = client[args.db]
+ assert_collections(docs_old2New)
+ assert_collections(fields_old2New)
+ assert_collections(collections_old2New)
+ change_docs(docs_old2New)
+ rename_fields(fields_old2New)
+ rename_collections(collections_old2New)
+
+
+if __name__ == '__main__':
+ main(update, parser)
diff --git a/cvp/update/templates/utils.py b/cvp/update/templates/utils.py
new file mode 100644
index 00000000..4254fee3
--- /dev/null
+++ b/cvp/update/templates/utils.py
@@ -0,0 +1,48 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import urlparse
+import subprocess
+
+
+def get_abspath(path):
+ assert os.path.isdir(path), 'Path %s can\'t be found.' % path
+ return os.path.abspath(path)
+
+
+def parse_mongodb_url(url):
+ url = urlparse.urlparse(url)
+ assert url.scheme == 'mongodb', 'URL must be a MongoDB URL'
+ return url
+
+
+def url_parse(url):
+ url = parse_mongodb_url(url)
+ return url.username, url.password, url.hostname, url.port
+
+
+def execute(cmd, args):
+ (username, password, hostname, port) = url_parse(args.url)
+ cmd.extend(['--host', '%s' % hostname, '--port', '%s' % port])
+ db = args.db
+ if db is not None:
+ cmd.extend(['--db', '%s' % db])
+ if username is not None:
+ cmd.extend(['-u', '%s' % username, '-p', '%s' % password])
+ print('execute: %s' % cmd)
+ execute_output = subprocess.check_output(cmd)
+ print(execute_output)
+
+
+def main(method, parser):
+ args = parser.parse_args()
+ try:
+ method(args)
+ except AssertionError as msg:
+ print(msg)
diff --git a/cvp/update/test.yml b/cvp/update/test.yml
new file mode 100644
index 00000000..943105c5
--- /dev/null
+++ b/cvp/update/test.yml
@@ -0,0 +1,12 @@
+---
+- hosts: "{{ host }}"
+ remote_user: "{{ user }}"
+ become: "yes"
+ become_method: sudo
+ vars:
+ user: "root"
+ tasks:
+ - name: test connectivity
+ command: "echo hello {{ host }}"
+ register: result
+ - debug: msg="{{ result }}"
diff --git a/cvp/update/update.yml b/cvp/update/update.yml
new file mode 100644
index 00000000..18b75b6b
--- /dev/null
+++ b/cvp/update/update.yml
@@ -0,0 +1,50 @@
+---
+- hosts: "{{ host }}"
+ remote_user: "{{ user }}"
+ become: "yes"
+ become_method: sudo
+ vars:
+ user: "root"
+ port: "8000"
+ update_path: "/tmp/testapi"
+ image: "opnfv/testapi"
+ mode: "pull"
+ mongodb_url: "mongodb://172.17.0.1:27017"
+ swagger_url: "http://{{ host }}:{{ port }}"
+ tasks:
+ - name: create temporary update directory
+ file:
+ path: "{{ update_path }}"
+ state: directory
+ - name: transfer files in templates
+ copy:
+ src: templates/
+ dest: "{{ update_path }}"
+ - name: transfer Dockerfile
+ copy:
+ src: ../docker/Dockerfile
+ dest: "{{ update_path }}"
+ when: mode == "build"
+ - name: backup mongodb database
+ command: "python {{ update_path }}/backup_mongodb.py -u {{ mongodb_url }} -o {{ update_path }}"
+ - name: stop and remove old versions
+ command: bash "{{ update_path }}/rm_olds.sh"
+ register: rm_result
+ - debug: msg="{{ rm_result.stderr }}"
+ - name: delete old docker images
+ command: bash "{{ update_path }}/rm_images.sh"
+ ignore_errors: true
+ - name: update mongodb
+ command: "python {{ update_path }}/update_mongodb.py -u {{ mongodb_url }}"
+ - name: docker build image
+ command: "docker build -t {{ image }} {{ update_path }}"
+ when: mode == "build"
+ - name: docker start testapi server
+ command: docker run -dti -p "{{ port }}:8000"
+ -e "mongodb_url={{ mongodb_url }}"
+ -e "swagger_url={{ swagger_url }}"
+ "{{ image }}"
+ - name: remove temporary update directory
+ file:
+ path: "{{ update_path }}"
+ state: absent
diff --git a/cvp/update/update_api.py b/cvp/update/update_api.py
new file mode 100644
index 00000000..db8ad2d9
--- /dev/null
+++ b/cvp/update/update_api.py
@@ -0,0 +1,55 @@
+##############################################################################
+# Copyright (c) 2016 ZTE Corporation
+# feng.xiaowei@zte.com.cn
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import subprocess
+
+possible_processes = [
+ 'result_collection_api',
+ 'opnfv-testapi'
+]
+
+
+def kill_olds():
+ for proc in possible_processes:
+ query = 'ps -ef | grep {} | grep -v grep'.format(proc)
+ runnings = execute_with_output(query)
+ if runnings:
+ for running in runnings:
+ kill = 'kill -kill ' + running.split()[1]
+ execute_with_output(kill)
+ runnings = execute_with_output(query)
+ assert len(runnings) == 0, 'kill %s failed'.format(proc)
+
+
+def install_dependencies():
+ execute_with_assert('pip install -r ../requirements.txt')
+
+
+def install_new():
+ execute_with_assert('cd ../ && python setup.py install')
+
+
+def run_new():
+ execute_with_assert('opnfv-testapi &')
+
+
+def execute_with_output(cmd):
+ return subprocess.Popen(cmd, shell=True,
+ stdout=subprocess.PIPE).stdout.readlines()
+
+
+def execute_with_assert(cmd):
+ execute_output = subprocess.call(cmd, shell=True)
+ assert execute_output == 0
+
+
+if __name__ == '__main__':
+ kill_olds()
+ install_dependencies()
+ install_new()
+ run_new()