diff options
Diffstat (limited to 'cvp/update')
| -rw-r--r-- | cvp/update/README.md | 100 | ||||
| -rw-r--r-- | cvp/update/__init__.py | 8 | ||||
| -rwxr-xr-x | cvp/update/playbook-update.sh | 90 | ||||
| -rw-r--r-- | cvp/update/templates/__init__.py | 0 | ||||
| -rw-r--r-- | cvp/update/templates/backup_mongodb.py | 45 | ||||
| -rw-r--r-- | cvp/update/templates/changes_in_mongodb.py | 51 | ||||
| -rw-r--r-- | cvp/update/templates/restore_mongodb.py | 38 | ||||
| -rwxr-xr-x | cvp/update/templates/rm_images.sh | 8 | ||||
| -rw-r--r-- | cvp/update/templates/rm_olds.sh | 15 | ||||
| -rw-r--r-- | cvp/update/templates/update_mongodb.py | 90 | ||||
| -rw-r--r-- | cvp/update/templates/utils.py | 48 | ||||
| -rw-r--r-- | cvp/update/test.yml | 12 | ||||
| -rw-r--r-- | cvp/update/update.yml | 50 | ||||
| -rw-r--r-- | cvp/update/update_api.py | 55 |
14 files changed, 610 insertions, 0 deletions
diff --git a/cvp/update/README.md b/cvp/update/README.md new file mode 100644 index 00000000..cb0e67b3 --- /dev/null +++ b/cvp/update/README.md @@ -0,0 +1,100 @@ +Welcome to TESTAPI Update! +======================== + + +This file is used to describe how testapi update works + +---------- +How to use +--------------- + +#### <i class="icon-file"></i> backup mongodb + +arguments: +: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ +the backup output will be put under dir/db__XXXX_XX_XX_XXXXXX/db +-d/--db: database for the backup, default = test_results_collection + +usage: +``` +python backup_mongodb.py +``` + +#### <i class="icon-file"></i> restore mongodb + +arguments: +: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ + -i/--input_dir: Input directory for the Restore, must be specified, + the restore input must be specified to dir/db__XXXX_XX_XX_XXXXXX/db + -d/--db: database name after the restore, default = basename of input_dir + +usage: +``` +python restore_mongodb.py +``` +#### <i class="icon-file"></i> update mongodb + + arguments: +: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ + -d/--db: database name to be updated, default = test_results_collection + +changes need to be done: +change collection name, modify changes.collections_old2New + > collections_old2New = { + 'old_collection': 'new_collection', + } + + change field name, modify changes.fields_old2New + > fields_old2New = { + 'collection': [(query, {'old_field': 'new_field'})] + } + + change the doc, modify changes.docs_old2New + > docs_old2New = { + 'test_results': [ + ({'field': 'old_value'}, {'field': 'new_value'}), + (query, {'field': 'new_value'}), + ] + } + +#### <i class="icon-file"></i> update opnfv-testapi process +This script must be run right in this directory and remember to +change ../etc/config.ini before running this script. + +operations includes: +: kill running test_collection_api & opnfv-testapi +install or update dependencies according to ../requirements.txt +install opnfv-testapi +run opnfv-testapi + +usage: +``` +python update_api.py +``` +#### <i class="icon-file"></i> update opnfv/testapi container +Here ansible-playbook is used to implement auto update. +Please make sure that the remote server is accessible via ssh. + +install ansible, please refer: +``` +http://docs.ansible.com/ansible/intro_installation.html +``` + +playbook-update.sh + +arguments: +: -h|--help show this help text +-r|--remote remote server +-u|--user ssh username used to access to remote server +-i|--identity ssh PublicKey file used to access to remote server +-e|--execute execute update, if not set just check the ansible connectivity + +usage: +``` +ssh-agent ./playbook-update.sh -r testresults.opnfv.org -u serena -i ~/.ssh/id_rsa -e +``` + +> **Note:** + +> - If documents need to be changed, please modify file +templates/changes_in_mongodb.py, and refer section **update mongodb** diff --git a/cvp/update/__init__.py b/cvp/update/__init__.py new file mode 100644 index 00000000..363bc388 --- /dev/null +++ b/cvp/update/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/cvp/update/playbook-update.sh b/cvp/update/playbook-update.sh new file mode 100755 index 00000000..86d30e4b --- /dev/null +++ b/cvp/update/playbook-update.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# +# Author: Serena Feng (feng.xiaoewi@zte.com.cn) +# Update testapi on remote server using ansible playbook automatically +# +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +set -e + +usage="Script to trigger update automatically. + +usage: + bash $(basename "$0") [-h|--help] [-h <host>] [-u username] [-i identityfile] [-e|--execute] + +where: + -h|--help show this help text + -r|--remote remote server + -u|--user ssh username used to access to remote server + -i|--identity ssh PublicKey file used to access to remote server + -e|--execute execute update, if not set just check the ansible connectivity" + +remote=testresults.opnfv.org +user=root +identity=~/.ssh/id_rsa +hosts=./hosts +execute=false + +# Parse parameters +while [[ $# > 0 ]] + do + key="$1" + case $key in + -h|--help) + echo "$usage" + exit 0 + shift + ;; + -r|--remote) + remote="$2" + shift + ;; + -u|--user) + user="$2" + shift + ;; + -i|--identity) + identity="$2" + shift + ;; + -e|--execute) + execute=true + ;; + *) + echo "unknown option" + exit 1 + ;; + esac + shift # past argument or value +done + +echo $remote > $hosts + +echo "add authentication" +ssh-add $identity + +echo "test ansible connectivity" +ansible -i ./hosts $remote -m ping -u $user + +echo "test playbook connectivity" +ansible-playbook -i $hosts test.yml -e "host=$remote user=$user" + +if [ $execute == true ]; then + echo "do update" + ansible-playbook -i $hosts update.yml -e "host=$remote \ + user=$user \ + port=8082 \ + image=opnfv/testapi \ + update_path=/home/$user/testapi \ + mongodb_url=mongodb://172.17.0.1:27017 \ + swagger_url=http://testresults.opnfv.org/test" +fi + +rm -fr $hosts +ssh-agent -k diff --git a/cvp/update/templates/__init__.py b/cvp/update/templates/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/cvp/update/templates/__init__.py diff --git a/cvp/update/templates/backup_mongodb.py b/cvp/update/templates/backup_mongodb.py new file mode 100644 index 00000000..9c243776 --- /dev/null +++ b/cvp/update/templates/backup_mongodb.py @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import argparse +import datetime +import os + +from utils import execute, main, get_abspath + +parser = argparse.ArgumentParser(description='Backup MongoDBs') + +parser.add_argument('-u', '--url', + type=str, + required=False, + default='mongodb://127.0.0.1:27017/', + help='Mongo DB URL for Backups') +parser.add_argument('-o', '--output_dir', + type=str, + required=False, + default='./', + help='Output directory for the backup.') + +parser.add_argument('-d', '--db', + type=str, + required=False, + default='test_results_collection', + help='database for the backup.') + + +def backup(args): + db = args.db + out = get_abspath(args.output_dir) + now = datetime.datetime.now() + out = os.path.join(out, '%s__%s' % (db, now.strftime('%Y_%m_%d_%H%M%S'))) + cmd = ['mongodump', '-o', '%s' % out] + execute(cmd, args) + + +if __name__ == '__main__': + main(backup, parser) diff --git a/cvp/update/templates/changes_in_mongodb.py b/cvp/update/templates/changes_in_mongodb.py new file mode 100644 index 00000000..1a4d5a16 --- /dev/null +++ b/cvp/update/templates/changes_in_mongodb.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# 09/06/2016: change for migration after refactoring +# 16/06/2016: Alignment of test name (JIRA: FUNCTEST-304) +############################################################################## +collections_old2New = { + # 'pod': 'pods', + # 'test_projects': 'projects', + # 'test_testcases': 'testcases', + # 'test_results': 'results' +} + +fields_old2New = { + # 'test_results': [({}, {'creation_date': 'start_date'})] +} + +docs_old2New = { + # 'test_results': [ + # ({'criteria': 'failed'}, {'criteria': 'FAILED'}), + # ({'criteria': 'passed'}, {'criteria': 'PASS'}) + # ] + # 'testcases': [ + # ({'name': 'vPing'}, {'name': 'vping_ssh'}), + # ({'name': 'Tempest'}, {'name': 'tempest_smoke_serial'}), + # ({'name': 'Rally'}, {'name': 'rally_sanity'}), + # ({'name': 'ODL'}, {'name': 'odl'}), + # ({'name': 'vIMS'}, {'name': 'vims'}), + # ({'name': 'ONOS'}, {'name': 'onos'}), + # ({'name': 'vPing_userdata'}, {'name': 'vping_userdata'}), + # ({'name': 'ovno'}, {'name': 'ocl'}) + # ], + # 'results': [ + # ({'case_name': 'vPing'}, {'case_name': 'vping_ssh'}), + # ({'case_name': 'Tempest'}, {'case_name': 'tempest_smoke_serial'}), + # ({'case_name': 'Rally'}, {'case_name': 'rally_sanity'}), + # ({'case_name': 'ODL'}, {'case_name': 'odl'}), + # ({'case_name': 'vIMS'}, {'case_name': 'vims'}), + # ({'case_name': 'ONOS'}, {'case_name': 'onos'}), + # ({'case_name': 'vPing_userdata'}, {'case_name': 'vping_userdata'}), + # ({'case_name': 'ovno'}, {'case_name': 'ocl'}) + # ] + 'results': [ + ({'trust_indicator': 0}, + {'trust_indicator': {'current': 0, 'histories': []}}) + ] +} diff --git a/cvp/update/templates/restore_mongodb.py b/cvp/update/templates/restore_mongodb.py new file mode 100644 index 00000000..c45a0e62 --- /dev/null +++ b/cvp/update/templates/restore_mongodb.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import argparse + +from utils import execute, main, get_abspath + +parser = argparse.ArgumentParser(description='Restore MongoDBs') + +parser.add_argument('-u', '--url', + type=str, + required=False, + default='mongodb://127.0.0.1:27017/', + help='Mongo DB URL for Backup') +parser.add_argument('-i', '--input_dir', + type=str, + required=True, + help='Input directory for the Restore.') +parser.add_argument('-d', '--db', + type=str, + required=False, + default='test_results_collection', + help='database name after the restore.') + + +def restore(args): + input_dir = get_abspath(args.input_dir) + cmd = ['mongorestore', '%s' % input_dir] + execute(cmd, args) + + +if __name__ == '__main__': + main(restore, parser) diff --git a/cvp/update/templates/rm_images.sh b/cvp/update/templates/rm_images.sh new file mode 100755 index 00000000..6722573b --- /dev/null +++ b/cvp/update/templates/rm_images.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +number=`docker images | awk 'NR != 1' | grep testapi | wc -l` +if [ $number -gt 0 ]; then + images=`docker images -a | awk 'NR != 1' | grep testapi | awk '{print $1}'` + echo "begin to rm images $images" + docker images | awk 'NR != 1' | grep testapi | awk '{print $3}' | xargs docker rmi -f &>/dev/null +fi diff --git a/cvp/update/templates/rm_olds.sh b/cvp/update/templates/rm_olds.sh new file mode 100644 index 00000000..c6bca186 --- /dev/null +++ b/cvp/update/templates/rm_olds.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +proc_number=`ps -ef | grep opnfv-testapi | grep -v grep | wc -l` +if [ $proc_number -gt 0 ]; then + procs=`ps -ef | grep opnfv-testapi | grep -v grep` + echo "begin to kill opnfv-testapi $procs" + ps -ef | grep opnfv-testapi | grep -v grep | awk '{print $2}' | xargs kill -kill &>/dev/null +fi + +number=`docker ps -a | awk 'NR != 1' | grep testapi | wc -l` +if [ $number -gt 0 ]; then + containers=number=`docker ps -a | awk 'NR != 1' | grep testapi` + echo "begin to rm containers $containers" + docker ps -a | awk 'NR != 1' | grep testapi | awk '{print $1}' | xargs docker rm -f &>/dev/null +fi diff --git a/cvp/update/templates/update_mongodb.py b/cvp/update/templates/update_mongodb.py new file mode 100644 index 00000000..f7595928 --- /dev/null +++ b/cvp/update/templates/update_mongodb.py @@ -0,0 +1,90 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import argparse + +from pymongo import MongoClient + +from changes_in_mongodb import collections_old2New, \ + fields_old2New, docs_old2New +from utils import main, parse_mongodb_url + +parser = argparse.ArgumentParser(description='Update MongoDBs') + +parser.add_argument('-u', '--url', + type=str, + required=False, + default='mongodb://127.0.0.1:27017/', + help='Mongo DB URL for Backups') + +parser.add_argument('-d', '--db', + type=str, + required=False, + default='test_results_collection', + help='database for the update.') + + +def assert_collections(a_dict): + if a_dict is not None: + collections = eval_db('collection_names') + no_collections = [] + for collection in a_dict.keys(): + if collection not in collections: + no_collections.append(collection) + assert len(no_collections) == 0, \ + 'collections {} not exist'.format(no_collections) + + +def rename_collections(a_dict): + if a_dict is not None: + for collection, new_name in a_dict.iteritems(): + eval_collection(collection, 'rename', new_name) + + +def rename_fields(a_dict): + collection_update(a_dict, '$rename') + + +def change_docs(a_dict): + collection_update(a_dict, '$set') + + +def eval_db(method, *args, **kwargs): + exec_db = db.__getattribute__(method) + return exec_db(*args, **kwargs) + + +def eval_collection(collection, method, *args, **kwargs): + exec_collection = db.__getattr__(collection) + return exec_collection.__getattribute__(method)(*args, **kwargs) + + +def collection_update(a_dict, operator): + if a_dict is not None: + for collection, updates in a_dict.iteritems(): + for (query, doc) in updates: + doc_dict = {operator: doc} + eval_collection(collection, 'update', query, + doc_dict, upsert=False, multi=True) + + +def update(args): + parse_mongodb_url(args.url) + client = MongoClient(args.url) + global db + db = client[args.db] + assert_collections(docs_old2New) + assert_collections(fields_old2New) + assert_collections(collections_old2New) + change_docs(docs_old2New) + rename_fields(fields_old2New) + rename_collections(collections_old2New) + + +if __name__ == '__main__': + main(update, parser) diff --git a/cvp/update/templates/utils.py b/cvp/update/templates/utils.py new file mode 100644 index 00000000..4254fee3 --- /dev/null +++ b/cvp/update/templates/utils.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import urlparse +import subprocess + + +def get_abspath(path): + assert os.path.isdir(path), 'Path %s can\'t be found.' % path + return os.path.abspath(path) + + +def parse_mongodb_url(url): + url = urlparse.urlparse(url) + assert url.scheme == 'mongodb', 'URL must be a MongoDB URL' + return url + + +def url_parse(url): + url = parse_mongodb_url(url) + return url.username, url.password, url.hostname, url.port + + +def execute(cmd, args): + (username, password, hostname, port) = url_parse(args.url) + cmd.extend(['--host', '%s' % hostname, '--port', '%s' % port]) + db = args.db + if db is not None: + cmd.extend(['--db', '%s' % db]) + if username is not None: + cmd.extend(['-u', '%s' % username, '-p', '%s' % password]) + print('execute: %s' % cmd) + execute_output = subprocess.check_output(cmd) + print(execute_output) + + +def main(method, parser): + args = parser.parse_args() + try: + method(args) + except AssertionError as msg: + print(msg) diff --git a/cvp/update/test.yml b/cvp/update/test.yml new file mode 100644 index 00000000..943105c5 --- /dev/null +++ b/cvp/update/test.yml @@ -0,0 +1,12 @@ +--- +- hosts: "{{ host }}" + remote_user: "{{ user }}" + become: "yes" + become_method: sudo + vars: + user: "root" + tasks: + - name: test connectivity + command: "echo hello {{ host }}" + register: result + - debug: msg="{{ result }}" diff --git a/cvp/update/update.yml b/cvp/update/update.yml new file mode 100644 index 00000000..18b75b6b --- /dev/null +++ b/cvp/update/update.yml @@ -0,0 +1,50 @@ +--- +- hosts: "{{ host }}" + remote_user: "{{ user }}" + become: "yes" + become_method: sudo + vars: + user: "root" + port: "8000" + update_path: "/tmp/testapi" + image: "opnfv/testapi" + mode: "pull" + mongodb_url: "mongodb://172.17.0.1:27017" + swagger_url: "http://{{ host }}:{{ port }}" + tasks: + - name: create temporary update directory + file: + path: "{{ update_path }}" + state: directory + - name: transfer files in templates + copy: + src: templates/ + dest: "{{ update_path }}" + - name: transfer Dockerfile + copy: + src: ../docker/Dockerfile + dest: "{{ update_path }}" + when: mode == "build" + - name: backup mongodb database + command: "python {{ update_path }}/backup_mongodb.py -u {{ mongodb_url }} -o {{ update_path }}" + - name: stop and remove old versions + command: bash "{{ update_path }}/rm_olds.sh" + register: rm_result + - debug: msg="{{ rm_result.stderr }}" + - name: delete old docker images + command: bash "{{ update_path }}/rm_images.sh" + ignore_errors: true + - name: update mongodb + command: "python {{ update_path }}/update_mongodb.py -u {{ mongodb_url }}" + - name: docker build image + command: "docker build -t {{ image }} {{ update_path }}" + when: mode == "build" + - name: docker start testapi server + command: docker run -dti -p "{{ port }}:8000" + -e "mongodb_url={{ mongodb_url }}" + -e "swagger_url={{ swagger_url }}" + "{{ image }}" + - name: remove temporary update directory + file: + path: "{{ update_path }}" + state: absent diff --git a/cvp/update/update_api.py b/cvp/update/update_api.py new file mode 100644 index 00000000..db8ad2d9 --- /dev/null +++ b/cvp/update/update_api.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corporation +# feng.xiaowei@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import subprocess + +possible_processes = [ + 'result_collection_api', + 'opnfv-testapi' +] + + +def kill_olds(): + for proc in possible_processes: + query = 'ps -ef | grep {} | grep -v grep'.format(proc) + runnings = execute_with_output(query) + if runnings: + for running in runnings: + kill = 'kill -kill ' + running.split()[1] + execute_with_output(kill) + runnings = execute_with_output(query) + assert len(runnings) == 0, 'kill %s failed'.format(proc) + + +def install_dependencies(): + execute_with_assert('pip install -r ../requirements.txt') + + +def install_new(): + execute_with_assert('cd ../ && python setup.py install') + + +def run_new(): + execute_with_assert('opnfv-testapi &') + + +def execute_with_output(cmd): + return subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE).stdout.readlines() + + +def execute_with_assert(cmd): + execute_output = subprocess.call(cmd, shell=True) + assert execute_output == 0 + + +if __name__ == '__main__': + kill_olds() + install_dependencies() + install_new() + run_new() |