diff options
Diffstat (limited to 'cvp/update')
| -rw-r--r-- | cvp/update/README.md | 100 | ||||
| -rw-r--r-- | cvp/update/__init__.py | 8 | ||||
| -rwxr-xr-x | cvp/update/playbook-update.sh | 90 | ||||
| -rw-r--r-- | cvp/update/templates/__init__.py | 0 | ||||
| -rw-r--r-- | cvp/update/templates/backup_mongodb.py | 45 | ||||
| -rw-r--r-- | cvp/update/templates/changes_in_mongodb.py | 51 | ||||
| -rw-r--r-- | cvp/update/templates/restore_mongodb.py | 38 | ||||
| -rwxr-xr-x | cvp/update/templates/rm_images.sh | 8 | ||||
| -rw-r--r-- | cvp/update/templates/rm_olds.sh | 15 | ||||
| -rw-r--r-- | cvp/update/templates/update_mongodb.py | 90 | ||||
| -rw-r--r-- | cvp/update/templates/utils.py | 48 | ||||
| -rw-r--r-- | cvp/update/test.yml | 12 | ||||
| -rw-r--r-- | cvp/update/update.yml | 50 | ||||
| -rw-r--r-- | cvp/update/update_api.py | 55 |
14 files changed, 0 insertions, 610 deletions
diff --git a/cvp/update/README.md b/cvp/update/README.md deleted file mode 100644 index cb0e67b3..00000000 --- a/cvp/update/README.md +++ /dev/null @@ -1,100 +0,0 @@ -Welcome to TESTAPI Update! -======================== - - -This file is used to describe how testapi update works - ----------- -How to use ---------------- - -#### <i class="icon-file"></i> backup mongodb - -arguments: -: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ -the backup output will be put under dir/db__XXXX_XX_XX_XXXXXX/db --d/--db: database for the backup, default = test_results_collection - -usage: -``` -python backup_mongodb.py -``` - -#### <i class="icon-file"></i> restore mongodb - -arguments: -: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ - -i/--input_dir: Input directory for the Restore, must be specified, - the restore input must be specified to dir/db__XXXX_XX_XX_XXXXXX/db - -d/--db: database name after the restore, default = basename of input_dir - -usage: -``` -python restore_mongodb.py -``` -#### <i class="icon-file"></i> update mongodb - - arguments: -: -u/--url: Mongo DB URL, default = mongodb://127.0.0.1:27017/ - -d/--db: database name to be updated, default = test_results_collection - -changes need to be done: -change collection name, modify changes.collections_old2New - > collections_old2New = { - 'old_collection': 'new_collection', - } - - change field name, modify changes.fields_old2New - > fields_old2New = { - 'collection': [(query, {'old_field': 'new_field'})] - } - - change the doc, modify changes.docs_old2New - > docs_old2New = { - 'test_results': [ - ({'field': 'old_value'}, {'field': 'new_value'}), - (query, {'field': 'new_value'}), - ] - } - -#### <i class="icon-file"></i> update opnfv-testapi process -This script must be run right in this directory and remember to -change ../etc/config.ini before running this script. - -operations includes: -: kill running test_collection_api & opnfv-testapi -install or update dependencies according to ../requirements.txt -install opnfv-testapi -run opnfv-testapi - -usage: -``` -python update_api.py -``` -#### <i class="icon-file"></i> update opnfv/testapi container -Here ansible-playbook is used to implement auto update. -Please make sure that the remote server is accessible via ssh. - -install ansible, please refer: -``` -http://docs.ansible.com/ansible/intro_installation.html -``` - -playbook-update.sh - -arguments: -: -h|--help show this help text --r|--remote remote server --u|--user ssh username used to access to remote server --i|--identity ssh PublicKey file used to access to remote server --e|--execute execute update, if not set just check the ansible connectivity - -usage: -``` -ssh-agent ./playbook-update.sh -r testresults.opnfv.org -u serena -i ~/.ssh/id_rsa -e -``` - -> **Note:** - -> - If documents need to be changed, please modify file -templates/changes_in_mongodb.py, and refer section **update mongodb** diff --git a/cvp/update/__init__.py b/cvp/update/__init__.py deleted file mode 100644 index 363bc388..00000000 --- a/cvp/update/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/cvp/update/playbook-update.sh b/cvp/update/playbook-update.sh deleted file mode 100755 index 86d30e4b..00000000 --- a/cvp/update/playbook-update.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# -# Author: Serena Feng (feng.xiaoewi@zte.com.cn) -# Update testapi on remote server using ansible playbook automatically -# -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# - -set -e - -usage="Script to trigger update automatically. - -usage: - bash $(basename "$0") [-h|--help] [-h <host>] [-u username] [-i identityfile] [-e|--execute] - -where: - -h|--help show this help text - -r|--remote remote server - -u|--user ssh username used to access to remote server - -i|--identity ssh PublicKey file used to access to remote server - -e|--execute execute update, if not set just check the ansible connectivity" - -remote=testresults.opnfv.org -user=root -identity=~/.ssh/id_rsa -hosts=./hosts -execute=false - -# Parse parameters -while [[ $# > 0 ]] - do - key="$1" - case $key in - -h|--help) - echo "$usage" - exit 0 - shift - ;; - -r|--remote) - remote="$2" - shift - ;; - -u|--user) - user="$2" - shift - ;; - -i|--identity) - identity="$2" - shift - ;; - -e|--execute) - execute=true - ;; - *) - echo "unknown option" - exit 1 - ;; - esac - shift # past argument or value -done - -echo $remote > $hosts - -echo "add authentication" -ssh-add $identity - -echo "test ansible connectivity" -ansible -i ./hosts $remote -m ping -u $user - -echo "test playbook connectivity" -ansible-playbook -i $hosts test.yml -e "host=$remote user=$user" - -if [ $execute == true ]; then - echo "do update" - ansible-playbook -i $hosts update.yml -e "host=$remote \ - user=$user \ - port=8082 \ - image=opnfv/testapi \ - update_path=/home/$user/testapi \ - mongodb_url=mongodb://172.17.0.1:27017 \ - swagger_url=http://testresults.opnfv.org/test" -fi - -rm -fr $hosts -ssh-agent -k diff --git a/cvp/update/templates/__init__.py b/cvp/update/templates/__init__.py deleted file mode 100644 index e69de29b..00000000 --- a/cvp/update/templates/__init__.py +++ /dev/null diff --git a/cvp/update/templates/backup_mongodb.py b/cvp/update/templates/backup_mongodb.py deleted file mode 100644 index 9c243776..00000000 --- a/cvp/update/templates/backup_mongodb.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import argparse -import datetime -import os - -from utils import execute, main, get_abspath - -parser = argparse.ArgumentParser(description='Backup MongoDBs') - -parser.add_argument('-u', '--url', - type=str, - required=False, - default='mongodb://127.0.0.1:27017/', - help='Mongo DB URL for Backups') -parser.add_argument('-o', '--output_dir', - type=str, - required=False, - default='./', - help='Output directory for the backup.') - -parser.add_argument('-d', '--db', - type=str, - required=False, - default='test_results_collection', - help='database for the backup.') - - -def backup(args): - db = args.db - out = get_abspath(args.output_dir) - now = datetime.datetime.now() - out = os.path.join(out, '%s__%s' % (db, now.strftime('%Y_%m_%d_%H%M%S'))) - cmd = ['mongodump', '-o', '%s' % out] - execute(cmd, args) - - -if __name__ == '__main__': - main(backup, parser) diff --git a/cvp/update/templates/changes_in_mongodb.py b/cvp/update/templates/changes_in_mongodb.py deleted file mode 100644 index 1a4d5a16..00000000 --- a/cvp/update/templates/changes_in_mongodb.py +++ /dev/null @@ -1,51 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# 09/06/2016: change for migration after refactoring -# 16/06/2016: Alignment of test name (JIRA: FUNCTEST-304) -############################################################################## -collections_old2New = { - # 'pod': 'pods', - # 'test_projects': 'projects', - # 'test_testcases': 'testcases', - # 'test_results': 'results' -} - -fields_old2New = { - # 'test_results': [({}, {'creation_date': 'start_date'})] -} - -docs_old2New = { - # 'test_results': [ - # ({'criteria': 'failed'}, {'criteria': 'FAILED'}), - # ({'criteria': 'passed'}, {'criteria': 'PASS'}) - # ] - # 'testcases': [ - # ({'name': 'vPing'}, {'name': 'vping_ssh'}), - # ({'name': 'Tempest'}, {'name': 'tempest_smoke_serial'}), - # ({'name': 'Rally'}, {'name': 'rally_sanity'}), - # ({'name': 'ODL'}, {'name': 'odl'}), - # ({'name': 'vIMS'}, {'name': 'vims'}), - # ({'name': 'ONOS'}, {'name': 'onos'}), - # ({'name': 'vPing_userdata'}, {'name': 'vping_userdata'}), - # ({'name': 'ovno'}, {'name': 'ocl'}) - # ], - # 'results': [ - # ({'case_name': 'vPing'}, {'case_name': 'vping_ssh'}), - # ({'case_name': 'Tempest'}, {'case_name': 'tempest_smoke_serial'}), - # ({'case_name': 'Rally'}, {'case_name': 'rally_sanity'}), - # ({'case_name': 'ODL'}, {'case_name': 'odl'}), - # ({'case_name': 'vIMS'}, {'case_name': 'vims'}), - # ({'case_name': 'ONOS'}, {'case_name': 'onos'}), - # ({'case_name': 'vPing_userdata'}, {'case_name': 'vping_userdata'}), - # ({'case_name': 'ovno'}, {'case_name': 'ocl'}) - # ] - 'results': [ - ({'trust_indicator': 0}, - {'trust_indicator': {'current': 0, 'histories': []}}) - ] -} diff --git a/cvp/update/templates/restore_mongodb.py b/cvp/update/templates/restore_mongodb.py deleted file mode 100644 index c45a0e62..00000000 --- a/cvp/update/templates/restore_mongodb.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import argparse - -from utils import execute, main, get_abspath - -parser = argparse.ArgumentParser(description='Restore MongoDBs') - -parser.add_argument('-u', '--url', - type=str, - required=False, - default='mongodb://127.0.0.1:27017/', - help='Mongo DB URL for Backup') -parser.add_argument('-i', '--input_dir', - type=str, - required=True, - help='Input directory for the Restore.') -parser.add_argument('-d', '--db', - type=str, - required=False, - default='test_results_collection', - help='database name after the restore.') - - -def restore(args): - input_dir = get_abspath(args.input_dir) - cmd = ['mongorestore', '%s' % input_dir] - execute(cmd, args) - - -if __name__ == '__main__': - main(restore, parser) diff --git a/cvp/update/templates/rm_images.sh b/cvp/update/templates/rm_images.sh deleted file mode 100755 index 6722573b..00000000 --- a/cvp/update/templates/rm_images.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -number=`docker images | awk 'NR != 1' | grep testapi | wc -l` -if [ $number -gt 0 ]; then - images=`docker images -a | awk 'NR != 1' | grep testapi | awk '{print $1}'` - echo "begin to rm images $images" - docker images | awk 'NR != 1' | grep testapi | awk '{print $3}' | xargs docker rmi -f &>/dev/null -fi diff --git a/cvp/update/templates/rm_olds.sh b/cvp/update/templates/rm_olds.sh deleted file mode 100644 index c6bca186..00000000 --- a/cvp/update/templates/rm_olds.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -proc_number=`ps -ef | grep opnfv-testapi | grep -v grep | wc -l` -if [ $proc_number -gt 0 ]; then - procs=`ps -ef | grep opnfv-testapi | grep -v grep` - echo "begin to kill opnfv-testapi $procs" - ps -ef | grep opnfv-testapi | grep -v grep | awk '{print $2}' | xargs kill -kill &>/dev/null -fi - -number=`docker ps -a | awk 'NR != 1' | grep testapi | wc -l` -if [ $number -gt 0 ]; then - containers=number=`docker ps -a | awk 'NR != 1' | grep testapi` - echo "begin to rm containers $containers" - docker ps -a | awk 'NR != 1' | grep testapi | awk '{print $1}' | xargs docker rm -f &>/dev/null -fi diff --git a/cvp/update/templates/update_mongodb.py b/cvp/update/templates/update_mongodb.py deleted file mode 100644 index f7595928..00000000 --- a/cvp/update/templates/update_mongodb.py +++ /dev/null @@ -1,90 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import argparse - -from pymongo import MongoClient - -from changes_in_mongodb import collections_old2New, \ - fields_old2New, docs_old2New -from utils import main, parse_mongodb_url - -parser = argparse.ArgumentParser(description='Update MongoDBs') - -parser.add_argument('-u', '--url', - type=str, - required=False, - default='mongodb://127.0.0.1:27017/', - help='Mongo DB URL for Backups') - -parser.add_argument('-d', '--db', - type=str, - required=False, - default='test_results_collection', - help='database for the update.') - - -def assert_collections(a_dict): - if a_dict is not None: - collections = eval_db('collection_names') - no_collections = [] - for collection in a_dict.keys(): - if collection not in collections: - no_collections.append(collection) - assert len(no_collections) == 0, \ - 'collections {} not exist'.format(no_collections) - - -def rename_collections(a_dict): - if a_dict is not None: - for collection, new_name in a_dict.iteritems(): - eval_collection(collection, 'rename', new_name) - - -def rename_fields(a_dict): - collection_update(a_dict, '$rename') - - -def change_docs(a_dict): - collection_update(a_dict, '$set') - - -def eval_db(method, *args, **kwargs): - exec_db = db.__getattribute__(method) - return exec_db(*args, **kwargs) - - -def eval_collection(collection, method, *args, **kwargs): - exec_collection = db.__getattr__(collection) - return exec_collection.__getattribute__(method)(*args, **kwargs) - - -def collection_update(a_dict, operator): - if a_dict is not None: - for collection, updates in a_dict.iteritems(): - for (query, doc) in updates: - doc_dict = {operator: doc} - eval_collection(collection, 'update', query, - doc_dict, upsert=False, multi=True) - - -def update(args): - parse_mongodb_url(args.url) - client = MongoClient(args.url) - global db - db = client[args.db] - assert_collections(docs_old2New) - assert_collections(fields_old2New) - assert_collections(collections_old2New) - change_docs(docs_old2New) - rename_fields(fields_old2New) - rename_collections(collections_old2New) - - -if __name__ == '__main__': - main(update, parser) diff --git a/cvp/update/templates/utils.py b/cvp/update/templates/utils.py deleted file mode 100644 index 4254fee3..00000000 --- a/cvp/update/templates/utils.py +++ /dev/null @@ -1,48 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import os -import urlparse -import subprocess - - -def get_abspath(path): - assert os.path.isdir(path), 'Path %s can\'t be found.' % path - return os.path.abspath(path) - - -def parse_mongodb_url(url): - url = urlparse.urlparse(url) - assert url.scheme == 'mongodb', 'URL must be a MongoDB URL' - return url - - -def url_parse(url): - url = parse_mongodb_url(url) - return url.username, url.password, url.hostname, url.port - - -def execute(cmd, args): - (username, password, hostname, port) = url_parse(args.url) - cmd.extend(['--host', '%s' % hostname, '--port', '%s' % port]) - db = args.db - if db is not None: - cmd.extend(['--db', '%s' % db]) - if username is not None: - cmd.extend(['-u', '%s' % username, '-p', '%s' % password]) - print('execute: %s' % cmd) - execute_output = subprocess.check_output(cmd) - print(execute_output) - - -def main(method, parser): - args = parser.parse_args() - try: - method(args) - except AssertionError as msg: - print(msg) diff --git a/cvp/update/test.yml b/cvp/update/test.yml deleted file mode 100644 index 943105c5..00000000 --- a/cvp/update/test.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: "{{ host }}" - remote_user: "{{ user }}" - become: "yes" - become_method: sudo - vars: - user: "root" - tasks: - - name: test connectivity - command: "echo hello {{ host }}" - register: result - - debug: msg="{{ result }}" diff --git a/cvp/update/update.yml b/cvp/update/update.yml deleted file mode 100644 index 18b75b6b..00000000 --- a/cvp/update/update.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -- hosts: "{{ host }}" - remote_user: "{{ user }}" - become: "yes" - become_method: sudo - vars: - user: "root" - port: "8000" - update_path: "/tmp/testapi" - image: "opnfv/testapi" - mode: "pull" - mongodb_url: "mongodb://172.17.0.1:27017" - swagger_url: "http://{{ host }}:{{ port }}" - tasks: - - name: create temporary update directory - file: - path: "{{ update_path }}" - state: directory - - name: transfer files in templates - copy: - src: templates/ - dest: "{{ update_path }}" - - name: transfer Dockerfile - copy: - src: ../docker/Dockerfile - dest: "{{ update_path }}" - when: mode == "build" - - name: backup mongodb database - command: "python {{ update_path }}/backup_mongodb.py -u {{ mongodb_url }} -o {{ update_path }}" - - name: stop and remove old versions - command: bash "{{ update_path }}/rm_olds.sh" - register: rm_result - - debug: msg="{{ rm_result.stderr }}" - - name: delete old docker images - command: bash "{{ update_path }}/rm_images.sh" - ignore_errors: true - - name: update mongodb - command: "python {{ update_path }}/update_mongodb.py -u {{ mongodb_url }}" - - name: docker build image - command: "docker build -t {{ image }} {{ update_path }}" - when: mode == "build" - - name: docker start testapi server - command: docker run -dti -p "{{ port }}:8000" - -e "mongodb_url={{ mongodb_url }}" - -e "swagger_url={{ swagger_url }}" - "{{ image }}" - - name: remove temporary update directory - file: - path: "{{ update_path }}" - state: absent diff --git a/cvp/update/update_api.py b/cvp/update/update_api.py deleted file mode 100644 index db8ad2d9..00000000 --- a/cvp/update/update_api.py +++ /dev/null @@ -1,55 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation -# feng.xiaowei@zte.com.cn -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import subprocess - -possible_processes = [ - 'result_collection_api', - 'opnfv-testapi' -] - - -def kill_olds(): - for proc in possible_processes: - query = 'ps -ef | grep {} | grep -v grep'.format(proc) - runnings = execute_with_output(query) - if runnings: - for running in runnings: - kill = 'kill -kill ' + running.split()[1] - execute_with_output(kill) - runnings = execute_with_output(query) - assert len(runnings) == 0, 'kill %s failed'.format(proc) - - -def install_dependencies(): - execute_with_assert('pip install -r ../requirements.txt') - - -def install_new(): - execute_with_assert('cd ../ && python setup.py install') - - -def run_new(): - execute_with_assert('opnfv-testapi &') - - -def execute_with_output(cmd): - return subprocess.Popen(cmd, shell=True, - stdout=subprocess.PIPE).stdout.readlines() - - -def execute_with_assert(cmd): - execute_output = subprocess.call(cmd, shell=True) - assert execute_output == 0 - - -if __name__ == '__main__': - kill_olds() - install_dependencies() - install_new() - run_new() |