diff options
Diffstat (limited to 'api')
| -rw-r--r-- | api/actions/env.py | 255 | ||||
| -rw-r--r-- | api/actions/result.py | 26 | ||||
| -rw-r--r-- | api/actions/samples.py | 37 | ||||
| -rw-r--r-- | api/actions/test.py | 14 | ||||
| -rwxr-xr-x | api/api-prepare.sh | 66 | ||||
| -rw-r--r-- | api/conf.py | 2 | ||||
| -rw-r--r-- | api/server.py | 7 | ||||
| -rw-r--r-- | api/swagger/__init__.py | 0 | ||||
| -rw-r--r-- | api/swagger/docs/results.yaml | 41 | ||||
| -rw-r--r-- | api/swagger/docs/testcases.yaml | 50 | ||||
| -rw-r--r-- | api/swagger/models.py | 51 | ||||
| -rw-r--r-- | api/urls.py | 6 | ||||
| -rw-r--r-- | api/utils/common.py | 9 | ||||
| -rw-r--r-- | api/views.py | 44 | ||||
| -rw-r--r-- | api/yardstick.ini | 2 |
15 files changed, 542 insertions, 68 deletions
diff --git a/api/actions/env.py b/api/actions/env.py new file mode 100644 index 000000000..9e53dde4d --- /dev/null +++ b/api/actions/env.py @@ -0,0 +1,255 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import threading +import subprocess +import time +import json +import os +import errno + +from docker import Client + +from yardstick.common import constants as config +from yardstick.common import utils as yardstick_utils +from yardstick.common.httpClient import HttpClient +from api import conf as api_conf +from api.utils import influx +from api.utils.common import result_handler + +logger = logging.getLogger(__name__) + + +def createGrafanaContainer(args): + thread = threading.Thread(target=_create_grafana) + thread.start() + return result_handler('success', []) + + +def _create_grafana(): + client = Client(base_url=config.DOCKER_URL) + + try: + if not _check_image_exist(client, '%s:%s' % (config.GRAFANA_IMAGE, + config.GRAFANA_TAGS)): + client.pull(config.GRAFANA_IMAGE, config.GRAFANA_TAGS) + + _create_grafana_container(client) + + time.sleep(5) + + _create_data_source() + + _create_dashboard() + except Exception as e: + logger.debug('Error: %s', e) + + +def _create_dashboard(): + url = 'http://admin:admin@%s:3000/api/dashboards/db' % api_conf.GATEWAY_IP + data = json.load(file('../dashboard/ping_dashboard.json')) + HttpClient().post(url, data) + + +def _create_data_source(): + url = 'http://admin:admin@%s:3000/api/datasources' % api_conf.GATEWAY_IP + data = { + "name": "yardstick", + "type": "influxdb", + "access": "proxy", + "url": "http://%s:8086" % api_conf.GATEWAY_IP, + "password": "root", + "user": "root", + "database": "yardstick", + "basicAuth": True, + "basicAuthUser": "admin", + "basicAuthPassword": "admin", + "isDefault": False, + } + HttpClient().post(url, data) + + +def _create_grafana_container(client): + ports = [3000] + port_bindings = {k: k for k in ports} + host_config = client.create_host_config(port_bindings=port_bindings) + + container = client.create_container(image='%s:%s' % (config.GRAFANA_IMAGE, + config.GRAFANA_TAGS), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + client.start(container) + + +def _check_image_exist(client, t): + return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags']) + + +def createInfluxDBContainer(args): + thread = threading.Thread(target=_create_influxdb) + thread.start() + return result_handler('success', []) + + +def _create_influxdb(): + client = Client(base_url=config.DOCKER_URL) + + try: + _config_output_file() + + if not _check_image_exist(client, '%s:%s' % (config.INFLUXDB_IMAGE, + config.INFLUXDB_TAG)): + client.pull(config.INFLUXDB_IMAGE, tag=config.INFLUXDB_TAG) + + _create_influxdb_container(client) + + time.sleep(5) + + _config_influxdb() + except Exception as e: + logger.debug('Error: %s', e) + + +def _create_influxdb_container(client): + + ports = [8083, 8086] + port_bindings = {k: k for k in ports} + host_config = client.create_host_config(port_bindings=port_bindings) + + container = client.create_container(image='%s:%s' % (config.INFLUXDB_IMAGE, + config.INFLUXDB_TAG), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + client.start(container) + + +def _config_influxdb(): + try: + client = influx.get_data_db_client() + client.create_user(config.USER, config.PASSWORD, config.DATABASE) + client.create_database(config.DATABASE) + logger.info('Success to config influxDB') + except Exception as e: + logger.debug('Failed to config influxDB: %s', e) + + +def _config_output_file(): + yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + with open(config.YARDSTICK_CONFIG_FILE, 'w') as f: + f.write("""\ +[DEFAULT] +debug = False +dispatcher = influxdb + +[dispatcher_file] +file_path = /tmp/yardstick.out + +[dispatcher_http] +timeout = 5 +# target = http://127.0.0.1:8000/results + +[dispatcher_influxdb] +timeout = 5 +target = http://%s:8086 +db_name = yardstick +username = root +password = root +""" + % api_conf.GATEWAY_IP) + + +def prepareYardstickEnv(args): + thread = threading.Thread(target=_prepare_env_daemon) + thread.start() + return result_handler('success', []) + + +def _prepare_env_daemon(): + + installer_ip = os.environ.get('INSTALLER_IP', 'undefined') + installer_type = os.environ.get('INSTALLER_TYPE', 'undefined') + + _check_variables(installer_ip, installer_type) + + _create_directories() + + rc_file = config.OPENSTACK_RC_FILE + + _get_remote_rc_file(rc_file, installer_ip, installer_type) + + _source_file(rc_file) + + _append_external_network(rc_file) + + _load_images() + + +def _check_variables(installer_ip, installer_type): + + if installer_ip == 'undefined': + raise SystemExit('Missing INSTALLER_IP') + + if installer_type == 'undefined': + raise SystemExit('Missing INSTALLER_TYPE') + elif installer_type not in config.INSTALLERS: + raise SystemExit('INSTALLER_TYPE is not correct') + + +def _create_directories(): + yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR) + + +def _source_file(rc_file): + yardstick_utils.source_env(rc_file) + + +def _get_remote_rc_file(rc_file, installer_ip, installer_type): + + os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT) + + try: + cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, + '-a', installer_ip] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p.communicate()[0] + + if p.returncode != 0: + logger.debug('Failed to fetch credentials from installer') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def _append_external_network(rc_file): + neutron_client = yardstick_utils.get_neutron_client() + networks = neutron_client.list_networks()['networks'] + try: + ext_network = next(n['name'] for n in networks if n['router:external']) + except StopIteration: + logger.warning("Can't find external network") + else: + cmd = 'export EXTERNAL_NETWORK=%s' % ext_network + try: + with open(rc_file, 'a') as f: + f.write(cmd + '\n') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def _load_images(): + cmd = [config.LOAD_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=config.YARDSTICK_REPOS_DIR) + output = p.communicate()[0] + logger.debug('The result is: %s', output) diff --git a/api/actions/result.py b/api/actions/result.py index 9f606d2cb..1f200fbcc 100644 --- a/api/actions/result.py +++ b/api/actions/result.py @@ -7,6 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import logging +import uuid +import re from api.utils import influx as influx_utils from api.utils import common as common_utils @@ -19,23 +21,29 @@ def getResult(args): try: measurement = args['measurement'] task_id = args['task_id'] + + if re.search("[^a-zA-Z0-9_-]", measurement): + raise ValueError('invalid measurement parameter') + + uuid.UUID(task_id) except KeyError: - message = 'measurement and task_id must be needed' + message = 'measurement and task_id must be provided' return common_utils.error_handler(message) - measurement = conf.TEST_CASE_PRE + measurement - - query_sql = "select * from $table where task_id='$task_id'" - param = {'table': 'tasklist', 'task_id': task_id} - data = common_utils.translate_to_str(influx_utils.query(query_sql, param)) + query_template = "select * from %s where task_id='%s'" + query_sql = query_template % ('tasklist', task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) def _unfinished(): return common_utils.result_handler(0, []) def _finished(): - param = {'table': measurement, 'task_id': task_id} - data = common_utils.translate_to_str(influx_utils.query(query_sql, - param)) + query_sql = query_template % (conf.TEST_CASE_PRE + measurement, + task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) + if not data: + query_sql = query_template % (measurement, task_id) + data = common_utils.translate_to_str(influx_utils.query(query_sql)) return common_utils.result_handler(1, data) diff --git a/api/actions/samples.py b/api/actions/samples.py new file mode 100644 index 000000000..545447aec --- /dev/null +++ b/api/actions/samples.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import os +import logging + +from api import conf +from api.utils import common as common_utils + +logger = logging.getLogger(__name__) + + +def runTestCase(args): + try: + opts = args.get('opts', {}) + testcase = args['testcase'] + except KeyError: + return common_utils.error_handler('Lack of testcase argument') + + testcase = os.path.join(conf.SAMPLE_PATH, testcase + '.yaml') + + task_id = str(uuid.uuid4()) + + command_list = ['task', 'start'] + command_list = common_utils.get_command_list(command_list, opts, testcase) + logger.debug('The command_list is: %s', command_list) + + logger.debug('Start to execute command list') + common_utils.exec_command_task(command_list, task_id) + + return common_utils.result_handler('success', task_id) diff --git a/api/actions/test.py b/api/actions/test.py index b1dc212c2..fda0ffd32 100644 --- a/api/actions/test.py +++ b/api/actions/test.py @@ -7,7 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import uuid -import json import os import logging @@ -22,12 +21,7 @@ def runTestCase(args): opts = args.get('opts', {}) testcase = args['testcase'] except KeyError: - logger.error('Lack of testcase argument') - result = { - 'status': 'error', - 'message': 'need testcase name' - } - return json.dumps(result) + return common_utils.error_handler('Lack of testcase argument') testcase = os.path.join(conf.TEST_CASE_PATH, conf.TEST_CASE_PRE + testcase + '.yaml') @@ -41,8 +35,4 @@ def runTestCase(args): logger.debug('Start to execute command list') common_utils.exec_command_task(command_list, task_id) - result = { - 'status': 'success', - 'task_id': task_id - } - return json.dumps(result) + return common_utils.result_handler('success', task_id) diff --git a/api/api-prepare.sh b/api/api-prepare.sh index 075d7875c..fade8ccc6 100755 --- a/api/api-prepare.sh +++ b/api/api-prepare.sh @@ -1,41 +1,19 @@ #!/bin/bash - -# yardstick output config -output_config='/etc/yardstick/yardstick.conf' - -if [[ ! -e "${output_config}" ]];then - gateway_ip=$(ip route | grep default | awk '{print $3}') - echo "${gateway_ip}" - - install -d /etc/yardstick -m 0755 -o root - - cat << EOF > "${output_config}" -[DEFAULT] -debug = True -dispatcher = influxdb - -[dispatcher_file] -file_path = /tmp/yardstick.out - -[dispatcher_http] -timeout = 5 -# target = http://127.0.0.1:8000/results - -[dispatcher_influxdb] -timeout = 5 -target = http://${gateway_ip}:8086 -db_name = yardstick -username = root -password = root -EOF -fi +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## # nginx config nginx_config='/etc/nginx/conf.d/yardstick.conf' if [[ ! -e "${nginx_config}" ]];then - cat << EOF >> "${nginx_config}" + cat << EOF > "${nginx_config}" server { listen 5000; server_name localhost; @@ -46,14 +24,26 @@ server { } } EOF +echo "daemon off;" >> /etc/nginx/nginx.conf fi # nginx service start when boot -cat << EOF >> /root/.bashrc - -nginx_status=\$(service nginx status | grep not) -if [ -n "\${nginx_status}" ];then - service nginx restart - uwsgi -i /home/opnfv/repos/yardstick/api/yardstick.ini -fi +supervisor_config='/etc/supervisor/conf.d/yardstick.conf' + +if [[ ! -e "${supervisor_config}" ]];then + cat << EOF > "${supervisor_config}" +[supervisord] +nodaemon = true + +[program:yardstick_nginx] +user = root +command = service nginx restart +autorestart = true + +[program:yardstick_uwsgi] +user = root +directory = /home/opnfv/repos/yardstick/api +command = uwsgi -i yardstick.ini +autorestart = true EOF +fi diff --git a/api/conf.py b/api/conf.py index e1da4aba0..df44042b1 100644 --- a/api/conf.py +++ b/api/conf.py @@ -18,6 +18,8 @@ TEST_ACTION = ['runTestCase'] TEST_CASE_PATH = '../tests/opnfv/test_cases/' +SAMPLE_PATH = '../samples/' + TEST_CASE_PRE = 'opnfv_yardstick_' TEST_SUITE_PATH = '../tests/opnfv/test_suites/' diff --git a/api/server.py b/api/server.py index 1cbe1725d..64a2b4f96 100644 --- a/api/server.py +++ b/api/server.py @@ -10,18 +10,25 @@ import logging from flask import Flask from flask_restful import Api +from flasgger import Swagger from api.urls import urlpatterns +from yardstick import _init_logging logger = logging.getLogger(__name__) app = Flask(__name__) +Swagger(app) + api = Api(app) + reduce(lambda a, b: a.add_resource(b.resource, b.url, endpoint=b.endpoint) or a, urlpatterns, api) if __name__ == '__main__': + _init_logging() + logger.setLevel(logging.DEBUG) logger.info('Starting server') app.run(host='0.0.0.0') diff --git a/api/swagger/__init__.py b/api/swagger/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/swagger/__init__.py diff --git a/api/swagger/docs/results.yaml b/api/swagger/docs/results.yaml new file mode 100644 index 000000000..7bdab3eb6 --- /dev/null +++ b/api/swagger/docs/results.yaml @@ -0,0 +1,41 @@ +Query task result data + +This api offer the interface to get the result data via task_id +We will return a result json dict +--- +tags: + - Results +parameters: + - + in: query + name: action + type: string + default: getResult + required: true + - + in: query + name: measurement + type: string + description: test case name + required: true + - + in: query + name: task_id + type: string + description: the task_id you get before + required: true +responses: + 200: + description: a result json dict + schema: + id: ResultModel + properties: + status: + type: string + description: the status of the certain task + default: success + result: + schema: + type: array + items: + type: object diff --git a/api/swagger/docs/testcases.yaml b/api/swagger/docs/testcases.yaml new file mode 100644 index 000000000..7bfe5e647 --- /dev/null +++ b/api/swagger/docs/testcases.yaml @@ -0,0 +1,50 @@ +TestCases Actions
+
+This API may offer many actions, including runTestCase
+
+action: runTestCase
+This api offer the interface to run a test case in yardstick
+we will return a task_id for querying
+you can use the returned task_id to get the result data
+---
+tags:
+ - Release Action
+parameters:
+ - in: body
+ name: body
+ description: this is the input json dict
+ schema:
+ id: TestCaseActionModel
+ required:
+ - action
+ - args
+ properties:
+ action:
+ type: string
+ description: this is action for testcases
+ default: runTestCase
+ args:
+ schema:
+ id: TestCaseActionArgsModel
+ required:
+ - testcase
+ properties:
+ testcase:
+ type: string
+ description: this is the test case name
+ default: tc002
+ opts:
+ schema:
+ id: TestCaseActionArgsOptsModel
+responses:
+ 200:
+ description: A result json dict
+ schema:
+ id: result
+ properties:
+ status:
+ type: string
+ default: success
+ result:
+ type: string
+ description: task_id of this task
diff --git a/api/swagger/models.py b/api/swagger/models.py new file mode 100644 index 000000000..7c65fbbf5 --- /dev/null +++ b/api/swagger/models.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask_restful import fields +from flask_restful_swagger import swagger + + +# for testcases/action runTestCase action +@swagger.model +class TestCaseActionArgsOptsTaskArgModel: + resource_fields = { + } + + +@swagger.model +class TestCaseActionArgsOptsModel: + resource_fields = { + 'task-args': TestCaseActionArgsOptsTaskArgModel, + 'keep-deploy': fields.String, + 'suite': fields.String + } + + +@swagger.model +class TestCaseActionArgsModel: + resource_fields = { + 'testcase': fields.String, + 'opts': TestCaseActionArgsOptsModel + } + + +@swagger.model +class TestCaseActionModel: + resource_fields = { + 'action': fields.String, + 'args': TestCaseActionArgsModel + } + + +# for results +@swagger.model +class ResultModel: + resource_fields = { + 'status': fields.String, + 'result': fields.List + } diff --git a/api/urls.py b/api/urls.py index 2a9e72a76..50be91ead 100644 --- a/api/urls.py +++ b/api/urls.py @@ -11,6 +11,8 @@ from api.utils.common import Url urlpatterns = [ - Url('/yardstick/test/action', views.Test, 'test'), - Url('/yardstick/result/action', views.Result, 'result') + Url('/yardstick/testcases/release/action', views.Release, 'release'), + Url('/yardstick/testcases/samples/action', views.Samples, 'samples'), + Url('/yardstick/results', views.Results, 'results'), + Url('/yardstick/env/action', views.Env, 'env') ] diff --git a/api/utils/common.py b/api/utils/common.py index 04a6fe0d6..e3e64a72b 100644 --- a/api/utils/common.py +++ b/api/utils/common.py @@ -8,7 +8,8 @@ ############################################################################## import collections import logging -import json + +from flask import jsonify from api.utils.daemonthread import DaemonThread from yardstick.cmd.cli import YardstickCLI @@ -34,7 +35,7 @@ def get_command_list(command_list, opts, args): task_args = opts.get('task-args', '') if task_args: - command_list.extend(['--task-args', task_args]) + command_list.extend(['--task-args', str(task_args)]) return command_list @@ -50,7 +51,7 @@ def error_handler(message): 'status': 'error', 'message': message } - return json.dumps(result) + return jsonify(result) def result_handler(status, data): @@ -58,7 +59,7 @@ def result_handler(status, data): 'status': status, 'result': data } - return json.dumps(result) + return jsonify(result) class Url(object): diff --git a/api/views.py b/api/views.py index e78389f5a..928d8e9eb 100644 --- a/api/views.py +++ b/api/views.py @@ -7,18 +7,30 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import logging +import os from flask import request from flask_restful import Resource +from flasgger.utils import swag_from from api.utils import common as common_utils +from api.swagger import models from api.actions import test as test_action +from api.actions import samples as samples_action from api.actions import result as result_action +from api.actions import env as env_action logger = logging.getLogger(__name__) -class Test(Resource): +TestCaseActionModel = models.TestCaseActionModel +TestCaseActionArgsModel = models.TestCaseActionArgsModel +TestCaseActionArgsOptsModel = models.TestCaseActionArgsOptsModel +TestCaseActionArgsOptsTaskArgModel = models.TestCaseActionArgsOptsTaskArgModel + + +class Release(Resource): + @swag_from(os.getcwd() + '/swagger/docs/testcases.yaml') def post(self): action = common_utils.translate_to_str(request.json.get('action', '')) args = common_utils.translate_to_str(request.json.get('args', {})) @@ -30,7 +42,23 @@ class Test(Resource): return common_utils.error_handler('Wrong action') -class Result(Resource): +class Samples(Resource): + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(samples_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') + + +ResultModel = models.ResultModel + + +class Results(Resource): + @swag_from(os.getcwd() + '/swagger/docs/results.yaml') def get(self): args = common_utils.translate_to_str(request.args) action = args.get('action', '') @@ -40,3 +68,15 @@ class Result(Resource): return getattr(result_action, action)(args) except AttributeError: return common_utils.error_handler('Wrong action') + + +class Env(Resource): + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + try: + return getattr(env_action, action)(args) + except AttributeError: + return common_utils.error_handler('Wrong action') diff --git a/api/yardstick.ini b/api/yardstick.ini index 535022960..01025c2ef 100644 --- a/api/yardstick.ini +++ b/api/yardstick.ini @@ -12,5 +12,5 @@ chmod-socket = 666 callable = app enable-threads = true close-on-exec = 1 -daemonize=/home/kklt/kklt/api/uwsgi.log +daemonize=/home/opnfv/repos/yardstick/api/uwsgi.log socket = /home/opnfv/repos/yardstick/api/yardstick.sock |