diff options
| -rw-r--r-- | api/__init__.py | 0 | ||||
| -rw-r--r-- | api/actions/__init__.py | 0 | ||||
| -rw-r--r-- | api/actions/test.py | 40 | ||||
| -rw-r--r-- | api/conf.py | 17 | ||||
| -rw-r--r-- | api/server.py | 19 | ||||
| -rw-r--r-- | api/urls.py | 7 | ||||
| -rw-r--r-- | api/utils/__init__.py | 0 | ||||
| -rw-r--r-- | api/utils/common.py | 41 | ||||
| -rw-r--r-- | api/utils/daemonthread.py | 36 | ||||
| -rw-r--r-- | api/utils/influx.py | 55 | ||||
| -rw-r--r-- | api/views.py | 29 | ||||
| -rw-r--r-- | requirements.txt | 4 | ||||
| -rw-r--r-- | tests/unit/api/actions/test_test.py | 21 | ||||
| -rw-r--r-- | tests/unit/api/test_views.py | 24 | ||||
| -rw-r--r-- | tests/unit/api/utils/test_common.py | 57 | ||||
| -rw-r--r-- | tests/unit/api/utils/test_daemonthread.py | 29 | ||||
| -rw-r--r-- | tests/unit/api/utils/test_influx.py | 62 | ||||
| -rw-r--r-- | yardstick/cmd/cli.py | 8 | ||||
| -rw-r--r-- | yardstick/cmd/commands/task.py | 7 |
19 files changed, 449 insertions, 7 deletions
diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/__init__.py diff --git a/api/actions/__init__.py b/api/actions/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/actions/__init__.py diff --git a/api/actions/test.py b/api/actions/test.py new file mode 100644 index 000000000..0de70bb71 --- /dev/null +++ b/api/actions/test.py @@ -0,0 +1,40 @@ +import uuid +import json +import os +import logging + +from api import conf +from api.utils import common as common_utils + +logger = logging.getLogger(__name__) + + +def runTestCase(args): + try: + opts = args.get('opts', {}) + testcase = args['testcase'] + except KeyError: + logger.error('Lack of testcase argument') + result = { + 'status': 'error', + 'message': 'need testcase name' + } + return json.dumps(result) + + testcase = os.path.join(conf.TEST_CASE_PATH, + conf.TEST_CASE_PRE + testcase + '.yaml') + + task_id = str(uuid.uuid4()) + + command_list = ['task', 'start'] + command_list = common_utils.get_command_list(command_list, opts, testcase) + logger.debug('The command_list is: %s', command_list) + + logger.debug('Start to execute command list') + common_utils.exec_command_task(command_list, task_id) + + result = { + 'status': 'success', + 'task_id': task_id + } + return json.dumps(result) diff --git a/api/conf.py b/api/conf.py new file mode 100644 index 000000000..b5553f452 --- /dev/null +++ b/api/conf.py @@ -0,0 +1,17 @@ +from pyroute2 import IPDB + + +# configuration for influxdb +with IPDB() as ip: + GATEWAY_IP = ip.routes['default'].gateway +PORT = 8086 + +TEST_ACTION = ['runTestCase'] + +TEST_CASE_PATH = '../tests/opnfv/test_cases/' + +TEST_CASE_PRE = 'opnfv_yardstick_' + +TEST_SUITE_PATH = '../tests/opnfv/test_suites/' + +OUTPUT_CONFIG_FILE_PATH = '/etc/yardstick/yardstick.conf' diff --git a/api/server.py b/api/server.py new file mode 100644 index 000000000..d0e4d30a2 --- /dev/null +++ b/api/server.py @@ -0,0 +1,19 @@ +import logging + +from flask import Flask +from flask_restful import Api + +from api.urls import urlpatterns + +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +api = Api(app) + +reduce(lambda a, b: a.add_resource(b.resource, b.url, + endpoint=b.endpoint) or a, urlpatterns, api) + +if __name__ == '__main__': + logger.info('Starting server') + app.run(host='0.0.0.0') diff --git a/api/urls.py b/api/urls.py new file mode 100644 index 000000000..9fa0bc935 --- /dev/null +++ b/api/urls.py @@ -0,0 +1,7 @@ +from api import views +from api.utils.common import Url + + +urlpatterns = [ + Url('/yardstick/test/action', views.Test, 'test') +] diff --git a/api/utils/__init__.py b/api/utils/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/utils/__init__.py diff --git a/api/utils/common.py b/api/utils/common.py new file mode 100644 index 000000000..9d7998abd --- /dev/null +++ b/api/utils/common.py @@ -0,0 +1,41 @@ +import collections + +from api.utils.daemonthread import DaemonThread +from yardstick.cmd.cli import YardstickCLI + + +def translate_to_str(object): + if isinstance(object, collections.Mapping): + return {str(k): translate_to_str(v) for k, v in object.items()} + elif isinstance(object, list): + return [translate_to_str(ele) for ele in object] + elif isinstance(object, unicode): + return str(object) + return object + + +def get_command_list(command_list, opts, args): + + command_list.append(args) + + command_list.extend(('--{}'.format(k) for k in opts if 'task-args' != k)) + + task_args = opts.get('task_args', '') + if task_args: + command_list.extend(['--task-args', task_args]) + + return command_list + + +def exec_command_task(command_list, task_id): # pragma: no cover + daemonthread = DaemonThread(YardstickCLI().api, (command_list, task_id)) + daemonthread.start() + + +class Url(object): + + def __init__(self, url, resource, endpoint): + super(Url, self).__init__() + self.url = url + self.resource = resource + self.endpoint = endpoint diff --git a/api/utils/daemonthread.py b/api/utils/daemonthread.py new file mode 100644 index 000000000..77a0f6ab7 --- /dev/null +++ b/api/utils/daemonthread.py @@ -0,0 +1,36 @@ +import threading +import os +import datetime +import errno + +from api import conf +from api.utils.influx import write_data_tasklist + + +class DaemonThread(threading.Thread): + + def __init__(self, method, args): + super(DaemonThread, self).__init__(target=method, args=args) + self.method = method + self.command_list = args[0] + self.task_id = args[1] + + def run(self): + timestamp = datetime.datetime.now() + + try: + write_data_tasklist(self.task_id, timestamp, 0) + self.method(self.command_list, self.task_id) + write_data_tasklist(self.task_id, timestamp, 1) + except Exception as e: + write_data_tasklist(self.task_id, timestamp, 2, error=str(e)) + finally: + _handle_testsuite_file(self.task_id) + + +def _handle_testsuite_file(task_id): + try: + os.remove(os.path.join(conf.TEST_SUITE_PATH, task_id + '.yaml')) + except OSError as e: + if e.errno != errno.ENOENT: + raise diff --git a/api/utils/influx.py b/api/utils/influx.py new file mode 100644 index 000000000..52a90b61c --- /dev/null +++ b/api/utils/influx.py @@ -0,0 +1,55 @@ +import logging +from urlparse import urlsplit + +from influxdb import InfluxDBClient +import ConfigParser + +from api import conf + +logger = logging.getLogger(__name__) + + +def get_data_db_client(): + parser = ConfigParser.ConfigParser() + try: + parser.read(conf.OUTPUT_CONFIG_FILE_PATH) + dispatcher = parser.get('DEFAULT', 'dispatcher') + + if 'influxdb' != dispatcher: + raise RuntimeError + + ip = _get_ip(parser.get('dispatcher_influxdb', 'target')) + username = parser.get('dispatcher_influxdb', 'username') + password = parser.get('dispatcher_influxdb', 'password') + db_name = parser.get('dispatcher_influxdb', 'db_name') + return InfluxDBClient(ip, conf.PORT, username, password, db_name) + except ConfigParser.NoOptionError: + logger.error('can not find the key') + raise + + +def _get_ip(url): + return urlsplit(url).netloc.split(':')[0] + + +def _write_data(measurement, field, timestamp, tags): + point = { + 'measurement': measurement, + 'fields': field, + 'time': timestamp, + 'tags': tags + } + + try: + client = get_data_db_client() + + logger.debug('Start to write data: %s', point) + client.write_points([point]) + except RuntimeError: + logger.debug('dispatcher is not influxdb') + + +def write_data_tasklist(task_id, timestamp, status, error=''): + field = {'status': status, 'error': error} + tags = {'task_id': task_id} + _write_data('tasklist', field, timestamp, tags) diff --git a/api/views.py b/api/views.py new file mode 100644 index 000000000..883091222 --- /dev/null +++ b/api/views.py @@ -0,0 +1,29 @@ +import json +import logging + +from flask import request +from flask_restful import Resource + +from api.utils import common as common_utils +from api.actions import test as test_action +from api import conf + +logger = logging.getLogger(__name__) + + +class Test(Resource): + def post(self): + action = common_utils.translate_to_str(request.json.get('action', '')) + args = common_utils.translate_to_str(request.json.get('args', {})) + logger.debug('Input args is: action: %s, args: %s', action, args) + + if action not in conf.TEST_ACTION: + logger.error('Wrong action') + result = { + 'status': 'error', + 'message': 'wrong action' + } + return json.dumps(result) + + method = getattr(test_action, action) + return method(args) diff --git a/requirements.txt b/requirements.txt index 4d1a16993..ab20c7541 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,3 +77,7 @@ unicodecsv==0.14.1 unittest2==1.1.0 warlock==1.2.0 wrapt==1.10.6 +flask==0.11.1 +flask-restful==0.3.5 +influxdb==3.0.0 +pyroute2==0.4.10 diff --git a/tests/unit/api/actions/test_test.py b/tests/unit/api/actions/test_test.py new file mode 100644 index 000000000..158062ff4 --- /dev/null +++ b/tests/unit/api/actions/test_test.py @@ -0,0 +1,21 @@ +import unittest +import json + +from api.actions import test + + +class RunTestCase(unittest.TestCase): + + def test_runTestCase_with_no_testcase_arg(self): + args = {} + output = json.loads(test.runTestCase(args)) + + self.assertEqual('error', output['status']) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/api/test_views.py b/tests/unit/api/test_views.py new file mode 100644 index 000000000..650ed569c --- /dev/null +++ b/tests/unit/api/test_views.py @@ -0,0 +1,24 @@ +import unittest +import mock +import json + +from api.views import Test + + +class TestTestCase(unittest.TestCase): + + @mock.patch('api.views.request') + def test_post(self, mock_request): + mock_request.json.get.side_effect = ['runTestSuite', {}] + + result = json.loads(Test().post()) + + self.assertEqual('error', result['status']) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/api/utils/test_common.py b/tests/unit/api/utils/test_common.py new file mode 100644 index 000000000..5d858a32c --- /dev/null +++ b/tests/unit/api/utils/test_common.py @@ -0,0 +1,57 @@ +import unittest + +from api.utils import common + + +class TranslateToStrTestCase(unittest.TestCase): + + def test_translate_to_str_unicode(self): + input_str = u'hello' + output_str = common.translate_to_str(input_str) + + result = 'hello' + self.assertEqual(result, output_str) + + def test_translate_to_str_dict_list_unicode(self): + input_str = { + u'hello': {u'hello': [u'world']} + } + output_str = common.translate_to_str(input_str) + + result = { + 'hello': {'hello': ['world']} + } + self.assertEqual(result, output_str) + + +class GetCommandListTestCase(unittest.TestCase): + + def test_get_command_list_no_opts(self): + command_list = ['a'] + opts = {} + args = 'b' + output_list = common.get_command_list(command_list, opts, args) + + result_list = ['a', 'b'] + self.assertEqual(result_list, output_list) + + def test_get_command_list_with_opts_args(self): + command_list = ['a'] + opts = { + 'b': 'c', + 'task-args': 'd' + } + args = 'e' + + output_list = common.get_command_list(command_list, opts, args) + + result_list = ['a', 'e', '--b', '--task-args', 'd'] + self.assertEqual(result_list, output_list) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/api/utils/test_daemonthread.py b/tests/unit/api/utils/test_daemonthread.py new file mode 100644 index 000000000..918f1f506 --- /dev/null +++ b/tests/unit/api/utils/test_daemonthread.py @@ -0,0 +1,29 @@ +import unittest +import mock + +from api.utils.daemonthread import DaemonThread + + +class DaemonThreadTestCase(unittest.TestCase): + + @mock.patch('api.utils.daemonthread.os') + def test_run(self, mock_os): + def func(common_list, task_id): + return task_id + + common_list = [] + task_id = '1234' + thread = DaemonThread(func, (common_list, task_id)) + thread.run() + + mock_os.path.exist.return_value = True + pre_path = '../tests/opnfv/test_suites/' + mock_os.remove.assert_called_with(pre_path + '1234.yaml') + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/api/utils/test_influx.py b/tests/unit/api/utils/test_influx.py new file mode 100644 index 000000000..5f1e2c36f --- /dev/null +++ b/tests/unit/api/utils/test_influx.py @@ -0,0 +1,62 @@ +import unittest +import mock +import uuid +import datetime + +from api.utils import influx + + +class GetDataDbClientTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx.ConfigParser') + def test_get_data_db_client_dispatcher_not_influxdb(self, mock_parser): + mock_parser.ConfigParser().get.return_value = 'file' + try: + influx.get_data_db_client() + except Exception, e: + self.assertIsInstance(e, RuntimeError) + + +class GetIpTestCase(unittest.TestCase): + + def test_get_url(self): + url = 'http://localhost:8086/hello' + output = influx._get_ip(url) + + result = 'localhost' + self.assertEqual(result, output) + + +class WriteDataTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx.get_data_db_client') + def test_write_data(self, mock_get_client): + measurement = 'tasklist' + field = {'status': 1} + timestamp = datetime.datetime.now() + tags = {'task_id': str(uuid.uuid4())} + + influx._write_data(measurement, field, timestamp, tags) + mock_get_client.assert_called_with() + + +class WriteDataTasklistTestCase(unittest.TestCase): + + @mock.patch('api.utils.influx._write_data') + def test_write_data_tasklist(self, mock_write_data): + task_id = str(uuid.uuid4()) + timestamp = datetime.datetime.now() + status = 1 + influx.write_data_tasklist(task_id, timestamp, status) + + field = {'status': status, 'error': ''} + tags = {'task_id': task_id} + mock_write_data.assert_called_with('tasklist', field, timestamp, tags) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/yardstick/cmd/cli.py b/yardstick/cmd/cli.py index f2406bf08..3896ce47c 100644 --- a/yardstick/cmd/cli.py +++ b/yardstick/cmd/cli.py @@ -137,11 +137,11 @@ class YardstickCLI(): func = CONF.category.func func(CONF.category) - def _dispath_func_task(self, task_id, timestamp): + def _dispath_func_task(self, task_id): # dispatch to category parser func = CONF.category.func - func(CONF.category, task_id=task_id, timestamp=timestamp) + func(CONF.category, task_id=task_id) def main(self, argv): # pragma: no cover '''run the command line interface''' @@ -153,7 +153,7 @@ class YardstickCLI(): self._dispath_func_notask() - def api(self, argv, task_id, timestamp): # pragma: no cover + def api(self, argv, task_id): # pragma: no cover '''run the api interface''' self._register_cli_opt() @@ -161,4 +161,4 @@ class YardstickCLI(): self._handle_global_opts() - self._dispath_func_task(task_id, timestamp) + self._dispath_func_task(task_id) diff --git a/yardstick/cmd/commands/task.py b/yardstick/cmd/commands/task.py index a10a2a8a3..47fb2ee60 100644 --- a/yardstick/cmd/commands/task.py +++ b/yardstick/cmd/commands/task.py @@ -56,6 +56,8 @@ class TaskCommands(object): atexit.register(atexit_handler) + self.task_id = kwargs.get('task_id', str(uuid.uuid4())) + total_start_time = time.time() parser = TaskParser(args.inputfile[0]) @@ -81,7 +83,7 @@ class TaskCommands(object): one_task_start_time = time.time() parser.path = task_files[i] scenarios, run_in_parallel, meet_precondition = parser.parse_task( - task_args[i], task_args_fnames[i]) + self.task_id, task_args[i], task_args_fnames[i]) if not meet_precondition: LOG.info("meet_precondition is %s, please check envrionment", @@ -232,7 +234,7 @@ class TaskParser(object): return valid_task_files, valid_task_args, valid_task_args_fnames - def parse_task(self, task_args=None, task_args_file=None): + def parse_task(self, task_id, task_args=None, task_args_file=None): '''parses the task file and return an context and scenario instances''' print "Parsing task config:", self.path @@ -291,7 +293,6 @@ class TaskParser(object): run_in_parallel = cfg.get("run_in_parallel", False) # add tc and task id for influxdb extended tags - task_id = str(uuid.uuid4()) for scenario in cfg["scenarios"]: task_name = os.path.splitext(os.path.basename(self.path))[0] scenario["tc"] = task_name |