diff options
| -rw-r--r-- | contrib/TOM/README | 4 | ||||
| -rw-r--r-- | docker/Dockerfile | 2 | ||||
| -rw-r--r-- | docs/testing/developer/design/cli.rst | 92 | ||||
| -rw-r--r-- | docs/testing/user/userguide/cli.rst | 38 | ||||
| -rw-r--r-- | opt/infra/roles/qtip/files/run_qtip_server.sh | 22 | ||||
| -rw-r--r-- | qtip/api/__main__.py | 9 | ||||
| -rw-r--r-- | qtip/api/controllers/common.py | 15 | ||||
| -rw-r--r-- | qtip/api/controllers/metric.py | 13 | ||||
| -rw-r--r-- | qtip/api/controllers/plan.py | 25 | ||||
| -rw-r--r-- | qtip/api/controllers/qpi.py | 21 | ||||
| -rw-r--r-- | qtip/api/swagger/swagger.yaml | 64 | ||||
| -rw-r--r-- | tests/unit/api/conftest.py | 22 | ||||
| -rw-r--r-- | tests/unit/api/metric_controller_test.py | 37 | ||||
| -rw-r--r-- | tests/unit/api/plan_controller_test.py | 49 | ||||
| -rw-r--r-- | tests/unit/api/qpi_controller_test.py | 43 |
15 files changed, 365 insertions, 91 deletions
diff --git a/contrib/TOM/README b/contrib/TOM/README new file mode 100644 index 00000000..706573ce --- /dev/null +++ b/contrib/TOM/README @@ -0,0 +1,4 @@ +https://wiki.opnfv.org/display/testing/R+post-processing+of+the+Yardstick+results + +This folder is created for TOM, a performance dataset processing tool, by Alassane Samba <alassane.samba@orange.com> + diff --git a/docker/Dockerfile b/docker/Dockerfile index a4a7e477..d609273d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -55,7 +55,7 @@ RUN mkdir -p $HOME/qtip/results RUN chmod 700 /root/.ssh #Config ansible -COPY ansible.cfg.default /root/.ansible.cfg +COPY ansible.cfg.default $HOME/.ansible.cfg #Cloning Repos RUN git config --global http.sslVerify false diff --git a/docs/testing/developer/design/cli.rst b/docs/testing/developer/design/cli.rst new file mode 100644 index 00000000..72d1fbaf --- /dev/null +++ b/docs/testing/developer/design/cli.rst @@ -0,0 +1,92 @@ +*************************** +QTIP Command Line Interface +*************************** + +Abstract +######## + +QTIP consists of different tools(metrics) to benchmark the NFVI. These metrics fall under different NFVI +subsystems(QPI's) such as compute, storage and network. A plan consists of one or more QPI's, depending upon how +the end user would want to measure performance. CLI is designed to help the user, execute benchmarks and +view respective scores. + +Framework +========= + +QTIP CLI has been created using the Python package `Click`_, Command Line Interface Creation Kit. It has been +chosen for number of reasons. It presents the user with a very simple yet powerful API to build complex +applications. One of the most striking features is command nesting. + +As explained, QTIP consists of metrics, QPI's and plans. CLI is designed to provide interface to all +these components. It is responsible for execution, as well as provide listing and details of each individual +element making up these components. + +Design +====== + +CLI's entry point extends Click's built in MultiCommand class object. It provides two methods, which are +overridden to provide custom configurations. + +.. code-block:: python + + class QtipCli(click.MultiCommand): + + def list_commands(self, ctx): + rv = [] + for filename in os.listdir(cmd_folder): + if filename.endswith('.py') and \ + filename.startswith('cmd_'): + rv.append(filename[4:-3]) + rv.sort() + return rv + + def get_command(self, ctx, name): + try: + if sys.version_info[0] == 2: + name = name.encode('ascii', 'replace') + mod = __import__('qtip.cli.commands.cmd_' + name, + None, None, ['cli']) + except ImportError: + return + return mod.cli + +Commands and subcommands will then be loaded by the ``get_command`` method above. + +Extending the Framework +======================= + +Framework can be easily extended, as per the users requirements. One such example can be to override the builtin +configurations with user defined ones. These can be written in a file, loaded via a Click Context and passed +through to all the commands. + +.. code-block:: python + + class Context: + + def __init__(): + + self.config = ConfigParser.ConfigParser() + self.config.read('path/to/configuration_file') + + def get_paths(): + + paths = self.config.get('section', 'path') + return paths + +The above example loads configuration from user defined paths, which then need to be provided to the actual +command definitions. + +.. code-block:: python + + from qtip.cli.entry import Context + + pass_context = click.make_pass_decorator(Context, ensure=False) + + @cli.command('list', help='List the Plans') + @pass_context + def list(ctx): + plans = Plan.list_all(ctx.paths()) + table = utils.table('Plans', plans) + click.echo(table) + +.. _Click: http://click.pocoo.org/5/ diff --git a/docs/testing/user/userguide/cli.rst b/docs/testing/user/userguide/cli.rst new file mode 100644 index 00000000..e18a36f9 --- /dev/null +++ b/docs/testing/user/userguide/cli.rst @@ -0,0 +1,38 @@ +************** +QTIP CLI Usage +************** + +QTIP consists of a number of benchmarking tools or metrics, grouped under QPI's. QPI's map to the different +components of a NFVI ecosystem, such as compute, network and storage. Depending on the type of application, +a user may group them under plans. + +QTIP CLI provides interface to all of the above the components. A help page provides a list of all the commands +along with a short description. +:: + + qtip [-h|--help] + +Typically a complete plan is executed at the +target environment. QTIP defaults to a number of sample plans. One may be able to list them using +:: + + qtip plan list + +One can also be able to view the details about a specific plan. +:: + + qtip plan show <plan_name> + +where *plan_name* is one of those listed from the previous command. + +To execute a complete plan +:: + + qtip plan run <plan_name> + +Similarly, the same commands can be used for the other two components making up the plans, i.e QPI's and metrics. + +Debug option helps identify the error by providing a detailed traceback. It can be enabled as +:: + + qtip [-d|--debug] plan run <plan_name> diff --git a/opt/infra/roles/qtip/files/run_qtip_server.sh b/opt/infra/roles/qtip/files/run_qtip_server.sh index 0f5cafea..75145e2b 100644 --- a/opt/infra/roles/qtip/files/run_qtip_server.sh +++ b/opt/infra/roles/qtip/files/run_qtip_server.sh @@ -1,4 +1,24 @@ #!/bin/bash envs="INSTALLER_TYPE=fuel -e INSTALLER_IP=10.20.0.2 -e NODE_NAME=zte-pod1" -docker run --name qtip -id -e $envs -p 5000:5000 opnfv/qtip + +# use ramfs to fix docker socket connection issue with overlay mode in centos +ramfs=/tmp/qtip/ramfs +if [ ! -d $ramfs ]; then + mkdir -p $ramfs +fi + +if [ ! -z $(df $ramfs | tail -n -1 | grep $ramfs) ]; then + sudo mount -t tmpfs -o size=32M tmpfs $ramfs +fi + +# enable contro path in docker +echo <<EOF > /tmp/ansible.cfg +[defaults] +callback_whitelist = profile_tasks +[ssh_connection] +control_path=/mnt/ramfs/ansible-ssh-%%h-%%p-%%r +EOF + +docker run --name qtip -id -e $envs -p 5000:5000 -v $ramfs:/mnt/ramfs opnfv/qtip +docker cp qtip /tmp/ansible.cfg /home/opnfv/.ansible.cfg diff --git a/qtip/api/__main__.py b/qtip/api/__main__.py index 05d92315..381622af 100644 --- a/qtip/api/__main__.py +++ b/qtip/api/__main__.py @@ -14,9 +14,14 @@ import os swagger_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'swagger/')) -def main(): +def get_app(): app = connexion.App(__name__, specification_dir=swagger_dir) - app.add_api('swagger.yaml', base_path='/v1.0') + app.add_api('swagger.yaml', base_path='/v1.0', strict_validation=True) + return app + + +def main(): + app = get_app() app.run(host="0.0.0.0", port=5000) diff --git a/qtip/api/controllers/common.py b/qtip/api/controllers/common.py index eeae0fee..6ddab7a9 100644 --- a/qtip/api/controllers/common.py +++ b/qtip/api/controllers/common.py @@ -14,15 +14,22 @@ import connexion from qtip.base import error -def get_one_exceptions(resource): +def check_endpoint_for_error(resource, operation=None): def _decorator(func): - def _execute(name): + def _execute(name=None): try: return func(name), httplib.OK except error.NotFoundError: return connexion.problem( httplib.NOT_FOUND, - '{} Not Found'.format(resource), - 'Requested {} `{}` not found.'.format(resource, name)) + '{} not found'.format(resource), + 'Requested {} `{}` not found.' + .format(resource.lower(), name)) + except error.ToBeDoneError: + return connexion.problem( + httplib.NOT_IMPLEMENTED, + '{} handler not implemented'.format(operation), + 'Requested operation `{}` on {} not implemented.' + .format(operation.lower(), resource.lower())) return _execute return _decorator diff --git a/qtip/api/controllers/metric.py b/qtip/api/controllers/metric.py index dd4c8ac6..96cd985c 100644 --- a/qtip/api/controllers/metric.py +++ b/qtip/api/controllers/metric.py @@ -14,13 +14,12 @@ from qtip.loader import metric def list_metrics(): - metric_list = list(metric.MetricSpec.list_all()) - return metric_list, httplib.OK + metrics = list(metric.MetricSpec.list_all()) + metrics_by_name = [m['name'] for m in metrics] + return {'metrics': metrics_by_name}, httplib.OK -@common.get_one_exceptions(resource='metric') +@common.check_endpoint_for_error(resource='Metric') def get_metric(name): - metric_spec = metric.MetricSpec(name) - return {'name': metric_spec.name, - 'abspath': metric_spec.abspath, - 'content': metric_spec.content} + metric_spec = metric.MetricSpec(name) + return metric_spec.content diff --git a/qtip/api/controllers/plan.py b/qtip/api/controllers/plan.py index 93836a32..00593878 100644 --- a/qtip/api/controllers/plan.py +++ b/qtip/api/controllers/plan.py @@ -9,30 +9,23 @@ import httplib -import connexion - +from qtip.api.controllers import common from qtip.base import error from qtip.loader import plan def list_plans(): - plan_list = list(plan.Plan.list_all()) - return plan_list, httplib.OK + plans = list(plan.Plan.list_all()) + plans_by_name = [p['name'] for p in plans] + return {'plans': plans_by_name}, httplib.OK +@common.check_endpoint_for_error(resource='Plan') def get_plan(name): - try: - plan_spec = plan.Plan(name) - return {'name': plan_spec.name, - 'abspath': plan_spec.abspath, - 'content': plan_spec.content}, httplib.OK - except error.NotFoundError: - return connexion.problem(httplib.NOT_FOUND, - 'Plan Not Found', - 'requested plan `' + name + '` not found.') + plan_spec = plan.Plan(name) + return plan_spec.content +@common.check_endpoint_for_error(resource='Plan', operation='Run') def run_plan(name, action="run"): - return connexion.problem(httplib.NOT_IMPLEMENTED, - 'Run a plan', - 'Plan runner not implemented') + raise error.ToBeDoneError('run_plan', 'plan') diff --git a/qtip/api/controllers/qpi.py b/qtip/api/controllers/qpi.py index 3c4dd718..af08a824 100644 --- a/qtip/api/controllers/qpi.py +++ b/qtip/api/controllers/qpi.py @@ -9,24 +9,17 @@ import httplib -import connexion - -from qtip.base import error +from qtip.api.controllers import common from qtip.loader import qpi def list_qpis(): - qpi_spec_list = list(qpi.QPISpec.list_all()) - return qpi_spec_list, httplib.OK + qpi_specs = list(qpi.QPISpec.list_all()) + qpis_by_name = [q['name'] for q in qpi_specs] + return {'qpis': qpis_by_name}, httplib.OK +@common.check_endpoint_for_error(resource='QPI') def get_qpi(name): - try: - qpi_spec = qpi.QPISpec(name) - return {'name': qpi_spec.name, - 'abspath': qpi_spec.abspath, - 'content': qpi_spec.content}, httplib.OK - except error.NotFoundError: - return connexion.problem(httplib.NOT_FOUND, - 'QPI Not Found', - 'Requested QPI Spec `' + name + '` not found.') + qpi_spec = qpi.QPISpec(name) + return qpi_spec.content diff --git a/qtip/api/swagger/swagger.yaml b/qtip/api/swagger/swagger.yaml index c2de2d68..51c3ebb8 100644 --- a/qtip/api/swagger/swagger.yaml +++ b/qtip/api/swagger/swagger.yaml @@ -212,7 +212,7 @@ paths: schema: $ref: '#/definitions/Error' definitions: - PlanContent: + Plan: type: object required: - name @@ -232,23 +232,13 @@ definitions: Plans: type: object required: - - name - - abspath + - plans properties: - name: - type: string - abspath: - type: string - Plan: - allOf: - - $ref: '#/definitions/Plans' - - type: object - - required: - - content - properties: - content: - $ref: '#/definitions/PlanContent' - MetricContent: + plans: + type: array + items: + type: string + Metric: type: object required: - name @@ -268,22 +258,13 @@ definitions: Metrics: type: object required: - - name - - abspath + - metrics properties: - name: - type: string - abspath: - type: string - Metric: - allOf: - - $ref: '#/definitions/Metrics' - - required: - - content - properties: - content: - $ref: '#/definitions/MetricContent' - QPIContent: + metrics: + type: array + items: + type: string + QPI: type: object required: - name @@ -301,21 +282,12 @@ definitions: QPIs: type: object required: - - name - - abspath + - qpis properties: - name: - type: string - abspath: - type: string - QPI: - allOf: - - $ref: '#/definitions/QPIs' - - required: - - content - properties: - content: - $ref: '#/definitions/QPIContent' + qpis: + type: array + items: + type: string Error: type: object properties: diff --git a/tests/unit/api/conftest.py b/tests/unit/api/conftest.py new file mode 100644 index 00000000..23a3be82 --- /dev/null +++ b/tests/unit/api/conftest.py @@ -0,0 +1,22 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import pytest + +from qtip.api import __main__ + + +@pytest.fixture(scope="session") +def app(): + return __main__.get_app().app + + +@pytest.fixture(scope="session") +def app_client(app): + return app.test_client() diff --git a/tests/unit/api/metric_controller_test.py b/tests/unit/api/metric_controller_test.py new file mode 100644 index 00000000..caba7972 --- /dev/null +++ b/tests/unit/api/metric_controller_test.py @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import httplib +import json + +from qtip.base.constant import BaseProp + + +def test_get_list_metrics(app_client): + response_success = app_client.get("/v1.0/metrics") + assert response_success.status_code == httplib.OK + metric_list = json.loads(response_success.data)['metrics'] + assert len(metric_list) > 0 + assert metric_list[0].endswith('.yaml') + + +def test_get_metric(app_client): + response_success = app_client.get("/v1.0/metrics/dpi.yaml") + assert response_success.status_code == httplib.OK + metric_data = json.loads(response_success.data) + assert BaseProp.NAME in metric_data + assert BaseProp.WORKLOADS in metric_data + assert isinstance(metric_data[BaseProp.WORKLOADS], list) + + +def test_get_metric_not_found(app_client): + response_not_found = app_client.get("/v1.0/metrics/fake.yaml") + response_data = json.loads(response_not_found.data) + assert response_not_found.status_code == httplib.NOT_FOUND + assert response_data['title'] == "Metric not found" diff --git a/tests/unit/api/plan_controller_test.py b/tests/unit/api/plan_controller_test.py new file mode 100644 index 00000000..136bd3c6 --- /dev/null +++ b/tests/unit/api/plan_controller_test.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import httplib +import json + + +from qtip.loader.plan import PlanProp + + +def test_invalid_url(app_client): + response_url_not_found = app_client.get("/v1.0/fakeresource") + assert response_url_not_found.status_code == httplib.NOT_FOUND + + +def test_get_list_plans(app_client): + response_success = app_client.get("/v1.0/plans") + assert response_success.status_code == httplib.OK + plan_list = json.loads(response_success.data)['plans'] + assert len(plan_list) > 0 + assert plan_list[0].endswith('.yaml') + + +def test_get_plan(app_client): + response_success = app_client.get("/v1.0/plans/sample.yaml") + assert response_success.status_code == httplib.OK + plan_data = json.loads(response_success.data) + assert PlanProp.NAME in plan_data + assert PlanProp.DESCRIPTION in plan_data + assert PlanProp.CONFIG in plan_data + assert PlanProp.QPIS in plan_data + + +def test_get_plan_not_found(app_client): + response_not_found = app_client.get("/v1.0/plans/fake.yaml") + response_data = json.loads(response_not_found.data) + assert response_not_found.status_code == httplib.NOT_FOUND + assert response_data['title'] == "Plan not found" + + +def test_runner_not_implemented(app_client): + response_error = app_client.post("/v1.0/plans/fake.yaml?action=run", follow_redirects=False) + assert response_error.status_code == httplib.NOT_IMPLEMENTED diff --git a/tests/unit/api/qpi_controller_test.py b/tests/unit/api/qpi_controller_test.py new file mode 100644 index 00000000..6291dd9b --- /dev/null +++ b/tests/unit/api/qpi_controller_test.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import httplib +import json + +from qtip.base.constant import FormulaName +from qtip.base.constant import SpecProp + + +def test_get_list_qpis(app_client): + response_success = app_client.get("/v1.0/qpis") + assert response_success.status_code == httplib.OK + qpi_spec_list = json.loads(response_success.data)['qpis'] + assert len(qpi_spec_list) > 0 + assert qpi_spec_list[0].endswith('.yaml') + + +def test_get_qpi(app_client): + response_success = app_client.get("/v1.0/qpis/compute.yaml") + assert response_success.status_code == httplib.OK + qpi_data = json.loads(response_success.data) + assert SpecProp.DESCRIPTION in qpi_data + assert SpecProp.FORMULA in qpi_data + assert SpecProp.SECTIONS in qpi_data + assert qpi_data[SpecProp.FORMULA] in FormulaName.__dict__.values() + sections = qpi_data[SpecProp.SECTIONS] + assert isinstance(sections, list) + for section in sections: + assert SpecProp.NAME in section + + +def test_get_qpi_not_found(app_client): + response_not_found = app_client.get("/v1.0/qpis/fake.yaml") + response_data = json.loads(response_not_found.data) + assert response_not_found.status_code == httplib.NOT_FOUND + assert response_data['title'] == "QPI not found" |