diff options
25 files changed, 319 insertions, 191 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..93c8bc9d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,104 @@ +Contributing to QTIP +==================== + +First of all, thanks for taking your time to contribute. + +QTIP is a project in OPNFV. If you are new to OPNFV, you may read +[Developer Getting Started][gs] first. + +Peer Review +----------- + +Peer review is the most important communication channel between developers. +Every subtle change to the code or document **MUST** be reviewed before +submission. + +Add group `qtip-reviewers` in [gerrit][gr] when you consider a patch set is ready. + +Please make sure there is at least one `+1` or `+2` from others before +submitting a patch set. + +Note: only members in `ldap/opnfv-gerrit-qtip-submitters` have permission +to submit. The current members are listed in [INFO][if]. + +Active Reviewers +---------------- + +Current list of active reviewers in gerrit group `qtip-reviewers` + +* Serena Feng <feng.xiaowei@zte.com.cn> +* Taseer Ahmed <taseer94@gmail.com> +* Yujun Zhang <zhang.yujunz@zte.com.cn> +* Zhifeng Jiang <jiang.zhifeng@zte.com.cn> +* Zhihui Wu <wu.zhihui1@zte.com.cn> + +By becoming an active reviewer, you agree to allow others to invite you as +reviewers in QTIP project freely. Any one in OPNFV community can apply to join +QTIP reviewers group or leave by submitting a patch on this document. + +Tasks and Issues +---------------- + +Tasks and issues are management in [JIRA][jr]. The usage of different +[Issue Types][it] in QTIP are as following: + +* `Task`: it must be achievable in **one sprint**, otherwise it needs to be split. +* `Sub-Task`: it must be resolvable by **one developer** within **one sprint**, +otherwise it need to be split. + +`Bug`, `New Feature`, `Improvement`, `Story` and `Epic` are not +restricted by time frame. But it is recommended to to define the scope clearly +and break down into manageable tasks. + +Development Cycle +----------------- + +QTIP follows the cycle of [OPNFV Releases][or] which is approximately one release +every half year. + +The tasks are organized by sprints, three weeks for each. + +The target and content of each sprint is discussed in weekly meeting. + +Coding Style +------------ + +QTIP follows [OpenStack Style Guidelines][os] for source code and commit message. + +Specially, it is recommended to link each patch set with a JIRA issue. Put + + JIRA: QTIP-n + +in commit message to create an automatic link. + +Documentation +------------- + +The documents are built automatically by sphinx from reStructuredText (reST). +Please read [reStructuredText Primer][rp] if you are not familiar with it. + +A cheat sheet for headings are as following + +* `#` with overline, for parts +* `*` with overline, for chapters +* `=`, for sections +* `-`, for subsections +* `^`, for subsubsections +* `"`, for paragraphs + +Frequent Asked Questions +------------------------ + +Q: May I work on task which have already been assigned to others? + +A: Yes. But please make sure you have contacted the original assignee to avoid +overlapping. + +[gs]: https://wiki.opnfv.org/display/DEV/Developer+Getting+Started +[gr]: https://gerrit.opnfv.org/gerrit/#/q/project:+qtip +[jr]: https://jira.opnfv.org/browse/QTIP +[or]: https://wiki.opnfv.org/display/SWREL +[it]: https://jira.opnfv.org/secure/ShowConstantsHelp.jspa?decorator=popup#IssueTypes +[os]: http://docs.openstack.org/developer/hacking/ +[if]: https://git.opnfv.org/cgit/qtip/tree/INFO +[rp]: http://www.sphinx-doc.org/en/stable/rest.html diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst deleted file mode 100644 index d10f1393..00000000 --- a/CONTRIBUTING.rst +++ /dev/null @@ -1,114 +0,0 @@ -#################### -Contributing to QTIP -#################### - -First of all, thanks for taking your time to contribute. - -QTIP is a project in OPNFV. If you are new to OPNFV, you may read -:title:`Developer Getting Started`_ first. - -*********** -Peer Review -*********** - -Peer review is the most important communication channel between developers. -Every subtle change to the code or document **MUST** be reviewed before -submission. - -Add group ``qtip-reviewers`` in `gerrit`_ when you consider a patch set is ready. - -Please make sure there is at least one ``+1`` or ``+2`` from others before -submitting[#f1] a patch set. - -Active Reviewers -================ - -Current list of active reviewers in gerrit group ``qtip-reviewers`` - -* Serena Feng <feng.xiaowei@zte.com.cn> -* Taseer Ahmed <taseer94@gmail.com> -* Yujun Zhang <zhang.yujunz@zte.com.cn> -* Zhifeng Jiang <jiang.zhifeng@zte.com.cn> -* Zhihui Wu <wu.zhihui1@zte.com.cn> - -By becoming an active reviewer, you agree to allow others to invite you as -reviewers in QTIP project freely. Any one in OPNFV community can apply to join -QTIP reviewers group or leave by submitting a patch on this document. - -**************** -Tasks and Issues -**************** - -Tasks and issues are management in `JIRA`_. The usage of different -:title:`Issue Types`_ in QTIP are as following: - -* ``Task``: it must be achievable in **one sprint**, otherwise it needs to be split. -* ``Sub-Task``: it must be resolvable by **one developer** within **one sprint**, -otherwise it need to be split. - -``Bug``, ``New Feature``, ``Improvement``, ``Story`` and ``Epic`` are not -restricted by time frame. But it is recommended to to define the scope clearly -and break down into manageable tasks. - -***************** -Development Cycle -***************** - -QTIP follows the cycle of `OPNFV Releases`_ which is approximately one release -every half year. - -The tasks are organized by sprints, three weeks for each. - -The target and content of each sprint is discussed in weekly meeting. - -************ -Coding Style -************ - -QTIP follows :title:`OpenStack Style Guidelines`_ for source code and commit message. - -Specially, it is recommended to link each patch set with a JIRA issue. Put - - JIRA: QTIP-n - -in commit message to create an automatic link. - -************* -Documentation -************* - -The documents are built automatically by sphinx from reStructuredText (reST). -Please read `reStructuredText Primer`_ if you are not familiar with it. - -A cheat sheet for headings are as following - -* # with overline, for parts -* * with overline, for chapters -* =, for sections -* -, for subsections -* ^, for subsubsections -* ", for paragraphs - -************************ -Frequent Asked Questions -************************ - -Q: May I work on task which have already been assigned to others? -A: Yes. But please make sure you have contacted the original assignee to avoid -overlapping. - -.. rubric:: Footnotes - -.. [#f1] only members in ``ldap/opnfv-gerrit-qtip-submitters`` have permission -to submit. The current members are listed in `INFO`_. - -.. rubric:: Reference - -.. _Developer Getting Started: https://wiki.opnfv.org/display/DEV/Developer+Getting+Started -.. _gerrit: https://gerrit.opnfv.org/gerrit/#/q/project:+qtip -.. _JIRA: https://jira.opnfv.org/browse/QTIP -.. _OPNFV Releases: https://wiki.opnfv.org/display/SWREL -.. _Issue Types: https://jira.opnfv.org/secure/ShowConstantsHelp.jspa?decorator=popup#IssueTypes -.. _OpenStack Style Guidelines: http://docs.openstack.org/developer/hacking/ -.. _INFO: https://git.opnfv.org/cgit/qtip/tree/INFO -.. _reStructuredText Primer: http://www.sphinx-doc.org/en/stable/rest.html diff --git a/benchmarks/suite/compute.yaml b/benchmarks/suite/compute.yaml new file mode 100644 index 00000000..0761a87b --- /dev/null +++ b/benchmarks/suite/compute.yaml @@ -0,0 +1,46 @@ +QPI: compute +description: sample performance index of computing + +algorithm: weighted arithmetic mean + +section: +- name: Integer + weight: 0.3 + algorithm: geometric mean + perftests: + - name: dhrystone + workloads: + - single_cpu + - multi_cpu +- name: Floating + weight: 0.3 + algorithm: geometric mean + perftests: + - name: whetstone + workloads: + - single_cpu + - multi_cpu +- name: Memory + weight: 0.2 + algorithm: geometric mean + perftests: + - name: ramspeed + workloads: + - int: [add, average, copy, scale, triad] + - float: [add, average, copy, scale, triad] +- name: DPI + weight: 0.1 + algorithm: geometric mean + perftests: + - name: dpi + workloads: + - bps + - pps +- name: SSL + weight: 0.1 + algorithm: geometric mean + perftests: + - name: ssl + workloads: + - aes_128_cbc: [512, 1024, 2048, 4096] + - rsa_sig: [16, 64, 256, 1024, 8192] diff --git a/docker/Dockerfile b/docker/Dockerfile index 7cd24533..369fa6c8 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,7 +35,7 @@ supervisor \ python-setuptools \ --no-install-recommends -RUN easy_install -U setuptools +RUN pip install 'setuptools>=17.1' RUN apt-add-repository ppa:ansible/ansible -y RUN apt-key update -y diff --git a/opt/servers/roles/user/defaults/main.yml b/opt/servers/roles/user/defaults/main.yml index 41ee9853..ef496dd4 100644 --- a/opt/servers/roles/user/defaults/main.yml +++ b/opt/servers/roles/user/defaults/main.yml @@ -4,3 +4,4 @@ users: - { name: taseer, comment: "Taseer Ahmed <taseer94@gmail.com>" } - { name: serena, comment: "Serena Feng <feng.xiaowei@zte.com.cn>" } - { name: zhifeng, comment: "Zhifeng Jiang<jiang.zhifeng@zte.com.cn>" } + - { name: akhil, comment: "Akhil Batra<akhil.batra@research.iiit.ac.in"} diff --git a/opt/servers/roles/user/files/akhil.authorized_keys b/opt/servers/roles/user/files/akhil.authorized_keys new file mode 100644 index 00000000..43942621 --- /dev/null +++ b/opt/servers/roles/user/files/akhil.authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJ2zxPZ6gu19QhXlzCdZQjiRIXnzE0Tp2+7LL5hBbl39+dohPwKlSvXzlI9n3MpeDUBkEwUzcS/P4McbTYOU74nOb5vBoNhgayZIebG3gM5cgLwRVD219oOT6mBGNSgfuc54KvmDgIOX4p8o8TLFq+0Qv4YQXcj3uprAR3p5e9NDAfKX8k27M4Ba9Goqorda/0CguAT6b+MdWen554N9PJLWQwtN1Nm4CMf7R9QcDOQEwOXhoZoN+/xmgqQummoo+17IYaAslQhb3Cx16hy2JA8QhZa6k+KnAUvh9GHxGSZpex7w6jyDez5KngROQvIiLfjYkqlgD3O8CDfhrjccZ5 akhil.batra@research.iiit.ac.in diff --git a/qtip/api/cmd/__init__.py b/qtip/api/cmd/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/api/cmd/__init__.py diff --git a/qtip/api/cmd/server.py b/qtip/api/cmd/server.py new file mode 100644 index 00000000..852073a7 --- /dev/null +++ b/qtip/api/cmd/server.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from flask import Flask +from flask_restful import Api +from flask_restful_swagger import swagger +import qtip.api.router.mapper as mapper + +app = Flask(__name__) +api = swagger.docs(Api(app), apiVersion='0.1') + + +def add_routers(): + for (handler, url) in mapper.mappers: + api.add_resource(handler, url) + + +def main(): + add_routers() + app.run(host='0.0.0.0') + + +if __name__ == "__main__": + main() diff --git a/qtip/api/handler/__init__.py b/qtip/api/handler/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/api/handler/__init__.py diff --git a/qtip/api/db.py b/qtip/api/handler/db.py index 24fc27a5..24fc27a5 100644 --- a/qtip/api/db.py +++ b/qtip/api/handler/db.py diff --git a/qtip/api/qtip_server.py b/qtip/api/handler/job_handler.py index e2ee0d27..f230e596 100644 --- a/qtip/api/qtip_server.py +++ b/qtip/api/handler/job_handler.py @@ -1,46 +1,13 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from flask import Flask, abort -from flask_restful import Api, Resource, fields, reqparse -from flask_restful_swagger import swagger import threading from copy import copy -import db -import qtip.utils.args_handler as args_handler -import qtip.api.result_handler as result_handler +from flask.ext.restful import Resource, reqparse +from flask.ext.restful_swagger import swagger +from werkzeug.exceptions import abort -app = Flask(__name__) -api = swagger.docs(Api(app), apiVersion='0.1') - - -@swagger.model -class JobModel: - resource_fields = { - 'installer_type': fields.String, - 'installer_ip': fields.String, - 'max_minutes': fields.Integer, - 'pod_name': fields.String, - 'suite_name': fields.String, - 'type': fields.String, - 'benchmark_name': fields.String, - 'testdb_url': fields.String, - 'node_name': fields.String - } - required = ['installer_type', 'installer_ip'] - - -@swagger.model -class JobResponseModel: - resource_fields = { - 'job_id': fields.String - } +from qtip.api.handler import db, result_handler +from qtip.api.model.job_model import JobResponseModel +from qtip.utils import args_handler as args_handler class Job(Resource): @@ -197,10 +164,3 @@ default is all benchmarks in suite with specified type, if (result_handler.dump_suite_result(suite_name) and testdb_url): result_handler.push_suite_result_to_db(suite_name, testdb_url, installer_type, node_name) db.finish_job(job_id) - - -api.add_resource(JobList, '/api/v1.0/jobs') -api.add_resource(Job, '/api/v1.0/jobs/<string:id>') - -if __name__ == "__main__": - app.run(host='0.0.0.0') diff --git a/qtip/api/handler/result_handler.py b/qtip/api/handler/result_handler.py new file mode 100644 index 00000000..3d1d592e --- /dev/null +++ b/qtip/api/handler/result_handler.py @@ -0,0 +1,58 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import importlib +import json +from os.path import expanduser + +import qtip.utils.dashboard.pushtoDB as push_to_db +from qtip.utils import logger_utils + +logger = logger_utils.QtipLogger('suite_result').get + + +def get_benchmark_result(benchmark_name, suite_name): + benchmark_indices = importlib.import_module('scripts.ref_results' + '.{0}_benchmarks_indices'.format(suite_name)) + methodToCall = getattr(benchmark_indices, '{0}_index'.format(benchmark_name)) + return methodToCall() + + +def dump_suite_result(suite_name): + suite_dict = {} + suite_bench_list = {'compute': ['DPI', 'Dhrystone', 'Whetstone', 'SSL', 'RamSpeed'], + 'storage': ['FIO'], + 'network': ['IPERF']} + temp = 0 + l = len(suite_bench_list[suite_name]) + for benchmark in suite_bench_list[suite_name]: + try: + suite_dict[benchmark] = get_benchmark_result(benchmark.lower(), suite_name) + temp = temp + float(suite_dict[benchmark]['index']) + except OSError: + l = l - 1 + pass + + if l == 0: + logger.info("No {0} suite results found".format(suite_name)) + return False + else: + suite_index = temp / l + suite_dict_f = {'index': suite_index, + 'suite_results': suite_dict} + result_path = expanduser('~') + '/qtip/results' + with open('{0}/{1}_result.json'.format(result_path, suite_name), 'w+') as result_json: + json.dump(suite_dict_f, result_json, indent=4, sort_keys=True) + return True + + +def push_suite_result_to_db(suite_name, test_db_url, installer_type, node_name): + with open('results/{0}_result.json'.format(suite_name), 'r') as result_file: + j = json.load(result_file) + push_to_db.push_results_to_db(test_db_url, '{0}_test_suite'.format(suite_name), + j, installer_type, node_name) diff --git a/qtip/api/model/__init__.py b/qtip/api/model/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/api/model/__init__.py diff --git a/qtip/api/model/job_model.py b/qtip/api/model/job_model.py new file mode 100644 index 00000000..eef771b0 --- /dev/null +++ b/qtip/api/model/job_model.py @@ -0,0 +1,25 @@ +from flask.ext.restful import fields +from flask.ext.restful_swagger import swagger + + +@swagger.model +class JobModel: + resource_fields = { + 'installer_type': fields.String, + 'installer_ip': fields.String, + 'max_minutes': fields.Integer, + 'pod_name': fields.String, + 'suite_name': fields.String, + 'type': fields.String, + 'benchmark_name': fields.String, + 'testdb_url': fields.String, + 'node_name': fields.String + } + required = ['installer_type', 'installer_ip'] + + +@swagger.model +class JobResponseModel: + resource_fields = { + 'job_id': fields.String + } diff --git a/qtip/api/result_handler.py b/qtip/api/result_handler.py deleted file mode 100644 index de91cd2c..00000000 --- a/qtip/api/result_handler.py +++ /dev/null @@ -1,22 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import json -import scripts.ref_results.suite_result as suite_result -import qtip.utils.dashboard.pushtoDB as push_to_db - - -def dump_suite_result(suite_name): - return suite_result.get_suite_result(suite_name) - - -def push_suite_result_to_db(suite_name, test_db_url, installer_type, node_name): - with open('results/{0}_result.json'.format(suite_name), 'r') as result_file: - j = json.load(result_file) - push_to_db.push_results_to_db(test_db_url, '{0}_test_suite'.format(suite_name), - j, installer_type, node_name) diff --git a/qtip/api/router/__init__.py b/qtip/api/router/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/api/router/__init__.py diff --git a/qtip/api/router/mapper.py b/qtip/api/router/mapper.py new file mode 100644 index 00000000..a5f029ac --- /dev/null +++ b/qtip/api/router/mapper.py @@ -0,0 +1,7 @@ +from qtip.api.handler.job_handler import Job, JobList + + +mappers = [ + (JobList, '/api/v1.0/jobs'), + (Job, '/api/v1.0/jobs/<string:id>'), +] diff --git a/qtip/runner/perftest.py b/qtip/runner/perftest.py index c6d58397..a9b54716 100644 --- a/qtip/runner/perftest.py +++ b/qtip/runner/perftest.py @@ -13,7 +13,9 @@ from benchmark import Benchmark class PerfTest(Benchmark): - """PerfTest is the driver of external performance test tools""" + """WIP(yujunz): + a perftest is the driver of external performance test tools + It is usually referred in a suite to collect performance metric""" # paths to search for perftest _paths = [path.join(p, 'perftest') for p in Benchmark._paths] diff --git a/qtip/runner/suite.py b/qtip/runner/suite.py index 4179af64..0086a20e 100644 --- a/qtip/runner/suite.py +++ b/qtip/runner/suite.py @@ -13,7 +13,10 @@ from benchmark import Benchmark class Suite(Benchmark): - """A suite is consist of one or several perf tests and produces one QPI""" + """WIP(yujunz): + a suite is consist of one or several perf tests and produces one QPI. + It must be executed as part of testplan + """ # paths to search for suites _paths = [path.join(p, 'suite') for p in Benchmark._paths] diff --git a/qtip/runner/testplan.py b/qtip/runner/testplan.py index 57f3c978..f48a7147 100644 --- a/qtip/runner/testplan.py +++ b/qtip/runner/testplan.py @@ -13,7 +13,9 @@ from benchmark import Benchmark class TestPlan(Benchmark): - """A suite is consist of one or several perf tests and produces one QPI""" + """WIP(yujunz): + a test plan is consist of test condition and several suites which can be + executed by user""" # paths to search for suites _paths = [path.join(p, 'testplan') for p in Benchmark._paths] @@ -14,6 +14,7 @@ setup-hooks = [entry_points] console_scripts = qtip = qtip.cli.entry:cli + qtip-server = qtip.api.cmd.server:main [files] packages = diff --git a/tests/unit/api/qtip_server_test.py b/tests/unit/api/test_server.py index 96544c95..e9364d3d 100644 --- a/tests/unit/api/qtip_server_test.py +++ b/tests/unit/api/test_server.py @@ -1,9 +1,15 @@ -import qtip.api.qtip_server as server -import pytest import json -import mock import time +import mock +import pytest + +import qtip.api.cmd.server as server + + +def setup_module(): + server.add_routers() + @pytest.fixture def app(): @@ -66,7 +72,7 @@ class TestClass: 'state_detail': [{u'state': u'finished', u'benchmark': u'dhrystone_vm.yaml'}], 'result': 0}) ]) - @mock.patch('qtip.api.qtip_server.args_handler.prepare_and_run_benchmark') + @mock.patch('qtip.utils.args_handler.prepare_and_run_benchmark') def test_post_get_delete_job_successful(self, mock_args_handler, app_client, body, expected): mock_args_handler.return_value = {'result': 0, 'detail': {'host': [(u'10.20.6.14', {'unreachable': 0, @@ -107,7 +113,7 @@ class TestClass: ['job_id', 'It already has one job running now!']) ]) - @mock.patch('qtip.api.qtip_server.args_handler.prepare_and_run_benchmark', + @mock.patch('qtip.utils.args_handler.prepare_and_run_benchmark', side_effect=[side_effect_sleep(0.5), side_effect_pass]) def test_post_two_jobs_unsuccessful(self, mock_args_hanler, app_client, body, expected): reply_1 = app_client.post("/api/v1.0/jobs", data=body[0]) diff --git a/tests/unit/runner/perftest_test.py b/tests/unit/runner/perftest_test.py index 1f0b5631..2b400ac8 100644 --- a/tests/unit/runner/perftest_test.py +++ b/tests/unit/runner/perftest_test.py @@ -40,3 +40,9 @@ class TestPerfTest: assert Property.DESCRIPTION in desc assert Property.ABSPATH in desc assert Property.ABSPATH is not None + + def test_describe(self): + desc = PerfTest('test-a').describe() + assert Property.NAME in desc + assert Property.DESCRIPTION in desc + assert Property.ABSPATH in desc diff --git a/tests/unit/runner/suite_test.py b/tests/unit/runner/suite_test.py index 10205801..acfed82c 100644 --- a/tests/unit/runner/suite_test.py +++ b/tests/unit/runner/suite_test.py @@ -40,3 +40,9 @@ class TestSuite: assert Property.DESCRIPTION in suite_desc assert Property.ABSPATH in suite_desc assert Property.ABSPATH is not None + + def test_describe(self): + desc = Suite('suite-a').describe() + assert Property.NAME in desc + assert Property.DESCRIPTION in desc + assert Property.ABSPATH in desc diff --git a/tests/unit/runner/testplan_test.py b/tests/unit/runner/testplan_test.py index de8bf1e1..7e42f557 100644 --- a/tests/unit/runner/testplan_test.py +++ b/tests/unit/runner/testplan_test.py @@ -40,3 +40,9 @@ class TestTestPlan: assert Property.DESCRIPTION in desc assert Property.ABSPATH in desc assert Property.ABSPATH is not None + + def test_describe(self): + desc = TestPlan('plan-a').describe() + assert Property.NAME in desc + assert Property.DESCRIPTION in desc + assert Property.ABSPATH in desc |