diff options
-rw-r--r-- | INFO | 11 | ||||
-rw-r--r-- | docs/apidocs/.keep | 0 | ||||
-rw-r--r-- | docs/userguide/index.rst | 6 | ||||
-rw-r--r-- | docs/userguide/introduction.rst | 14 | ||||
-rw-r--r-- | func/args_handler.py | 16 | ||||
-rw-r--r-- | func/cli.py | 26 | ||||
-rw-r--r-- | func/spawn_vm.py | 2 | ||||
-rw-r--r-- | restful_server/db.py | 6 | ||||
-rw-r--r-- | restful_server/qtip_server.py | 31 | ||||
-rw-r--r-- | tests/qtip_server_test.py | 15 | ||||
-rw-r--r-- | tests/spawn_vm_test.py | 3 |
11 files changed, 76 insertions, 54 deletions
@@ -1,13 +1,14 @@ Project Name: QTIP, Platform Performance Benchmarking -Project Repository: QTIP Project Category: Integration & Testing -Etherpad: https://etherpad.opnfv.org/p/qtip +Project Leader: Yujun Zhang <zhang.yujunz@zte.com.cn> +Primary Contact: Yujun Zhang <zhang.yujunz@zte.com.cn> +Wiki: https://wiki.opnfv.org/display/QTIP Jira: https://jira.opnfv.org/browse/QTIP +Etherpad: https://etherpad.opnfv.org/p/qtip +Repository: https://git.opnfv.org/cgit/qtip/ Gerrit: https://gerrit.opnfv.org/gerrit/#/admin/projects/qtip -GIT: https://git.opnfv.org/cgit/qtip/ Mailing list tag: [QTIP] -IRC: #opnfv-qtip -Project Wiki: https://wiki.opnfv.org/platform_performance_benchmarking +IRC: #opnfv-qtip@freenode Committers: diff --git a/docs/apidocs/.keep b/docs/apidocs/.keep deleted file mode 100644 index e69de29b..00000000 --- a/docs/apidocs/.keep +++ /dev/null diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index fe24293a..202bf66e 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -16,6 +16,6 @@ Colorado 1.0 ./overview.rst ./introduction.rst - ./01-compute_testcases.rst - ./02-network_testcases.rst - ./03-storage_testcases.rst + ./01-compute.rst + ./02-network.rst + ./03-storage.rst diff --git a/docs/userguide/introduction.rst b/docs/userguide/introduction.rst index 823a2722..b874ea46 100644 --- a/docs/userguide/introduction.rst +++ b/docs/userguide/introduction.rst @@ -23,12 +23,12 @@ The QTIP directory has been sectioned off into multiple folders to facilitate **test_cases/:** This folder is used to store all the config files which are used to setup the - environment prior to a test. This folder is further divided into opnfv pods - which run QTIP. Inside each pod there are folders which contain the config - files segmented based on test cases. Namely, these include, `Compute`, - `Network` and `Storage`. The default folder is there for the end user who - is interested in testing their infrastructure which is installed by fuel -or compass but aren't part of a opnfv pod,and for opnfv CI. +environment prior to a test. This folder is further divided into opnfv pods +which run QTIP. Inside each pod there are folders which contain the config files +segmented based on test cases. Namely, these include, `Compute`, `Network` and +`Storage`. The default folder is there for the end user who is interested in +testing their infrastructure which is installed by fuel or compass but aren't +part of a opnfv pod,and for opnfv CI. The structure of the directory for the user appears as follows :: @@ -338,14 +338,12 @@ Running QTIP on the using 'default' as the pod name and for the 'compute' suite :: curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "BM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs - Running QTIP on the using 'default' as the pod name and for the 'compute' suite 'vm' type by restful api :: curl --trace-ascii debug.txt -X POST -d '{ "installer_ip": "10.20.6.2","installer_type":"fuel", "suite_name":"compute", "type": "VM"}' -H "Content-Type: application/json" http://qtip_server_ip:5000/api/v1.0/jobs - Running QTIP on the using `default` as the pod name and for the `network` suite by cli :: diff --git a/func/args_handler.py b/func/args_handler.py index 50d803eb..59712800 100644 --- a/func/args_handler.py +++ b/func/args_handler.py @@ -14,14 +14,14 @@ from func.spawn_vm import SpawnVM from func.driver import Driver -def get_files_in_test_list(suit_name, case_type='all'): - benchmark_list = json.load(file('test_list/{0}'.format(suit_name))) +def get_files_in_test_list(suite_name, case_type='all'): + benchmark_list = json.load(file('test_list/{0}'.format(suite_name))) return reduce(add, benchmark_list.values()) \ if case_type == 'all' else benchmark_list[case_type] -def get_files_in_test_case(lab, suit_name, case_type='all'): - test_case_all = os.listdir('./test_cases/{0}/{1}'.format(lab, suit_name)) +def get_files_in_test_case(lab, suite_name, case_type='all'): + test_case_all = os.listdir('./test_cases/{0}/{1}'.format(lab, suite_name)) return test_case_all if case_type == 'all' else \ filter(lambda x: case_type in x, test_case_all) @@ -30,14 +30,18 @@ def get_benchmark_path(lab, suit, benchmark): return './test_cases/{0}/{1}/{2}'.format(lab, suit, benchmark) -def check_suit_in_test_list(suit_name): - return True if os.path.isfile('test_list/' + suit_name) else False +def check_suite_in_test_list(suite_name): + return True if os.path.isfile('test_list/' + suite_name) else False def check_lab_name(lab_name): return True if os.path.isdir('test_cases/' + lab_name) else False +def check_benchmark_name(lab, file, benchmark): + return os.path.isfile('test_cases/' + lab + '/' + file + '/' + benchmark) + + def _get_f_name(test_case_path): return test_case_path.split('/')[-1] diff --git a/func/cli.py b/func/cli.py index 66ab2277..9222da51 100644 --- a/func/cli.py +++ b/func/cli.py @@ -32,27 +32,41 @@ class cli: '\n network ' 'They contain all the tests that will be run. They are listed by suite.' 'Please ensure there are no empty lines') + parser.add_argument('-b', '--benchmark', help='Name of the benchmark.' + 'Can be found in test_lists/file_name') + return parser.parse_args(args) def __init__(self, args=sys.argv[1:]): args = self._parse_args(args) - if not args_handler.check_suit_in_test_list(args.file): + if not args_handler.check_suite_in_test_list(args.file): print('\n\n ERROR: Test File Does not exist in test_list/ please enter correct file \n\n') - sys.exit(0) + sys.exit(1) if not args_handler.check_lab_name(args.lab): print('\n\n You have specified a lab that is not present in test_cases/ please enter \ correct file. If unsure how to proceed, use -l default.\n\n') - sys.exit(0) + sys.exit(1) suite = args.file benchmarks = args_handler.get_files_in_test_list(suite) test_cases = args_handler.get_files_in_test_case(args.lab, suite) benchmarks_list = filter(lambda x: x in test_cases, benchmarks) - map(lambda x: args_handler.prepare_and_run_benchmark( - os.environ['INSTALLER_TYPE'], os.environ['PWD'], - args_handler.get_benchmark_path(args.lab.lower(), suite, x)), benchmarks_list) + if args.benchmark: + if not args_handler.check_benchmark_name(args.lab, args.file, args.benchmark): + print('\n\n You have specified an incorrect benchmark. Please' + 'enter the correct one.\n\n') + sys.exit(1) + else: + print("Starting with " + args.benchmark) + args_handler.prepare_and_run_benchmark( + os.environ['INSTALLER_TYPE'], os.environ['PWD'], + args_handler.get_benchmark_path(args.lab.lower(), args.file, args.benchmark)) + else: + map(lambda x: args_handler.prepare_and_run_benchmark( + os.environ['INSTALLER_TYPE'], os.environ['PWD'], + args_handler.get_benchmark_path(args.lab.lower(), suite, x)), benchmarks_list) print('{0} is not a Template in the Directory Enter a Valid file name.' 'or use qtip.py -h for list'.format(filter(lambda x: x not in test_cases, benchmarks))) diff --git a/func/spawn_vm.py b/func/spawn_vm.py index 7ac4340e..b467ab1f 100644 --- a/func/spawn_vm.py +++ b/func/spawn_vm.py @@ -69,7 +69,7 @@ class SpawnVM(Env_setup): mark = exc.problem_mark
print 'Error in qtip/heat/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1, mark.column + 1)
print 'EXITING PROGRAM. Correct File and restart'
- sys.exit(0)
+ sys.exit(1)
fopen = open('./data/QtipKey.pub', 'r')
fopenstr = fopen.read()
fopenstr = fopenstr.rstrip()
diff --git a/restful_server/db.py b/restful_server/db.py index cf6ebfbb..8da64187 100644 --- a/restful_server/db.py +++ b/restful_server/db.py @@ -23,7 +23,7 @@ def create_job(args): 'installer_ip': args["installer_ip"], 'pod_name': args["pod_name"], 'suite_name': args["suite_name"], - 'max-minutes': args["max-minutes"], + 'max_minutes': args["max_minutes"], 'type': args["type"], 'start_time': str(datetime.now()), 'end_time': None, @@ -73,7 +73,7 @@ def update_job_result_detail(job_id, benchmark, result): def is_job_timeout(job_id): period = datetime.now() - datetime.strptime(jobs[job_id]['start_time'], "%Y-%m-%d %H:%M:%S.%f") - return True if jobs[job_id]['max-minutes'] * 60 < period.total_seconds()\ + return True if jobs[job_id]['max_minutes'] * 60 < period.total_seconds()\ else False @@ -91,6 +91,6 @@ def stop_thread(job_id): del threads[job_id] -def update_benmark_state_in_state_detail(job_id, benchmark, benchmark_state): +def update_benchmark_state(job_id, benchmark, benchmark_state): filter(lambda x: x["benchmark"] == benchmark, get_job_info(job_id)["state_detail"])[0]['state'] = benchmark_state diff --git a/restful_server/qtip_server.py b/restful_server/qtip_server.py index 734a471c..a059ca3c 100644 --- a/restful_server/qtip_server.py +++ b/restful_server/qtip_server.py @@ -24,12 +24,13 @@ class JobModel: resource_fields = { 'installer_type': fields.String, 'installer_ip': fields.String, - 'max-minutes': fields.Integer, + 'max_minutes': fields.Integer, 'pod_name': fields.String, 'suite_name': fields.String, - 'type': fields.String + 'type': fields.String, + 'benchmark_name': fields.String } - required = ['installer_type', 'install_ip'] + required = ['installer_type', 'installer_ip'] @swagger.model @@ -91,7 +92,7 @@ class JobList(Resource): "installer_ip": The installer ip of the pod, -"max-minutes": If specified, the maximum duration in minutes +"max_minutes": If specified, the maximum duration in minutes for any single test iteration, default is '60', "pod_name": If specified, the Pod name, default is 'default', @@ -99,6 +100,8 @@ for any single test iteration, default is '60', "suite_name": If specified, Test suite name, for example 'compute', 'network', 'storage', default is 'compute' "type": BM or VM,default is 'BM' +"benchmark_name": If specified, benchmark name in suite, for example 'dhrystone_bm.yaml', +default is all benchmarks in suite with specified type """, "required": True, "type": "JobModel", @@ -123,15 +126,16 @@ default is 'compute' ) def post(self): parser = reqparse.RequestParser() - parser.add_argument('installer_type', type=str, required=True, help='Installer_type is required') - parser.add_argument('installer_ip', type=str, required=True, help='Installer_ip is required') - parser.add_argument('max-minutes', type=int, required=False, default=60, help='max-minutes should be integer') + parser.add_argument('installer_type', type=str, required=True, help='installer_type is required') + parser.add_argument('installer_ip', type=str, required=True, help='installer_ip is required') + parser.add_argument('max_minutes', type=int, required=False, default=60, help='max_minutes should be integer') parser.add_argument('pod_name', type=str, required=False, default='default', help='pod_name should be string') parser.add_argument('suite_name', type=str, required=False, default='compute', help='suite_name should be string') parser.add_argument('type', type=str, required=False, default='BM', help='type should be BM, VM and ALL') + parser.add_argument('benchmark_name', type=str, required=False, default='all', help='benchmark_name should be string') args = parser.parse_args() - if not args_handler.check_suit_in_test_list(args["suite_name"]): - return abort(404, 'message:Test Suit {0} does not exist in test_list'.format(args["suite_name"])) + if not args_handler.check_suite_in_test_list(args["suite_name"]): + return abort(404, 'message:Test suite {0} does not exist in test_list'.format(args["suite_name"])) if not args_handler.check_lab_name(args["pod_name"]): return abort(404, 'message: You have specified a lab {0}\ that is not present in test_cases'.format(args['pod_name'])) @@ -146,6 +150,11 @@ default is 'compute' args["suite_name"], args["type"].lower()) benchmarks_list = filter(lambda x: x in test_cases, benchmarks) + if args["benchmark_name"] in benchmarks_list: + benchmarks_list = [args["benchmark_name"]] + if (args["benchmark_name"] is not 'all') and args["benchmark_name"] not in benchmarks_list: + return abort(404, 'message: Benchmark name {0} does not exist in suit {1}'.format(args["benchmark_name"], + args["suite_name"])) state_detail = map(lambda x: {'benchmark': x, 'state': 'idle'}, benchmarks_list) db.update_job_state_detail(job_id, copy(state_detail)) thread_stop = threading.Event() @@ -162,14 +171,14 @@ default is 'compute' for benchmark in benchmarks_list: if db.is_job_timeout(job_id) or stop_event.is_set(): break - db.update_benmark_state_in_state_detail(job_id, benchmark, 'processing') + db.update_benchmark_state(job_id, benchmark, 'processing') result = args_handler.prepare_and_run_benchmark(installer_type, '/home', args_handler.get_benchmark_path(pod_name, suite_name, benchmark)) db.update_job_result_detail(job_id, benchmark, copy(result)) - db.update_benmark_state_in_state_detail(job_id, benchmark, 'finished') + db.update_benchmark_state(job_id, benchmark, 'finished') db.finish_job(job_id) diff --git a/tests/qtip_server_test.py b/tests/qtip_server_test.py index 511d209a..2f9eebf1 100644 --- a/tests/qtip_server_test.py +++ b/tests/qtip_server_test.py @@ -33,7 +33,7 @@ class TestClass: 'installer_ip': '10.20.0.2', 'pod_name': 'default', 'suite_name': 'compute', - 'max-minutes': 60, + 'max_minutes': 60, 'type': 'BM', 'state': 'finished', 'state_detail': [{'state': 'finished', 'benchmark': 'dhrystone_bm.yaml'}, @@ -45,22 +45,19 @@ class TestClass: ({'installer_type': 'fuel', 'installer_ip': '10.20.0.2', 'pod_name': 'zte-pod1', - 'max-minutes': 20, + 'max_minutes': 20, 'suite_name': 'compute', - 'type': 'VM'}, + 'type': 'VM', + 'benchmark_name': 'dhrystone_vm.yaml'}, {'job_id': '', 'installer_type': 'fuel', 'installer_ip': '10.20.0.2', 'pod_name': 'zte-pod1', 'suite_name': 'compute', - 'max-minutes': 20, + 'max_minutes': 20, 'type': 'VM', 'state': 'finished', - 'state_detail': [{u'state': u'finished', u'benchmark': u'dhrystone_vm.yaml'}, - {u'state': u'finished', u'benchmark': u'whetstone_vm.yaml'}, - {u'state': u'finished', u'benchmark': u'ramspeed_vm.yaml'}, - {u'state': u'finished', u'benchmark': u'dpi_vm.yaml'}, - {u'state': u'finished', u'benchmark': u'ssl_vm.yaml'}], + 'state_detail': [{u'state': u'finished', u'benchmark': u'dhrystone_vm.yaml'}], 'result': 0}) ]) @mock.patch('restful_server.qtip_server.args_handler.prepare_and_run_benchmark') diff --git a/tests/spawn_vm_test.py b/tests/spawn_vm_test.py index b22745d7..0ec5c902 100644 --- a/tests/spawn_vm_test.py +++ b/tests/spawn_vm_test.py @@ -43,13 +43,12 @@ class TestClass: [('172.10.0.154', '')]), ]) @mock.patch('func.spawn_vm.Env_setup') - @mock.patch('func.spawn_vm.FetchImg') @mock.patch('func.spawn_vm.create_zones') @mock.patch('func.spawn_vm.client', autospec=True) @mock.patch('func.spawn_vm.keystoneclient.v2_0', autospec=True) @mock.patch('func.spawn_vm.heatclient.client', autospec=True) def test_create_zones_success(self, mock_heat, mock_keystone, - mock_nova_client, mock_zone, mock_fetch, + mock_nova_client, mock_zone, mock_setup, test_input, expected): mock_nova_client.Client.return_value = Mock() mock_heat.Client.return_value = Mock(stacks=HeatMock()) |