diff options
38 files changed, 390 insertions, 878 deletions
diff --git a/benchmarks/metric/dhrystone.yaml b/benchmarks/metric/dhrystone.yaml deleted file mode 100644 index e4d26248..00000000 --- a/benchmarks/metric/dhrystone.yaml +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -name: dhrystone -description: > - A synthetic computing benchmark program intended to be representative of - system (integer) programming. -links: - - https://en.wikipedia.org/wiki/Dhrystone -workloads: - - single_cpu - - multi_cpu diff --git a/benchmarks/metric/ramspeed.yaml b/benchmarks/metric/ramspeed.yaml deleted file mode 100644 index e2e2b98f..00000000 --- a/benchmarks/metric/ramspeed.yaml +++ /dev/null @@ -1,13 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -name: ramspeed -description: a memory performance benchmarking -workloads: - - int: [add, average, copy, scale, triad] - - float: [add, average, copy, scale, triad] diff --git a/benchmarks/metric/whetstone.yaml b/benchmarks/metric/whetstone.yaml deleted file mode 100644 index 625ea87f..00000000 --- a/benchmarks/metric/whetstone.yaml +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -name: whetstone -description: > - A synthetic benchmark for evaluating the performance of computers. - The Whetstone benchmark primarily measures the floating-point arithmetic - performance. -links: - - https://en.wikipedia.org/wiki/Whetstone_(benchmark) -workloads: - - single_cpu - - multi_cpu diff --git a/benchmarks/plan/compute.yaml b/benchmarks/plan/compute.yaml deleted file mode 100644 index f4a7a2dc..00000000 --- a/benchmarks/plan/compute.yaml +++ /dev/null @@ -1,120 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corporation and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -name: compute QPI -description: compute QPI profile -info: - facility: local - engineer: local -config: - driver: ansible - collectors: - - type: logfile - paths: - - '../../external/dpi/' - logs: - - filename: dpi_dump.txt - parsers: - - type: grep - regex: |- - ^\s+nDPI throughput:.+?(?P<pps>\d+.\d+)\sM\spps.+ - ?(?P<bps>\d+.\d+)\sGb\/sec - - type: logfile - paths: - - '../../external/ramspeed/' - logs: - - filename: Intmem - parsers: - - type: grep - regex: '^INTEGER\s+BatchRun\s+Copy:\s+?(?P<integer_copy>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^INTEGER\s+BatchRun\s+Scale:\s+?(?P<integer_scale>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^INTEGER\s+BatchRun\s+Add:\s+?(?P<integer_add>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^INTEGER\s+BatchRun\s+Triad:\s+?(?P<integer_triad>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^INTEGER\s+BatchRun\s+AVERAGE:\s+?(?P<integer_average>\d+\.\d+)\sMB/s$' - - filename: Floatmem - parsers: - - type: grep - regex: '^FL-POINT\s+BatchRun\s+Copy:\s+?(?P<float_copy>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^FL-POINT\s+BatchRun\s+Scale:\s+?(?P<float_scale>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^FL-POINT\s+BatchRun\s+Add:\s+?(?P<float_add>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^FL-POINT\s+BatchRun\s+Triad:\s+?(?P<float_triad>\d+\.\d+)\sMB/s$' - - type: grep - regex: '^FL-POINT\s+BatchRun\s+AVERAGE:\s+?(?P<float_average>\d+\.\d+)\sMB/s$' - - type: logfile - paths: - - '../../external/ssl/' - logs: - - filename: RSA_dump - parsers: - - type: grep - regex: |- - ^rsa\s+512\sbits\s.+ - ?(?P<rsa_sign_512>\d+\.\d)\s+ - ?(?P<rsa_verify_512>\d+\.\d)$ - - type: grep - regex: |- - ^rsa\s+1024\sbits\s.+ - ?(?P<rsa_sign_1024>\d+\.\d)\s+ - ?(?P<rsa_verify_1024>\d+\.\d)$ - - type: grep - regex: |- - ^rsa\s+2048\sbits\s.+ - ?(?P<rsa_sign_2048>\d+\.\d)\s+ - ?(?P<rsa_verify_2048>\d+\.\d)$ - - type: grep - regex: |- - ^rsa\s+4096\sbits\s.+ - ?(?P<rsa_sign_4096>\d+\.\d)\s+ - ?(?P<rsa_verify_4096>\d+\.\d)$ - - filename: AES-128-CBC_dump - parsers: - - type: grep - regex: |- - ^aes-128-cbc\s+ - ?(?P<aes_128_cbc_16_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_64_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_256_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_1024_bytes>\d+\.\w+)\s+ - ?(?P<aes_128_cbc_8192_bytes>\d+\.\w+)$ - - type: logfile - paths: - - '../../external/sysinfo' - logs: - - filename: top.log - parsers: - - type: grep - regex: 'Cpu\(s\):.+?(?P<cpu_idle>\d+\.\d)\sid' - - filename: inxi.log - parsers: - - type: grep - regex: '.+\s+Host:\s+(?P<hostname>.+)\sKernel' - - type: grep - regex: '.+\sMemory:\s+(?P<memory>.+MB)\s' - - type: grep - regex: '^CPU\(s\):\s+(?P<cpu>.+)' - - type: grep - regex: '.+\sDistro:\s+(?P<os>.+)' - - type: grep - regex: '.+\sKernel:\s+(?P<kernel>.+)\sConsole' - - type: grep - regex: '.+\s+HDD Total Size:\s+(?P<disk>.+)\s' - - type: grep - regex: '.+\sproduct:\s+(?P<product>.+)\sversion' - reporter: - name: console - # transform collected data into timeline - transformer: timeline -QPIs: - - compute.yaml diff --git a/docs/proposal/storage-qpi.rst b/docs/proposal/storage-qpi.rst index af7c7954..8a82241e 100644 --- a/docs/proposal/storage-qpi.rst +++ b/docs/proposal/storage-qpi.rst @@ -13,6 +13,48 @@ The measurement is done by `StorPerf`_. .. _StorPerf: https://wiki.opnfv.org/display/storperf +System Information +================== + +System Information are environmental parameters and factors may affect storage performance: + ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| System Factors | Detail | Extraction Method | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Ceph Node List | List of nodes which has ceph-osd roles. | Getting from return result of installer node list CLI command. | +| | For example [node-2, node-3, node-4]. | | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Ceph Client RDB Cache Mode | Values: "None", "write-through", "write-back". | Getting from value of "rbd cache" and "rbd cache max dirty" keys | +| | | in client section of ceph configuration; To enable write-through mode, | +| | | set rbd cache max dirty to 0. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Ceph Client RDB Cache Size | The RBD cache size in bytes. Default is 32 MiB. | Getting from value of "rdb cache size" key in client section | +| | | of ceph configuration. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Ceph OSD Tier Cache Mode | Values: "None", "Write-back", "Readonly". | Getting from ceph CLI "ceph report" output info. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Use SSD Backed OSD Cache | Values: "Yes", "No". | Getting from POD description and CEPH CLI "ceph-disk list" output info. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Use SSD For Journal | Values: "Yes", "No". | Getting from POD description and CEPH CLI "ceph-disk list" output info. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ +| Ceph Cluster Network Bandwidth | Values: "1G", "10G", "40G". | Getting from physical interface information in POD description, | +| | | "ifconfig" output info on ceph osd node, | +| | | and value of "cluster network" key in global section of ceph configuration. | ++--------------------------------+-------------------------------------------------+-----------------------------------------------------------------------------+ + +Test Condition +============== + ++-----------------------------+------------------------------------------+-------------------------------------------------------------+ +| Test Condition | Detail | Extraction Method | ++-----------------------------+------------------------------------------+-------------------------------------------------------------+ +| Number of Testing VMs | Number of VMs which are created, | It equals the number of Cinder nodes of the SUT. | +| | during running Storperf test case. | | ++-----------------------------+------------------------------------------+-------------------------------------------------------------+ +| Distribution of Testing VMS | Number of VMs on each computer node, | Recording the distribution when runing Storperf test case. | +| | for example [(node-2: 1), (node-3: 2))]. | | ++-----------------------------+------------------------------------------+-------------------------------------------------------------+ + Baseline ======== diff --git a/docs/testing/user/userguide/cli.rst b/docs/testing/user/userguide/cli.rst index 99efd930..8a983155 100644 --- a/docs/testing/user/userguide/cli.rst +++ b/docs/testing/user/userguide/cli.rst @@ -10,12 +10,26 @@ QTIP consists of a number of benchmarking tools or metrics, grouped under QPI's. components of a NFVI ecosystem, such as compute, network and storage. Depending on the type of application, a user may group them under plans. +Bash Command Completion +======================= + +To enable command completion, an environment variable needs to be enabled. +Add the following line to the **.bashrc** file +:: + + eval "$(_QTIP_COMPLETE=source qtip)" + +Getting help +============ + QTIP CLI provides interface to all of the above the components. A help page provides a list of all the commands along with a short description. :: qtip [-h|--help] +Usage +===== Typically a complete plan is executed at the target environment. QTIP defaults to a number of sample plans. A list of all the available plans can be viewed :: @@ -55,6 +69,9 @@ the results are stored. Extending the example above qtip report show <metric_name> -p $PWD/result +Debugging options +================= + Debug option helps identify the error by providing a detailed traceback. It can be enabled as :: diff --git a/qtip/api/controllers/metric.py b/qtip/api/controllers/metric.py deleted file mode 100644 index 96cd985c..00000000 --- a/qtip/api/controllers/metric.py +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import httplib - -from qtip.api.controllers import common -from qtip.loader import metric - - -def list_metrics(): - metrics = list(metric.MetricSpec.list_all()) - metrics_by_name = [m['name'] for m in metrics] - return {'metrics': metrics_by_name}, httplib.OK - - -@common.check_endpoint_for_error(resource='Metric') -def get_metric(name): - metric_spec = metric.MetricSpec(name) - return metric_spec.content diff --git a/qtip/api/controllers/plan.py b/qtip/api/controllers/plan.py deleted file mode 100644 index 00593878..00000000 --- a/qtip/api/controllers/plan.py +++ /dev/null @@ -1,31 +0,0 @@ -############################################################################## -# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import httplib - -from qtip.api.controllers import common -from qtip.base import error -from qtip.loader import plan - - -def list_plans(): - plans = list(plan.Plan.list_all()) - plans_by_name = [p['name'] for p in plans] - return {'plans': plans_by_name}, httplib.OK - - -@common.check_endpoint_for_error(resource='Plan') -def get_plan(name): - plan_spec = plan.Plan(name) - return plan_spec.content - - -@common.check_endpoint_for_error(resource='Plan', operation='Run') -def run_plan(name, action="run"): - raise error.ToBeDoneError('run_plan', 'plan') diff --git a/qtip/api/swagger/swagger.yaml b/qtip/api/swagger/swagger.yaml index 51c3ebb8..8cb0e23b 100644 --- a/qtip/api/swagger/swagger.yaml +++ b/qtip/api/swagger/swagger.yaml @@ -16,95 +16,6 @@ consumes: produces: - application/json paths: - /plans: - get: - summary: List all plans - operationId: qtip.api.controllers.plan.list_plans - tags: - - Plan - - Standalone - responses: - 200: - description: A list of plans - schema: - type: array - items: - $ref: '#/definitions/Plans' - 501: - description: Resource not implemented - schema: - $ref: '#/definitions/Error' - default: - description: Unexpected error - schema: - $ref: '#/definitions/Error' - /plans/{name}: - get: - summary: Get a plan by plan name - operationId: qtip.api.controllers.plan.get_plan - tags: - - Plan - - Standalone - parameters: - - name: name - in: path - description: Plan name - required: true - type: string - responses: - 200: - description: Plan information - schema: - $ref: '#/definitions/Plan' - 404: - description: Plan not found - schema: - $ref: '#/definitions/Error' - 501: - description: Resource not implemented - schema: - $ref: '#/definitions/Error' - default: - description: Unexpected error - schema: - $ref: '#/definitions/Error' - post: - summary: Run a plan and return results - operationId: qtip.api.controllers.plan.run_plan - tags: - - Plan - - Standalone - parameters: - - name: name - in: path - description: Plan name - required: true - type: string - - name: action - in: query - description: action for a plan - required: true - type: string - responses: - 200: - description: Result of the run of the plan - #TODO (akhil) define schema - 404: - description: Plan not found - schema: - $ref: '#/definitions/Error' - 400: - description: Invalid parameters - schema: - $ref: '#/definitions/Error' - 501: - description: Resource not implemented - schema: - $ref: '#/definitions/Error' - default: - description: Unexpected error - schema: - $ref: '#/definitions/Error' /qpis: get: summary: List all QPIs @@ -158,112 +69,7 @@ paths: description: Unexpected error schema: $ref: '#/definitions/Error' - /metrics: - get: - summary: List all metrics - operationId: qtip.api.controllers.metric.list_metrics - tags: - - Metric - - Standalone - - Agent - responses: - 200: - description: A list of metrics - schema: - items: - $ref: '#/definitions/Metrics' - 501: - description: Resource not implemented - schema: - $ref: '#/definitions/Error' - default: - description: Unexpected error - schema: - $ref: '#/definitions/Error' - /metrics/{name}: - get: - summary: Get a metric - operationId: qtip.api.controllers.metric.get_metric - tags: - - Metric - - Standalone - - Agent - parameters: - - name: name - in: path - description: Metric name - required: true - type: string - responses: - 200: - description: Metric information - schema: - $ref: '#/definitions/Metric' - 404: - description: Metric not found - schema: - $ref: '#/definitions/Error' - 501: - description: Resource not implemented - schema: - $ref: '#/definitions/Error' - default: - description: Unexpected error - schema: - $ref: '#/definitions/Error' definitions: - Plan: - type: object - required: - - name - properties: - name: - type: string - description: - type: string - info: - type: object - config: - type: object - QPIs: - type: array - items: - type: object - Plans: - type: object - required: - - plans - properties: - plans: - type: array - items: - type: string - Metric: - type: object - required: - - name - properties: - name: - type: string - description: - type: string - links: - type: array - items: - type: string - workloads: - type: array - items: - type: string - Metrics: - type: object - required: - - metrics - properties: - metrics: - type: array - items: - type: string QPI: type: object required: diff --git a/qtip/cli/commands/cmd_metric.py b/qtip/cli/commands/cmd_metric.py deleted file mode 100644 index 0a385898..00000000 --- a/qtip/cli/commands/cmd_metric.py +++ /dev/null @@ -1,54 +0,0 @@ -############################################################################## -# Copyright (c) 2017 taseer94@gmail.com and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import click -from colorama import Fore -import os - -from qtip.base.error import InvalidContentError -from qtip.base.error import NotFoundError -from qtip.cli import utils -from qtip.loader.metric import MetricSpec - - -@click.group() -def cli(): - ''' Performance Metrics Group ''' - pass - - -@cli.command('list', help='List all the Metric Groups') -def cmd_list(): - metrics = MetricSpec.list_all() - table = utils.table('Metrics', metrics) - click.echo(table) - - -@cli.command('show', help='View details of a Metric') -@click.argument('name') -def show(name): - try: - metric = MetricSpec('{}.yaml'.format(name)) - except NotFoundError as nf: - click.echo(Fore.RED + "ERROR: metric spec: " + nf.message) - except InvalidContentError as ice: - click.echo(Fore.RED + "ERROR: metric spec " + ice.message) - else: - cnt = metric.content - output = utils.render('metric', cnt) - click.echo(output) - - -@cli.command('run', help='Run performance test') -@click.argument('name') -@click.option('-p', '--path', help='Path to store results') -def run(name, path): - runner_path = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir, - 'runner/runner.py') - os.system('python {0} -b {1} -d {2}'.format(runner_path, name, path)) diff --git a/qtip/loader/metric.py b/qtip/loader/metric.py deleted file mode 100644 index 842fcdbf..00000000 --- a/qtip/loader/metric.py +++ /dev/null @@ -1,16 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from yaml_file import YamlFileLoader - - -class MetricSpec(YamlFileLoader): - """metrics in QTIP are categorized by performance test tools, such as - dhrystone, whetstone and etc""" - RELATIVE_PATH = 'metric' diff --git a/qtip/loader/plan.py b/qtip/loader/plan.py deleted file mode 100644 index e15651a3..00000000 --- a/qtip/loader/plan.py +++ /dev/null @@ -1,58 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from qtip.base.constant import BaseProp -from qtip.collector import CollectorProp as CProp -from qtip.collector.logfile import LogfileCollector -from qtip.loader.yaml_file import YamlFileLoader -from qtip.loader.qpi import QPISpec - - -# TODO(yujunz) more elegant way to load module dynamically -def load_collector(type_name): - if type_name == LogfileCollector.TYPE: - return LogfileCollector - else: - raise Exception("Invalid collector type: {}".format(type_name)) - - -class Plan(YamlFileLoader): - """ - a benchmark plan is consist of configuration and a QPI list - """ - - RELATIVE_PATH = 'plan' - - def __init__(self, name, paths=None): - super(Plan, self).__init__(name, paths) - - _config = self.content[PlanProp.CONFIG] - - self.collectors = [load_collector(c[CProp.TYPE])(c, self) - for c in _config[PlanProp.COLLECTORS]] - - self.qpis = [QPISpec(qpi, paths=paths) - for qpi in self.content[PlanProp.QPIS]] - - -class PlanProp(BaseProp): - # plan - INFO = 'info' - - FACILITY = 'facility' - ENGINEER = 'engineer' - - CONFIG = 'config' - - DRIVER = 'driver' - COLLECTORS = 'collectors' - REPORTER = 'reporter' - - QPIS = 'QPIs' diff --git a/qtip/loader/qpi.py b/qtip/loader/qpi.py index 73da61e9..2a85766e 100644 --- a/qtip/loader/qpi.py +++ b/qtip/loader/qpi.py @@ -8,7 +8,6 @@ ############################################################################## from yaml_file import YamlFileLoader -from metric import MetricSpec from qtip.base.constant import SpecProp from qtip.util.formula import Formula @@ -34,5 +33,3 @@ class Section(object): self.name = content[SpecProp.NAME] self.weight = content[SpecProp.WEIGHT] self.formula = Formula(content[SpecProp.FORMULA]) - self.metrics = [MetricSpec(record, paths=paths) - for record in content[SpecProp.METRICS]] diff --git a/benchmarks/metric/ssl.yaml b/qtip/web/bench/forms.py index 2b01dccd..d897aca7 100644 --- a/benchmarks/metric/ssl.yaml +++ b/qtip/web/bench/forms.py @@ -1,13 +1,19 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corporation and others. +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -name: ssl -description: a Secure Sockets Layer performance benchmarking -workloads: - - aes_128_cbc: [512, 1024, 2048, 4096] - - rsa_sig: [16, 64, 256, 1024, 8192] + + +from django import forms + +import models + + +class TaskForm(forms.ModelForm): + class Meta: + model = models.Task + fields = ['repo'] diff --git a/qtip/web/bench/migrations/0005_auto_20170720_2115.py b/qtip/web/bench/migrations/0005_auto_20170720_2115.py new file mode 100644 index 00000000..5bd81a2c --- /dev/null +++ b/qtip/web/bench/migrations/0005_auto_20170720_2115.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-07-20 21:15 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bench', '0004_auto_20170715_0325'), + ] + + operations = [ + migrations.RenameField( + model_name='repo', + old_name='github_link', + new_name='git_link', + ), + migrations.AlterField( + model_name='task', + name='end_time', + field=models.DateTimeField(null=True), + ), + migrations.AlterField( + model_name='task', + name='log', + field=models.FilePathField(), + ), + migrations.AlterField( + model_name='task', + name='run_time', + field=models.DurationField(null=True), + ), + ] diff --git a/qtip/web/bench/migrations/0006_auto_20170722_0135.py b/qtip/web/bench/migrations/0006_auto_20170722_0135.py new file mode 100644 index 00000000..5d066702 --- /dev/null +++ b/qtip/web/bench/migrations/0006_auto_20170722_0135.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-07-22 01:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bench', '0005_auto_20170720_2115'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='status', + field=models.CharField(choices=[('P', 'Pending'), ('IP', 'In progress'), ('F', 'Finished')], default='P', max_length=20), + ), + migrations.AlterField( + model_name='task', + name='log', + field=models.FileField(upload_to='logs'), + ), + ] diff --git a/qtip/web/bench/models.py b/qtip/web/bench/models.py index 863af739..3f0439d9 100644 --- a/qtip/web/bench/models.py +++ b/qtip/web/bench/models.py @@ -18,15 +18,33 @@ from django.urls import reverse class Repo(models.Model): name = models.CharField(max_length=200, blank=False) - github_link = models.URLField(unique=True) + git_link = models.URLField(unique=True) def get_absolute_url(self): return reverse('repo_update', args=[self.pk]) + def __str__(self): + return "%s, %s" % (self.name, self.git_link) + class Task(models.Model): + TASK_STATUS_CHOICES = ( + ('P', 'Pending'), + ('IP', 'In progress'), + ('F', 'Finished') + ) + start_time = models.DateTimeField(auto_now_add=True) - end_time = models.DateTimeField() - run_time = models.DurationField() + status = models.CharField(choices=TASK_STATUS_CHOICES, default='P', max_length=20) + end_time = models.DateTimeField(null=True) + run_time = models.DurationField(null=True) repo = models.ForeignKey('Repo', on_delete=models.DO_NOTHING) - log = models.TextField() + log = models.FileField(upload_to='logs') + + def save(self, **kwargs): + if self.end_time: + self.run_time = self.end_time - self.start_time + super(Task, self).save(kwargs) + + def get_absolute_url(self): + return reverse('task_view', args=[self.pk]) diff --git a/qtip/web/bench/urls.py b/qtip/web/bench/urls.py index 3af4eb83..ae9738b6 100644 --- a/qtip/web/bench/urls.py +++ b/qtip/web/bench/urls.py @@ -18,4 +18,7 @@ urlpatterns = [ url('^', include('django.contrib.auth.urls')), url('^repos/$', views.ReposView.as_view(), name='repos'), url('^repos/(?P<pk>\d+)$', views.RepoUpdate.as_view(), name='repo_update'), + url('^run/$', views.Run.as_view(), name='run'), + url('^tasks/$', views.Logs.as_view(), name='tasks'), + url('^tasks/(?P<pk>\d+)$', views.TaskView.as_view(), name='task_view'), ] diff --git a/qtip/web/bench/utils.py b/qtip/web/bench/utils.py new file mode 100644 index 00000000..aac388e0 --- /dev/null +++ b/qtip/web/bench/utils.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +from subprocess import Popen, PIPE, STDOUT +import shutil + + +def run(repo): + if os.path.exists(repo.name): + shutil.rmtree(repo.name) + os.mkdir(repo.name) + os.chdir(repo.name) + output = Popen(("git clone %s ." % repo.git_link).split(), stdout=PIPE, stderr=STDOUT) + for line in output.stdout: + yield line + output.wait() + output = Popen("ansible-playbook run.yml".split(), stdout=PIPE, stderr=STDOUT) + for line in output.stdout: + yield line + os.chdir("../") diff --git a/qtip/web/bench/views.py b/qtip/web/bench/views.py index 786b67d5..6de5e4cc 100644 --- a/qtip/web/bench/views.py +++ b/qtip/web/bench/views.py @@ -9,16 +9,20 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from multiprocessing import Process from django.contrib.auth.mixins import LoginRequiredMixin -# from django.utils.decorators import method_decorator from django.views.generic.edit import CreateView, UpdateView +from django.views.generic.list import ListView +from django.views.generic.detail import DetailView +from django.views import View +from django.shortcuts import render, redirect +from django.core.files.base import ContentFile +from django.utils import timezone +import forms import models - -# from django.shortcuts import render - -# Create your views here. +import utils class ReposView(LoginRequiredMixin, CreateView): @@ -39,3 +43,52 @@ class RepoUpdate(LoginRequiredMixin, UpdateView): context = super(RepoUpdate, self).get_context_data(**kwargs) context["repos"] = self.model.objects.all() return context + + +class Run(LoginRequiredMixin, View): + template_name = 'bench/run.html' + form_class = forms.TaskForm + + def get(self, request): + task_form = self.form_class() + return render(request, self.template_name, {'form': task_form}) + + def post(self, request): + task_form = self.form_class(request.POST) + if task_form.is_valid(): + new_task = task_form.save() + new_task.log.save("run_%s.log" % new_task.pk, ContentFile('')) + p = Process(target=self.start_task, args=(new_task,)) + p.start() + return redirect('tasks') + p.join() + + def start_task(self, task): + task = models.Task.objects.get(pk=task.pk) + task.status = 'IP' + task.save() + with open(task.log.path, "a") as logfile: + for line in utils.run(task.repo): + logfile.write(line) + now = timezone.now() + task = models.Task.objects.get(pk=task.pk) + task.end_time = now + task.status = 'F' + task.save() + + +class Logs(LoginRequiredMixin, ListView): + model = models.Task + + +class TaskView(LoginRequiredMixin, DetailView): + model = models.Task + + def get_context_data(self, **kwargs): + context = super(TaskView, self).get_context_data(**kwargs) + try: + with open(context['object'].log.path, "r") as log_file: + context['log'] = log_file.read() + except ValueError: + context['log'] = "No log to show" + return context diff --git a/qtip/web/templates/bench/run.html b/qtip/web/templates/bench/run.html new file mode 100644 index 00000000..8eaa251b --- /dev/null +++ b/qtip/web/templates/bench/run.html @@ -0,0 +1,15 @@ +{% extends 'bench/base.html' %} +{% block content %} + <div> + <h3>Repos</h3> + {% for repo in repos %} + <li>{{ repo.name }}</li> + {% endfor %} + </div> + <form role="form" method="post" action="">{% csrf_token %} + {{ form }} + <div class=input-field" style="margin:30px 20px"> + <button type="submit" value="Login"/>Submit</button> + </div> + </form> +{% endblock %}
\ No newline at end of file diff --git a/qtip/web/templates/bench/task_detail.html b/qtip/web/templates/bench/task_detail.html new file mode 100644 index 00000000..d715022b --- /dev/null +++ b/qtip/web/templates/bench/task_detail.html @@ -0,0 +1,17 @@ +{% extends 'bench/base.html' %} +{% block content %} + <div> + <h3>Task</h3> + <table> + <tr> + <td>{{ object.repo}}</td><td>{{ object.get_status_display }}</td><td>{{ object.start_time }}</td> + <td>{{ object.end_time }}</td><td>{{ object.duration }}</td><td>{{ object.log }}</td> + </tr> + </table> + <div> + <p> + {{ log | linebreaks }} + </p> + </div> + </div> +{% endblock %}
\ No newline at end of file diff --git a/qtip/web/templates/bench/task_list.html b/qtip/web/templates/bench/task_list.html new file mode 100644 index 00000000..188d6b81 --- /dev/null +++ b/qtip/web/templates/bench/task_list.html @@ -0,0 +1,22 @@ +{% extends 'bench/base.html' %} +{% block content %} + <div> + <h3>Tasks Log</h3> + <table> + <th>Repo</th> + <th>Status</th> + <th>Start Time</th> + <th>End Time</th> + <th>Run time</th> + {% for task in object_list %} + <tr onclick="location.href='{% url 'task_view' task.id %}'"> + <td>{{ task.repo }}</td> + <td>{{ task.get_status_display }}</td> + <td>{{ task.start_time }}</td> + <td>{{ task.end_time }}</td> + <td>{{ task.run_time }}</td> + </tr> + {% endfor %} + <table> + </div> +{% endblock %}
\ No newline at end of file diff --git a/benchmarks/plan/sample.yaml b/resources/ansible_roles/ceph-cache-info/meta/main.yml index 04e8caf9..29b2992e 100644 --- a/benchmarks/plan/sample.yaml +++ b/resources/ansible_roles/ceph-cache-info/meta/main.yml @@ -6,9 +6,9 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -name: sample -description: sample benchmark plan for testing default path -config: - collectors: [] - reporters: [] -QPIs: [] + +--- + +allow_duplicates: yes +dependencies: +- { role: qtip-common, basename: ceph-cache-info } diff --git a/resources/ansible_roles/ceph-cache-info/tasks/main.yml b/resources/ansible_roles/ceph-cache-info/tasks/main.yml new file mode 100644 index 00000000..7e29a325 --- /dev/null +++ b/resources/ansible_roles/ceph-cache-info/tasks/main.yml @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- fetch: + src: /etc/ceph/ceph.conf + dest: "{{ qtip_results }}/sysinfo-{{ inventory_hostname }}/" + flat: yes + +- name: collect ceph catch info from ceph.conf + set_fact: {"{{ item }}":"{{ lookup('ini', '{{ item }} section=client default='' file={{ qtip_results }}/sysinfo-{{ inventory_hostname }}/ceph.conf') }}"} + with_items: + - rbd_cache + - rbd_cache_max_dirty + - rdb_cache_size + delegate_to: localhost + ignore_errors: True + register: result + +- name: saving output to log + copy: + content: "{{ item }} {{ item.stdout }}" + dest: "{{ logfile }}" + when: result|succeeded + delegate_to: localhost + with_items: + - rbd_cache + - rbd_cache_max_dirty + - rdb_cache_size diff --git a/benchmarks/metric/dpi.yaml b/resources/ansible_roles/ceph-network-info/meta/main.yml index c90075d2..4f5b88d4 100644 --- a/benchmarks/metric/dpi.yaml +++ b/resources/ansible_roles/ceph-network-info/meta/main.yml @@ -1,13 +1,14 @@ ############################################################################## -# Copyright (c) 2016 ZTE Corporation and others. +# Copyright (c) 2017 ZTE Corporation and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -name: dpi -description: deep packet inspection -workloads: - - bps - - pps + +--- + +allow_duplicates: yes +dependencies: + - { role: qtip-common, basename: ceph-network-info } diff --git a/resources/ansible_roles/ceph-network-info/tasks/main.yml b/resources/ansible_roles/ceph-network-info/tasks/main.yml new file mode 100644 index 00000000..7216d1a5 --- /dev/null +++ b/resources/ansible_roles/ceph-network-info/tasks/main.yml @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2017 ZTE Corporation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +- name: get system network info + shell: netstat -nr |grep "U" | awk '{printf "interface %s network %s mask %s \n",$8,$1, $3 }' + register: ceph_network_out + +- name: saving output to log + copy: + content: "{{ ceph_network_out.stdout }}" + dest: "{{ logfile }}" + delegate_to: localhost + +- fetch: + src: /etc/ceph/ceph.conf + dest: "{{ qtip_results }}/sysinfo-{{ inventory_hostname }}/" + flat: yes + +- name: collect ceph system info from ceph.conf + set_fact: {"{{ item }}":"{{ lookup('ini', '{{ item }} section=global default='' file={{ qtip_results }}/sysinfo-{{ inventory_hostname }}/ceph.conf') }}"} + with_items: + - cluster_network + delegate_to: localhost + +- name: append + lineinfile: + dest: "{{ logfile }}" + line: "cluster_network {{ cluster_network }}" + insertafter: EOF + delegate_to: localhost diff --git a/resources/ansible_roles/inxi/tasks/main.yml b/resources/ansible_roles/inxi/tasks/main.yml index c1f0b28c..af0f39e1 100644 --- a/resources/ansible_roles/inxi/tasks/main.yml +++ b/resources/ansible_roles/inxi/tasks/main.yml @@ -1,4 +1,4 @@ -############################################################################## + ############################################################################## # Copyright (c) 2017 ZTE Corporation and others. # # All rights reserved. This program and the accompanying materials diff --git a/tests/conftest.py b/tests/conftest.py index ddec94b8..8e812d66 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,9 +11,6 @@ from os import path import pytest -from qtip.loader.plan import Plan -from qtip.loader.plan import PlanProp - @pytest.fixture(scope='session') def data_root(): @@ -26,26 +23,6 @@ def benchmarks_root(data_root): @pytest.fixture(scope='session') -def plan(benchmarks_root): - return Plan('doctor.yaml', [benchmarks_root]) - - -@pytest.fixture(scope='session') -def plan_config(plan): - return plan.content[PlanProp.CONFIG] - - -@pytest.fixture(scope='session') -def collectors_config(plan_config): - return plan_config[PlanProp.COLLECTORS] - - -@pytest.fixture(scope='session') -def logfile_config(collectors_config): - return collectors_config[0] - - -@pytest.fixture(scope='session') def metrics(): return { "ssl_rsa": { diff --git a/tests/data/external/sysinfo/ceph-cache-info.log b/tests/data/external/sysinfo/ceph-cache-info.log new file mode 100644 index 00000000..6762f67b --- /dev/null +++ b/tests/data/external/sysinfo/ceph-cache-info.log @@ -0,0 +1,3 @@ +rdb_cache true +rdb_cache_size 24 +rbd_cache_max_dirty 16 diff --git a/tests/data/external/sysinfo/ceph-network-info.log b/tests/data/external/sysinfo/ceph-network-info.log new file mode 100644 index 00000000..5856ff52 --- /dev/null +++ b/tests/data/external/sysinfo/ceph-network-info.log @@ -0,0 +1,7 @@ +interface eth2 network 0.0.0.0 mask 0.0.0.0 +interface eth1 network 11.0.0.0 mask 255.255.255.0 +interface eth3 network 12.0.0.0 mask 255.255.255.0 +interface eth0 network 169.254.169.254 mask 255.255.255.255 +interface eth0 network 192.0.2.0 mask 255.255.255.0 +interface eth2 network 192.168.37.0 mask 255.255.255.0 +cluster_network 12.0.0.22/24 diff --git a/tests/unit/api/metric_controller_test.py b/tests/unit/api/metric_controller_test.py deleted file mode 100644 index caba7972..00000000 --- a/tests/unit/api/metric_controller_test.py +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import httplib -import json - -from qtip.base.constant import BaseProp - - -def test_get_list_metrics(app_client): - response_success = app_client.get("/v1.0/metrics") - assert response_success.status_code == httplib.OK - metric_list = json.loads(response_success.data)['metrics'] - assert len(metric_list) > 0 - assert metric_list[0].endswith('.yaml') - - -def test_get_metric(app_client): - response_success = app_client.get("/v1.0/metrics/dpi.yaml") - assert response_success.status_code == httplib.OK - metric_data = json.loads(response_success.data) - assert BaseProp.NAME in metric_data - assert BaseProp.WORKLOADS in metric_data - assert isinstance(metric_data[BaseProp.WORKLOADS], list) - - -def test_get_metric_not_found(app_client): - response_not_found = app_client.get("/v1.0/metrics/fake.yaml") - response_data = json.loads(response_not_found.data) - assert response_not_found.status_code == httplib.NOT_FOUND - assert response_data['title'] == "Metric not found" diff --git a/tests/unit/api/plan_controller_test.py b/tests/unit/api/plan_controller_test.py deleted file mode 100644 index 136bd3c6..00000000 --- a/tests/unit/api/plan_controller_test.py +++ /dev/null @@ -1,49 +0,0 @@ -############################################################################## -# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import httplib -import json - - -from qtip.loader.plan import PlanProp - - -def test_invalid_url(app_client): - response_url_not_found = app_client.get("/v1.0/fakeresource") - assert response_url_not_found.status_code == httplib.NOT_FOUND - - -def test_get_list_plans(app_client): - response_success = app_client.get("/v1.0/plans") - assert response_success.status_code == httplib.OK - plan_list = json.loads(response_success.data)['plans'] - assert len(plan_list) > 0 - assert plan_list[0].endswith('.yaml') - - -def test_get_plan(app_client): - response_success = app_client.get("/v1.0/plans/sample.yaml") - assert response_success.status_code == httplib.OK - plan_data = json.loads(response_success.data) - assert PlanProp.NAME in plan_data - assert PlanProp.DESCRIPTION in plan_data - assert PlanProp.CONFIG in plan_data - assert PlanProp.QPIS in plan_data - - -def test_get_plan_not_found(app_client): - response_not_found = app_client.get("/v1.0/plans/fake.yaml") - response_data = json.loads(response_not_found.data) - assert response_not_found.status_code == httplib.NOT_FOUND - assert response_data['title'] == "Plan not found" - - -def test_runner_not_implemented(app_client): - response_error = app_client.post("/v1.0/plans/fake.yaml?action=run", follow_redirects=False) - assert response_error.status_code == httplib.NOT_IMPLEMENTED diff --git a/tests/unit/cli/cmd_metric_test.py b/tests/unit/cli/cmd_metric_test.py deleted file mode 100644 index c92e944b..00000000 --- a/tests/unit/cli/cmd_metric_test.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################### -# Copyright (c) 2017 taseer94@gmail.com and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from click.testing import CliRunner -from qtip.cli.entry import cli - - -@pytest.fixture(scope="module") -def runner(): - return CliRunner() - - -def test_list(runner): - result = runner.invoke(cli, ['metric', 'list']) - assert 'dhrystone' and 'whetstone' and 'dpi' and \ - 'ramspeed' and 'fake-metric' and 'ssl' \ - in result.output - - -def test_run(runner): - result = runner.invoke(cli, ['metric', 'run', 'fake-metric']) - assert result.output == '' - - result = runner.invoke(cli, ['metric', 'run']) - assert 'Missing argument "name".' in result.output - - -def test_show(runner): - result = runner.invoke(cli, ['metric', 'show', 'dhrystone']) - assert 'Name: dhrystone' in result.output - assert 'Description: A synthetic computing benchmark program intended to be representative of' \ - 'system (integer) programming.' - - result = runner.invoke(cli, ['metric', 'show']) - assert 'Missing argument "name".' in result.output - - result = runner.invoke(cli, ['metric', 'show', 'xyz']) - assert "ERROR: metric spec: xyz not found" in result.output diff --git a/tests/unit/collector/collector_test.py b/tests/unit/collector/collector_test.py deleted file mode 100644 index 17fe1af1..00000000 --- a/tests/unit/collector/collector_test.py +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from qtip.loader.plan import load_collector -from qtip.collector import CollectorProp as CProp - - -def test_load_collector(collectors_config): - for c in collectors_config: - collector = load_collector(c[CProp.TYPE]) - assert collector.TYPE == c[CProp.TYPE] diff --git a/tests/unit/collector/logfile_test.py b/tests/unit/collector/logfile_test.py deleted file mode 100644 index a76aa3ee..00000000 --- a/tests/unit/collector/logfile_test.py +++ /dev/null @@ -1,33 +0,0 @@ -############################################################################## -# Copyright (c) 2017 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.collector.logfile import LogfileCollector - - -@pytest.fixture -def logfile_collector(logfile_config, plan): - return LogfileCollector(logfile_config, plan) - - -def test_run(logfile_collector): - collected = logfile_collector.run() - assert collected['groupdict'] == { - 'event_posted': '1482894965.3', - 'host_down': '1482894965.51', - 'network_down': '1482894965.164096803', - 'notified': '1482894965.63', - 'vm_error': '1482894965.3' - } - assert list(collected['groups']) == ['1482894965.63', - '1482894965.3', - '1482894965.3', - '1482894965.51', - '1482894965.164096803'] diff --git a/tests/unit/loader/metric_test.py b/tests/unit/loader/metric_test.py deleted file mode 100644 index 619d5e00..00000000 --- a/tests/unit/loader/metric_test.py +++ /dev/null @@ -1,43 +0,0 @@ -############################################################### -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.base.constant import BaseProp -from qtip.loader.metric import MetricSpec - - -@pytest.fixture(scope='module') -def metric_spec(benchmarks_root): - return MetricSpec('dhrystone.yaml', paths=[benchmarks_root]) - - -def init_test(metric_spec): - assert metric_spec.name == 'dhrystone' - - with pytest.raises(TypeError) as excinfo: - MetricSpec() - assert '__init__() takes at least 2 arguments (1 given)' \ - in str(excinfo.value) - - -def list_all_test(benchmarks_root): - metric_list = list(MetricSpec.list_all(paths=[benchmarks_root])) - assert len(metric_list) is 6 - for desc in metric_list: - assert BaseProp.NAME in desc - assert BaseProp.ABSPATH in desc - assert BaseProp.ABSPATH is not None - - -def content_test(metric_spec): - content = metric_spec.content - assert BaseProp.NAME in content - assert BaseProp.WORKLOADS in content - assert isinstance(content[BaseProp.WORKLOADS], list) diff --git a/tests/unit/loader/plan_test.py b/tests/unit/loader/plan_test.py deleted file mode 100644 index 4c92e8d5..00000000 --- a/tests/unit/loader/plan_test.py +++ /dev/null @@ -1,52 +0,0 @@ -############################################################################## -# Copyright (c) 2016 ZTE Corp and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import pytest - -from qtip.collector.logfile import LogfileCollector -from qtip.loader.plan import load_collector -from qtip.loader.plan import Plan -from qtip.loader.plan import PlanProp - - -def test_construct(benchmarks_root): - sample = Plan('sample.yaml') - assert isinstance(sample, Plan) - - # fixture can not be used in pytest.mark.parametrized - sample = Plan('sample.yaml', [benchmarks_root]) - assert isinstance(sample, Plan) - - -def test_invalid_construct(): - with pytest.raises(TypeError) as excinfo: - Plan() - assert '__init__() takes at least 2 arguments (1 given)' \ - in str(excinfo.value) - - -def test_list_all(benchmarks_root): - plan_list = list(Plan.list_all(paths=[benchmarks_root])) - assert len(plan_list) is 3 - for desc in plan_list: - assert PlanProp.NAME in desc - assert PlanProp.ABSPATH in desc - assert PlanProp.ABSPATH is not None - - -def test_content(plan): - content = plan.content - assert PlanProp.NAME in content - assert PlanProp.DESCRIPTION in content - assert PlanProp.CONFIG in content - assert PlanProp.QPIS in content - - -def test_load_collector(): - assert load_collector(LogfileCollector.TYPE) is LogfileCollector |