diff options
| -rw-r--r-- | api/base.py | 8 | ||||
| -rw-r--r-- | api/database/v1/handlers.py | 28 | ||||
| -rw-r--r-- | api/resources/asynctask.py | 51 | ||||
| -rw-r--r-- | api/resources/case_docs.py | 30 | ||||
| -rw-r--r-- | api/resources/env_action.py | 128 | ||||
| -rw-r--r-- | api/urls.py | 1 | ||||
| -rw-r--r-- | api/utils/common.py | 13 | ||||
| -rw-r--r-- | api/views.py | 6 | ||||
| -rw-r--r-- | docker/Dockerfile.aarch64.patch | 42 | ||||
| -rw-r--r-- | docs/testing/user/userguide/04-installation.rst | 2 | ||||
| -rwxr-xr-x | install.sh | 44 | ||||
| -rwxr-xr-x | tests/ci/clean_images.sh | 2 | ||||
| -rwxr-xr-x | tests/ci/load_images.sh | 14 | ||||
| -rwxr-xr-x | tests/ci/prepare_storperf_admin-rc.sh | 1 | ||||
| -rw-r--r-- | tests/unit/apiserver/resources/test_env_action.py | 2 | ||||
| -rw-r--r-- | tests/unit/apiserver/utils/test_common.py | 25 | ||||
| -rw-r--r-- | tests/unit/cmd/commands/test_env.py | 2 | ||||
| -rw-r--r-- | yardstick/benchmark/contexts/heat.py | 3 | ||||
| -rw-r--r-- | yardstick/benchmark/contexts/model.py | 1 | ||||
| -rw-r--r-- | yardstick/benchmark/core/task.py | 6 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/storage/storperf.py | 14 | ||||
| -rw-r--r-- | yardstick/cmd/commands/env.py | 6 | ||||
| -rw-r--r-- | yardstick/common/constants.py | 1 | ||||
| -rw-r--r-- | yardstick/orchestrator/heat.py | 8 |
24 files changed, 295 insertions, 143 deletions
diff --git a/api/base.py b/api/base.py index 6fa2777ce..0f1e76a57 100644 --- a/api/base.py +++ b/api/base.py @@ -15,6 +15,7 @@ from flask import request from flask_restful import Resource from api.utils import common as common_utils +from yardstick.common import constants as consts logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -47,8 +48,9 @@ class ApiResource(Resource): action, args = self._post_args() return self._dispatch(args, action) - def _dispatch_get(self): + def _dispatch_get(self, **kwargs): args = self._get_args() + args.update(kwargs) return self._dispatch(args) def _dispatch(self, args, action='default'): @@ -59,5 +61,5 @@ class ApiResource(Resource): resources = importlib.import_module(module_name) try: return getattr(resources, action)(args) - except NameError: - common_utils.error_handler('Wrong action') + except AttributeError: + common_utils.result_handler(consts.API_ERROR, 'No such action') diff --git a/api/database/v1/handlers.py b/api/database/v1/handlers.py index f7c448cda..938015d82 100644 --- a/api/database/v1/handlers.py +++ b/api/database/v1/handlers.py @@ -19,14 +19,6 @@ class TasksHandler(object): db_session.commit() return task - def update_status(self, task, status): - task.status = status - db_session.commit() - - def update_error(self, task, error): - task.error = error - db_session.commit() - def get_task_by_taskid(self, task_id): task = Tasks.query.filter_by(task_id=task_id).first() if not task: @@ -35,7 +27,7 @@ class TasksHandler(object): return task def update_attr(self, task_id, attr): - task = self.get_task_by_taskid(task_id) + task = self.get_task_by_taskid(task_id) for k, v in attr.items(): setattr(task, k, v) @@ -49,14 +41,16 @@ class AsyncTaskHandler(object): db_session.commit() return task - def update_status(self, task, status): - task.status = status - db_session.commit() - - def update_error(self, task, error): - task.error = error - db_session.commit() - def get_task_by_taskid(self, task_id): task = AsyncTasks.query.filter_by(task_id=task_id).first() + if not task: + raise ValueError + return task + + def update_attr(self, task_id, attr): + task = self.get_task_by_taskid(task_id) + + for k, v in attr.items(): + setattr(task, k, v) + db_session.commit() diff --git a/api/resources/asynctask.py b/api/resources/asynctask.py index 1f70501d2..39b47c0ee 100644 --- a/api/resources/asynctask.py +++ b/api/resources/asynctask.py @@ -7,9 +7,14 @@ # http://www.apache.org/licenses/LICENSE-2.0 # ############################################################################ import uuid +import logging -from api.utils import common as common_utils -from api.database.v1.models import AsyncTasks +from api.utils.common import result_handler +from api.database.v1.handlers import AsyncTaskHandler +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) def default(args): @@ -19,17 +24,41 @@ def default(args): def _get_status(args): try: task_id = args['task_id'] - uuid.UUID(task_id) except KeyError: - message = 'measurement and task_id must be provided' - return common_utils.error_handler(message) + return result_handler(consts.API_ERROR, 'task_id must be provided') - asynctask = AsyncTasks.query.filter_by(task_id=task_id).first() + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + asynctask_handler = AsyncTaskHandler() try: - status = asynctask.status - error = asynctask.error if asynctask.error else [] + asynctask = asynctask_handler.get_task_by_taskid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + def _unfinished(): + return result_handler(consts.TASK_NOT_DONE, {}) + + def _finished(): + return result_handler(consts.TASK_DONE, {}) + + def _error(): + return result_handler(consts.TASK_FAILED, asynctask.error) + + status = asynctask.status + LOG.debug('Task status is: %s', status) + + if status not in [consts.TASK_NOT_DONE, + consts.TASK_DONE, + consts.TASK_FAILED]: + return result_handler(consts.API_ERROR, 'internal server error') + + switcher = { + consts.TASK_NOT_DONE: _unfinished, + consts.TASK_DONE: _finished, + consts.TASK_FAILED: _error + } - return common_utils.result_handler(status, error) - except AttributeError: - return common_utils.error_handler('no such task') + return switcher.get(status)() diff --git a/api/resources/case_docs.py b/api/resources/case_docs.py new file mode 100644 index 000000000..289410d2d --- /dev/null +++ b/api/resources/case_docs.py @@ -0,0 +1,30 @@ +import os +import logging + +from api.utils.common import result_handler +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +def default(args): + return get_case_docs(args) + + +def get_case_docs(args): + try: + case_name = args['case_name'] + except KeyError: + return result_handler(consts.API_ERROR, 'case_name must be provided') + + docs_path = os.path.join(consts.DOCS_DIR, '{}.rst'.format(case_name)) + + if not os.path.exists(docs_path): + return result_handler(consts.API_ERROR, 'case not exists') + + LOG.info('Reading %s', case_name) + with open(docs_path) as f: + content = f.read() + + return result_handler(consts.API_SUCCESS, {'docs': content}) diff --git a/api/resources/env_action.py b/api/resources/env_action.py index 4eda73368..2ea64ef1a 100644 --- a/api/resources/env_action.py +++ b/api/resources/env_action.py @@ -32,17 +32,19 @@ from yardstick.common import openstack_utils from yardstick.common.httpClient import HttpClient -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) +async_handler = AsyncTaskHandler() -def createGrafanaContainer(args): + +def create_grafana(args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=_create_grafana, args=(task_id,)) thread.start() - return result_handler('success', {'task_id': task_id}) + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) def _create_grafana(task_id): @@ -51,22 +53,29 @@ def _create_grafana(task_id): client = Client(base_url=consts.DOCKER_URL) try: + LOG.info('Checking if grafana image exist') image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) if not _check_image_exist(client, image): + LOG.info('Grafana image not exist, start pulling') client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + LOG.info('Createing grafana container') _create_grafana_container(client) + LOG.info('Grafana container is created') time.sleep(5) + LOG.info('Creating data source for grafana') _create_data_source() + LOG.info('Creating dashboard for grafana') _create_dashboard() _update_task_status(task_id) + LOG.info('Finished') except Exception as e: _update_task_error(task_id, str(e)) - logger.exception('Error: %s', e) + LOG.exception('Create grafana failed') def _create_dashboard(): @@ -76,7 +85,11 @@ def _create_dashboard(): for i in sorted(glob.iglob(path)): with open(i) as f: data = jsonutils.load(f) - HttpClient().post(url, data) + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create dashboard %s failed', i) + raise def _create_data_source(): @@ -94,7 +107,11 @@ def _create_data_source(): "basicAuthPassword": "admin", "isDefault": False, } - HttpClient().post(url, data) + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create datasources failed') + raise def _create_grafana_container(client): @@ -104,12 +121,14 @@ def _create_grafana_container(client): host_config = client.create_host_config(port_bindings=port_bindings, restart_policy=restart_policy) + LOG.info('Creating container') container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE, consts.GRAFANA_TAG), ports=ports, detach=True, tty=True, host_config=host_config) + LOG.info('Starting container') client.start(container) @@ -117,13 +136,13 @@ def _check_image_exist(client, t): return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags']) -def createInfluxDBContainer(args): +def create_influxdb(args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=_create_influxdb, args=(task_id,)) thread.start() - return result_handler('success', {'task_id': task_id}) + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) def _create_influxdb(task_id): @@ -132,22 +151,30 @@ def _create_influxdb(task_id): client = Client(base_url=consts.DOCKER_URL) try: + LOG.info('Changing output to influxdb') _change_output_to_influxdb() + LOG.info('Checking if influxdb image exist') if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE, consts.INFLUXDB_TAG)): + LOG.info('Influxdb image not exist, start pulling') client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG) + LOG.info('Createing influxdb container') _create_influxdb_container(client) + LOG.info('Influxdb container is created') time.sleep(5) + LOG.info('Config influxdb') _config_influxdb() _update_task_status(task_id) + + LOG.info('Finished') except Exception as e: _update_task_error(task_id, str(e)) - logger.debug('Error: %s', e) + LOG.exception('Creating influxdb failed') def _create_influxdb_container(client): @@ -158,12 +185,14 @@ def _create_influxdb_container(client): host_config = client.create_host_config(port_bindings=port_bindings, restart_policy=restart_policy) + LOG.info('Creating container') container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE, consts.INFLUXDB_TAG), ports=ports, detach=True, tty=True, host_config=host_config) + LOG.info('Starting container') client.start(container) @@ -174,32 +203,35 @@ def _config_influxdb(): consts.INFLUXDB_PASS, consts.INFLUXDB_DB_NAME) client.create_database(consts.INFLUXDB_DB_NAME) - logger.info('Success to config influxDB') - except Exception as e: - logger.debug('Failed to config influxDB: %s', e) + LOG.info('Success to config influxDB') + except Exception: + LOG.exception('Config influxdb failed') def _change_output_to_influxdb(): common_utils.makedirs(consts.CONF_DIR) parser = configparser.ConfigParser() + LOG.info('Reading output sample configuration') parser.read(consts.CONF_SAMPLE_FILE) + LOG.info('Set dispatcher to influxdb') parser.set('DEFAULT', 'dispatcher', 'influxdb') parser.set('dispatcher_influxdb', 'target', 'http://%s:8086' % consts.INFLUXDB_IP) + LOG.info('Writing to %s', consts.CONF_FILE) with open(consts.CONF_FILE, 'w') as f: parser.write(f) -def prepareYardstickEnv(args): +def prepare_env(args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,)) thread.start() - return result_handler('success', {'task_id': task_id}) + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) def _already_source_openrc(): @@ -216,23 +248,33 @@ def _prepare_env_daemon(task_id): rc_file = consts.OPENRC + LOG.info('Checkout Openrc Environment variable') if not _already_source_openrc(): + LOG.info('Openrc variable not found in Environment') if not os.path.exists(rc_file): + LOG.info('Openrc file not found') installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2') installer_type = os.environ.get('INSTALLER_TYPE', 'compass') + LOG.info('Getting openrc file from %s', installer_type) _get_remote_rc_file(rc_file, installer_ip, installer_type) + LOG.info('Source openrc file') _source_file(rc_file) + LOG.info('Appending external network') _append_external_network(rc_file) + LOG.info('Openrc file exist, source openrc file') _source_file(rc_file) + LOG.info('Cleaning images') _clean_images() + LOG.info('Loading images') _load_images() _update_task_status(task_id) + LOG.info('Finished') except Exception as e: _update_task_error(task_id, str(e)) - logger.debug('Error: %s', e) + LOG.exception('Prepare env failed') def _create_directories(): @@ -254,7 +296,7 @@ def _get_remote_rc_file(rc_file, installer_ip, installer_type): p.communicate() if p.returncode != 0: - logger.debug('Failed to fetch credentials from installer') + LOG.error('Failed to fetch credentials from installer') except OSError as e: if e.errno != errno.EEXIST: raise @@ -266,7 +308,7 @@ def _append_external_network(rc_file): try: ext_network = next(n['name'] for n in networks if n['router:external']) except StopIteration: - logger.warning("Can't find external network") + LOG.warning("Can't find external network") else: cmd = 'export EXTERNAL_NETWORK=%s' % ext_network try: @@ -281,38 +323,26 @@ def _clean_images(): cmd = [consts.CLEAN_IMAGES_SCRIPT] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) output = p.communicate()[0] - logger.debug('The result is: %s', output) + LOG.debug(output) def _load_images(): cmd = [consts.LOAD_IMAGES_SCRIPT] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) output = p.communicate()[0] - logger.debug('The result is: %s', output) + LOG.debug(output) def _create_task(task_id): - async_handler = AsyncTaskHandler() - task_dict = { - 'task_id': task_id, - 'status': 0 - } - async_handler.insert(task_dict) + async_handler.insert({'status': 0, 'task_id': task_id}) def _update_task_status(task_id): - async_handler = AsyncTaskHandler() - - task = async_handler.get_task_by_taskid(task_id) - async_handler.update_status(task, 1) + async_handler.update_attr(task_id, {'status': 1}) def _update_task_error(task_id, error): - async_handler = AsyncTaskHandler() - - task = async_handler.get_task_by_taskid(task_id) - async_handler.update_status(task, 2) - async_handler.update_error(task, error) + async_handler.update_attr(task_id, {'status': 2, 'error': error}) def update_openrc(args): @@ -325,22 +355,22 @@ def update_openrc(args): return result_handler(consts.API_ERROR, 'args should be a dict') lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()] - logger.debug('Writing: %s', ''.join(lines)) + LOG.debug('Writing: %s', ''.join(lines)) - logger.info('Writing openrc: Writing') + LOG.info('Writing openrc: Writing') common_utils.makedirs(consts.CONF_DIR) with open(consts.OPENRC, 'w') as f: f.writelines(lines) - logger.info('Writing openrc: Done') + LOG.info('Writing openrc: Done') - logger.info('Source openrc: Sourcing') + LOG.info('Source openrc: Sourcing') try: _source_file(consts.OPENRC) except Exception as e: - logger.exception('Failed to source openrc') + LOG.exception('Failed to source openrc') return result_handler(consts.API_ERROR, str(e)) - logger.info('Source openrc: Done') + LOG.info('Source openrc: Done') return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars}) @@ -351,15 +381,15 @@ def upload_pod_file(args): except KeyError: return result_handler(consts.API_ERROR, 'file must be provided') - logger.info('Checking file') + LOG.info('Checking file') data = yaml.load(pod_file.read()) if not isinstance(data, collections.Mapping): return result_handler(consts.API_ERROR, 'invalid yaml file') - logger.info('Writing file') + LOG.info('Writing file') with open(consts.POD_FILE, 'w') as f: yaml.dump(data, f, default_flow_style=False) - logger.info('Writing finished') + LOG.info('Writing finished') return result_handler(consts.API_SUCCESS, {'pod_info': data}) @@ -373,10 +403,10 @@ def update_pod_file(args): if not isinstance(pod_dic, collections.Mapping): return result_handler(consts.API_ERROR, 'pod should be a dict') - logger.info('Writing file') + LOG.info('Writing file') with open(consts.POD_FILE, 'w') as f: yaml.dump(pod_dic, f, default_flow_style=False) - logger.info('Writing finished') + LOG.info('Writing finished') return result_handler(consts.API_SUCCESS, {'pod_info': pod_dic}) @@ -384,11 +414,11 @@ def update_pod_file(args): def update_hosts(hosts_ip): if not isinstance(hosts_ip, dict): return result_handler(consts.API_ERROR, 'Error, args should be a dict') - logger.info('Writing hosts: Writing') + LOG.info('Writing hosts: Writing') hosts_list = ['\n{} {}'.format(ip, host_name) for host_name, ip in hosts_ip.items()] - logger.debug('Writing: %s', hosts_list) + LOG.debug('Writing: %s', hosts_list) with open(consts.ETC_HOSTS, 'a') as f: f.writelines(hosts_list) - logger.info('Writing hosts: Done') + LOG.info('Writing hosts: Done') return result_handler(consts.API_SUCCESS, 'success') diff --git a/api/urls.py b/api/urls.py index b9ddd4c72..13c6c7675 100644 --- a/api/urls.py +++ b/api/urls.py @@ -17,6 +17,7 @@ urlpatterns = [ Url('/yardstick/testcases', views.Testcases, 'testcases'), Url('/yardstick/testcases/release/action', views.ReleaseAction, 'release'), Url('/yardstick/testcases/samples/action', views.SamplesAction, 'samples'), + Url('/yardstick/testcases/<case_name>/docs', views.CaseDocs, 'casedocs'), Url('/yardstick/testsuites/action', views.TestsuitesAction, 'testsuites'), Url('/yardstick/results', views.Results, 'results'), Url('/yardstick/env/action', views.EnvAction, 'env') diff --git a/api/utils/common.py b/api/utils/common.py index 8398b8f60..eda9c17dd 100644 --- a/api/utils/common.py +++ b/api/utils/common.py @@ -13,8 +13,8 @@ import logging from flask import jsonify import six -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) def translate_to_str(obj): @@ -27,15 +27,6 @@ def translate_to_str(obj): return obj -def error_handler(message): - logger.debug(message) - result = { - 'status': 'error', - 'message': message - } - return jsonify(result) - - def result_handler(status, data): result = { 'status': status, diff --git a/api/views.py b/api/views.py index 9fd236fad..9c9ca4ef9 100644 --- a/api/views.py +++ b/api/views.py @@ -74,3 +74,9 @@ class EnvAction(ApiResource): def post(self): return self._dispatch_post() + + +class CaseDocs(ApiResource): + + def get(self, case_name): + return self._dispatch_get(case_name=case_name) diff --git a/docker/Dockerfile.aarch64.patch b/docker/Dockerfile.aarch64.patch new file mode 100644 index 000000000..33c352a1b --- /dev/null +++ b/docker/Dockerfile.aarch64.patch @@ -0,0 +1,42 @@ +From: Alexandru Nemes <alexandru.nemes@enea.com> +Date: Mon, 19 Jun 2017 14:18:24 +0300 +Subject: [PATCH] Patch for Yardstick AARCH64 Docker file + +Signed-off-by: Alexandru Nemes <alexandru.nemes@enea.com> +--- + docker/Dockerfile | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/docker/Dockerfile b/docker/Dockerfile +index 96a5d77..03307a2 100644 +--- a/docker/Dockerfile ++++ b/docker/Dockerfile +@@ -1,5 +1,5 @@ + ############################################################################## +-# Copyright (c) 2015 Ericsson AB and others. ++# Copyright (c) 2017 Enea AB and others. + # + # All rights reserved. This program and the accompanying materials + # are made available under the terms of the Apache License, Version 2.0 +@@ -7,9 +7,9 @@ + # http://www.apache.org/licenses/LICENSE-2.0 + ############################################################################## + +-FROM ubuntu:14.04 ++FROM aarch64/ubuntu:14.04 + +-LABEL image=opnfv/yardstick ++LABEL image=opnfv/yardstick_aarch64 + + ARG BRANCH=master + +@@ -40,8 +40,8 @@ RUN echo "daemon off;" >> /etc/nginx/nginx.conf + + EXPOSE 5000 + +-ADD http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img ${IMAGE_DIR} +-ADD http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img ${IMAGE_DIR} ++ADD http://download.cirros-cloud.net/daily/20161201/cirros-d161201-aarch64-disk.img ${IMAGE_DIR} ++ADD http://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-disk1.img ${IMAGE_DIR} + + COPY ./exec_tests.sh /usr/local/bin/
\ No newline at end of file diff --git a/docs/testing/user/userguide/04-installation.rst b/docs/testing/user/userguide/04-installation.rst index 660f3b5a8..37e4ba599 100644 --- a/docs/testing/user/userguide/04-installation.rst +++ b/docs/testing/user/userguide/04-installation.rst @@ -215,6 +215,8 @@ follwing the commands below:: This image can be built using the following command in the directory where Yardstick is installed:: + export YARD_IMG_ARCH='amd64' + sudo echo "Defaults env_keep += \'YARD_IMG_ARCH\'" >> /etc/sudoers sudo tools/yardstick-img-modify tools/ubuntu-server-cloudimg-modify.sh **Warning:** Before building the guest image inside the Yardstick container, make sure the container is granted with privilege. The script will create files by default in ``/tmp/workspace/yardstick`` and the files will be owned by root! diff --git a/install.sh b/install.sh index b8212d2f4..ad14b8e0b 100755 --- a/install.sh +++ b/install.sh @@ -9,24 +9,44 @@ ############################################################################## # fit for arm64 +DOCKER_ARCH="$(uname -m)" + +UBUNTU_PORTS_URL="http://ports.ubuntu.com/ubuntu-ports/" +UBUNTU_ARCHIVE_URL="http://archive.ubuntu.com/ubuntu/" + source_file=/etc/apt/sources.list -sed -i -e 's/^deb \([^/[]\)/deb [arch=amd64] \1/g' "${source_file}" + +if [[ "${DOCKER_ARCH}" == "aarch64" ]]; then + sed -i -e 's/^deb \([^/[]\)/deb [arch=arm64] \1/g' "${source_file}" + DOCKER_ARCH="arm64" + DOCKER_REPO="${UBUNTU_PORTS_URL}" + EXTRA_ARCH="amd64" + EXTRA_REPO="${UBUNTU_ARCHIVE_URL}" + dpkg --add-architecture amd64 +else + sed -i -e 's/^deb \([^/[]\)/deb [arch=amd64] \1/g' "${source_file}" + DOCKER_ARCH="amd64" + DOCKER_REPO="${UBUNTU_ARCHIVE_URL}" + EXTRA_ARCH="arm64" + EXTRA_REPO="${UBUNTU_PORTS_URL}" + dpkg --add-architecture arm64 +fi + sed -i -e 's/^deb-src /# deb-src /g' "${source_file}" echo "APT::Default-Release \"trusty\";" > /etc/apt/apt.conf.d/default-distro sub_source_file=/etc/apt/sources.list.d/yardstick.list touch "${sub_source_file}" -echo -e "deb [arch=amd64] http://archive.ubuntu.com/ubuntu/ xenial-updates universe -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted -deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" > "${sub_source_file}" -echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf -proc_type=$(uname -m) -if [[ $proc_type == "arm"* ]]; then - dpkg --add-architecture arm64 -fi +# first add xenial repo needed for installing qemu_static_user/xenial in the container +# then add complementary architecture repositories in case the cloud image is of different arch +echo -e "deb [arch="${DOCKER_ARCH}"] "${DOCKER_REPO}" xenial-updates universe +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-updates main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-security main universe multiverse restricted +deb [arch="${EXTRA_ARCH}"] "${EXTRA_REPO}" trusty-proposed main universe multiverse restricted" > "${sub_source_file}" + +echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf # install tools apt-get update && apt-get install -y \ @@ -52,7 +72,7 @@ apt-get update && apt-get install -y \ python-pip \ vim -if [[ $proc_type == "arm"* ]]; then +if [[ "${DOCKER_ARCH}" != "aarch64" ]]; then apt-get install -y libc6:arm64 fi diff --git a/tests/ci/clean_images.sh b/tests/ci/clean_images.sh index b3363cb12..832ade252 100755 --- a/tests/ci/clean_images.sh +++ b/tests/ci/clean_images.sh @@ -25,7 +25,7 @@ cleanup() return fi - for image in $(openstack ${SECURE} image list | grep -e cirros-0.3.5 -e yardstick-image -e Ubuntu-16.04 \ + for image in $(openstack ${SECURE} image list | grep -e cirros-0.3.5 -e cirros-d161201 -e yardstick-image -e Ubuntu-16.04 \ | awk '{print $2}'); do echo "Deleting image $image..." openstack ${SECURE} image delete $image || true diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh index 29d73dcbc..c2ae4fe7b 100755 --- a/tests/ci/load_images.sh +++ b/tests/ci/load_images.sh @@ -150,13 +150,21 @@ load_yardstick_image() load_cirros_image() { - if [[ -n $(openstack ${SECURE} image list | grep -e Cirros-0.3.5) ]]; then - echo "Cirros-0.3.5 image already exist, skip loading cirros image" + if [[ "${YARD_IMG_ARCH}" == "arm64" ]]; then + CIRROS_IMAGE_VERSION="cirros-d161201" + CIRROS_IMAGE_PATH="/home/opnfv/images/cirros-d161201-aarch64-disk.img" + else + CIRROS_IMAGE_VERSION="Cirros-0.3.5" + CIRROS_IMAGE_PATH="/home/opnfv/images/cirros-0.3.5-x86_64-disk.img" + fi + + if [[ -n $(openstack ${SECURE} image list | grep -e "${CIRROS_IMAGE_VERSION}") ]]; then + echo "${CIRROS_IMAGE_VERSION} image already exist, skip loading cirros image" else echo echo "========== Loading cirros cloud image ==========" - local image_file=/home/opnfv/images/cirros-0.3.5-x86_64-disk.img + local image_file="${CIRROS_IMAGE_PATH}" EXTRA_PARAMS="" # VPP requires guest memory to be backed by large pages diff --git a/tests/ci/prepare_storperf_admin-rc.sh b/tests/ci/prepare_storperf_admin-rc.sh index 979728e84..558375e62 100755 --- a/tests/ci/prepare_storperf_admin-rc.sh +++ b/tests/ci/prepare_storperf_admin-rc.sh @@ -35,3 +35,4 @@ echo "OS_TENANT_ID="$TENANT_ID >> ~/storperf_admin-rc echo "OS_USER_DOMAIN_ID="$USER_DOMAIN_ID >> ~/storperf_admin-rc echo "OS_PROJECT_DOMAIN_NAME="$OS_PROJECT_DOMAIN_NAME >> ~/storperf_admin-rc echo "OS_USER_DOMAIN_NAME="$OS_USER_DOMAIN_NAME >> ~/storperf_admin-rc +echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> ~/storperf_admin-rc diff --git a/tests/unit/apiserver/resources/test_env_action.py b/tests/unit/apiserver/resources/test_env_action.py index c7ae10a20..d61092dbc 100644 --- a/tests/unit/apiserver/resources/test_env_action.py +++ b/tests/unit/apiserver/resources/test_env_action.py @@ -18,7 +18,7 @@ class EnvTestCase(APITestCase): def test_create_grafana(self): url = 'yardstick/env/action' - data = {'action': 'createGrafanaContainer'} + data = {'action': 'create_grafana'} resp = self._post(url, data) time.sleep(1) diff --git a/tests/unit/apiserver/utils/test_common.py b/tests/unit/apiserver/utils/test_common.py index acf6e41b1..ad81cb76b 100644 --- a/tests/unit/apiserver/utils/test_common.py +++ b/tests/unit/apiserver/utils/test_common.py @@ -33,31 +33,6 @@ class TranslateToStrTestCase(unittest.TestCase): self.assertEqual(result, output_str) -class GetCommandListTestCase(unittest.TestCase): - - def test_get_command_list_no_opts(self): - command_list = ['a'] - opts = {} - args = 'b' - output_list = common.get_command_list(command_list, opts, args) - - result_list = ['a', 'b'] - self.assertEqual(result_list, output_list) - - def test_get_command_list_with_opts_args(self): - command_list = ['a'] - opts = { - 'b': 'c', - 'task-args': 'd' - } - args = 'e' - - output_list = common.get_command_list(command_list, opts, args) - - result_list = ['a', 'e', '--b', '--task-args', 'd'] - self.assertEqual(result_list, output_list) - - def main(): unittest.main() diff --git a/tests/unit/cmd/commands/test_env.py b/tests/unit/cmd/commands/test_env.py index c6e0e1d20..73cd5af47 100644 --- a/tests/unit/cmd/commands/test_env.py +++ b/tests/unit/cmd/commands/test_env.py @@ -42,7 +42,7 @@ class EnvCommandTestCase(unittest.TestCase): @mock.patch('yardstick.cmd.commands.env.HttpClient.post') def test_start_async_task(self, post_mock): - data = {'action': 'createGrafanaContainer'} + data = {'action': 'create_grafana'} EnvCommand()._start_async_task(data) self.assertTrue(post_mock.called) diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index aa134d694..fed8fc342 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -152,7 +152,8 @@ class HeatContext(Context): for network in self.networks.values(): template.add_network(network.stack_name, network.physical_network, - network.provider) + network.provider, + network.segmentation_id) template.add_subnet(network.subnet_stack_name, network.stack_name, network.subnet_cidr) diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py index 1f8c6f11c..5077a9786 100644 --- a/yardstick/benchmark/contexts/model.py +++ b/yardstick/benchmark/contexts/model.py @@ -107,6 +107,7 @@ class Network(Object): self.router = None self.physical_network = attrs.get('physical_network', 'physnet1') self.provider = attrs.get('provider', None) + self.segmentation_id = attrs.get('segmentation_id', None) if "external_network" in attrs: self.router = Router("router", self.name, diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py index 9c6caf03f..d1a4d7561 100644 --- a/yardstick/benchmark/core/task.py +++ b/yardstick/benchmark/core/task.py @@ -250,10 +250,12 @@ class Task(object): # pragma: no cover # Nuke if it did not stop nicely base_runner.Runner.terminate(runner) status = runner_join(runner) - self.outputs.update(runner.get_output()) - result.extend(runner.get_result()) else: base_runner.Runner.release(runner) + + self.outputs.update(runner.get_output()) + result.extend(runner.get_result()) + if status != 0: raise RuntimeError print("Background task ended") diff --git a/yardstick/benchmark/scenarios/storage/storperf.py b/yardstick/benchmark/scenarios/storage/storperf.py index 672cfaba2..c10118ad1 100644 --- a/yardstick/benchmark/scenarios/storage/storperf.py +++ b/yardstick/benchmark/scenarios/storage/storperf.py @@ -8,6 +8,7 @@ ############################################################################## from __future__ import absolute_import +import os import logging import time @@ -147,7 +148,18 @@ class StorPerf(base.Scenario): if not self.setup_done: self.setup() - job_args = {} + metadata = {"build_tag": "latest", "test_case": "opnfv_yardstick_tc074"} + metadata_payload_dict = {"pod_name": "NODE_NAME", + "scenario_name": "DEPLOY_SCENARIO", + "version": "YARDSTICK_BRANCH"} + + for key, value in metadata_payload_dict.items(): + try: + metadata[key] = os.environ[value] + except KeyError: + pass + + job_args = {"metadata": metadata} job_args_payload_list = ["block_sizes", "queue_depths", "deadline", "target", "nossd", "nowarm", "workload"] diff --git a/yardstick/cmd/commands/env.py b/yardstick/cmd/commands/env.py index d5aef7faf..dbef303b0 100644 --- a/yardstick/cmd/commands/env.py +++ b/yardstick/cmd/commands/env.py @@ -26,21 +26,21 @@ class EnvCommand(object): """ def do_influxdb(self, args): - data = {'action': 'createInfluxDBContainer'} + data = {'action': 'create_influxdb'} task_id = self._start_async_task(data) start = '* creating influxDB' self._check_status(task_id, start) def do_grafana(self, args): - data = {'action': 'createGrafanaContainer'} + data = {'action': 'create_grafana'} task_id = self._start_async_task(data) start = '* creating grafana' self._check_status(task_id, start) def do_prepare(self, args): - data = {'action': 'prepareYardstickEnv'} + data = {'action': 'prepare_env'} task_id = self._start_async_task(data) start = '* preparing yardstick environment' diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py index d445e86e6..d251341fc 100644 --- a/yardstick/common/constants.py +++ b/yardstick/common/constants.py @@ -39,6 +39,7 @@ ANSIBLE_DIR = join(REPOS_DIR, 'ansible') SAMPLE_CASE_DIR = join(REPOS_DIR, 'samples') TESTCASE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_cases/') TESTSUITE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_suites/') +DOCS_DIR = join(REPOS_DIR, 'docs/testing/user/userguide/') # file OPENRC = get_param('file.openrc', '/etc/yardstick/openstack.creds') diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py index a99d4631d..fd6c4f6ff 100644 --- a/yardstick/orchestrator/heat.py +++ b/yardstick/orchestrator/heat.py @@ -1,4 +1,4 @@ -############################################################################## +############################################################################# # Copyright (c) 2015-2017 Ericsson AB and others. # # All rights reserved. This program and the accompanying materials @@ -230,7 +230,8 @@ name (i.e. %s).\ 'value': {'get_resource': name} } - def add_network(self, name, physical_network='physnet1', provider=None): + def add_network(self, name, physical_network='physnet1', provider=None, + segmentation_id=None): """add to the template a Neutron Net""" log.debug("adding Neutron::Net '%s'", name) if provider is None: @@ -247,6 +248,9 @@ name (i.e. %s).\ 'physical_network': physical_network } } + if segmentation_id: + seg_id_dit = {'segmentation_id': segmentation_id} + self.resources[name]["properties"].update(seg_id_dit) def add_server_group(self, name, policies): # pragma: no cover """add to the template a ServerGroup""" |