diff options
Diffstat (limited to 'api')
| -rw-r--r-- | api/database/v2/models.py | 1 | ||||
| -rw-r--r-- | api/resources/v1/env.py | 26 | ||||
| -rw-r--r-- | api/resources/v1/testsuites.py | 3 | ||||
| -rw-r--r-- | api/resources/v2/environments.py | 35 | ||||
| -rw-r--r-- | api/resources/v2/images.py | 69 | ||||
| -rw-r--r-- | api/resources/v2/openrcs.py | 11 | ||||
| -rw-r--r-- | api/resources/v2/tasks.py | 24 | ||||
| -rw-r--r-- | api/server.py | 4 | ||||
| -rw-r--r-- | api/urls.py | 1 | ||||
| -rw-r--r-- | api/utils/influx.py | 41 |
10 files changed, 133 insertions, 82 deletions
diff --git a/api/database/v2/models.py b/api/database/v2/models.py index 59dab3ebc..0ee811698 100644 --- a/api/database/v2/models.py +++ b/api/database/v2/models.py @@ -92,6 +92,7 @@ class V2Task(Base): case_name = Column(String(30)) suite = Column(Boolean) content = Column(Text) + params = Column(Text) result = Column(Text) error = Column(Text) status = Column(Integer) diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py index 7c831fd74..6c9eb8324 100644 --- a/api/resources/v1/env.py +++ b/api/resources/v1/env.py @@ -10,18 +10,24 @@ from __future__ import absolute_import import errno import logging + +import ipaddress import os import subprocess import threading import time import uuid import glob + +import six import yaml import collections from six.moves import configparser from oslo_serialization import jsonutils from docker import Client +from docker.errors import APIError +from requests.exceptions import HTTPError from api.database.v1.handlers import AsyncTaskHandler from api.utils import influx @@ -44,7 +50,7 @@ class V1Env(ApiResource): def post(self): return self._dispatch_post() - def create_grafana(self, args): + def create_grafana(self, *args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=self._create_grafana, args=(task_id,)) @@ -82,7 +88,7 @@ class V1Env(ApiResource): self._update_task_status(task_id) LOG.info('Finished') - except Exception as e: + except (APIError, HTTPError) as e: self._update_task_error(task_id, str(e)) LOG.exception('Create grafana failed') @@ -117,7 +123,7 @@ class V1Env(ApiResource): "isDefault": True, } try: - HttpClient().post(url, data, timeout=10) + HttpClient().post(url, data, timeout=60) except Exception: LOG.exception('Create datasources failed') raise @@ -145,7 +151,7 @@ class V1Env(ApiResource): return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags']) - def create_influxdb(self, args): + def create_influxdb(self, *args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=self._create_influxdb, args=(task_id,)) @@ -185,7 +191,7 @@ class V1Env(ApiResource): self._update_task_status(task_id) LOG.info('Finished') - except Exception as e: + except APIError as e: self._update_task_error(task_id, str(e)) LOG.exception('Creating influxdb failed') @@ -217,7 +223,7 @@ class V1Env(ApiResource): consts.INFLUXDB_DB_NAME) client.create_database(consts.INFLUXDB_DB_NAME) LOG.info('Success to config influxDB') - except Exception: + except HTTPError: LOG.exception('Config influxdb failed') def _change_output_to_influxdb(self, ip): @@ -236,7 +242,7 @@ class V1Env(ApiResource): with open(consts.CONF_FILE, 'w') as f: parser.write(f) - def prepare_env(self, args): + def prepare_env(self, *args): task_id = str(uuid.uuid4()) thread = threading.Thread(target=self._prepare_env_daemon, @@ -267,6 +273,8 @@ class V1Env(ApiResource): LOG.info('Openrc file not found') installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2') + # validate installer_ip is a valid ipaddress + installer_ip = str(ipaddress.IPv4Address(six.u(installer_ip))) installer_type = os.environ.get('INSTALLER_TYPE', 'compass') LOG.info('Getting openrc file from %s', installer_type) self._get_remote_rc_file(rc_file, @@ -287,7 +295,7 @@ class V1Env(ApiResource): self._update_task_status(task_id) LOG.info('Finished') - except Exception as e: + except (subprocess.CalledProcessError, OSError) as e: self._update_task_error(task_id, str(e)) LOG.exception('Prepare env failed') @@ -373,7 +381,7 @@ class V1Env(ApiResource): LOG.info('Source openrc: Sourcing') try: self._source_file(consts.OPENRC) - except Exception as e: + except subprocess.CalledProcessError as e: LOG.exception('Failed to source openrc') return result_handler(consts.API_ERROR, str(e)) LOG.info('Source openrc: Done') diff --git a/api/resources/v1/testsuites.py b/api/resources/v1/testsuites.py index 5f72c2ea6..3e14670b4 100644 --- a/api/resources/v1/testsuites.py +++ b/api/resources/v1/testsuites.py @@ -20,6 +20,7 @@ from yardstick.common.utils import result_handler from yardstick.benchmark.core import Param from yardstick.benchmark.core.task import Task from api.swagger import models +from api.database.v1.handlers import TasksHandler LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -58,7 +59,7 @@ class V1Testsuite(ApiResource): task_args.update(args.get('opts', {})) param = Param(task_args) - task_thread = TaskThread(Task().start, param) + task_thread = TaskThread(Task().start, param, TasksHandler()) task_thread.start() return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/v2/environments.py b/api/resources/v2/environments.py index 158e98be7..7e587be85 100644 --- a/api/resources/v2/environments.py +++ b/api/resources/v2/environments.py @@ -11,6 +11,7 @@ import logging from oslo_serialization import jsonutils from docker import Client +from docker.errors import APIError from api import ApiResource from api.database.v2.handlers import V2EnvironmentHandler @@ -20,6 +21,7 @@ from api.database.v2.handlers import V2ContainerHandler from yardstick.common.utils import result_handler from yardstick.common.utils import change_obj_to_dict from yardstick.common import constants as consts +from yardstick.service.environment import Environment LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -124,10 +126,41 @@ class V2Environment(ApiResource): LOG.debug('container name: %s', container.name) try: client.remove_container(container.name, force=True) - except Exception: + except APIError: LOG.exception('remove container failed') container_handler.delete_by_uuid(v) environment_handler.delete_by_uuid(environment_id) return result_handler(consts.API_SUCCESS, {'environment': environment_id}) + + +class V2SUT(ApiResource): + + def get(self, environment_id): + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + if not environment.pod_id: + return result_handler(consts.API_SUCCESS, {'sut': {}}) + + pod_handler = V2PodHandler() + try: + pod = pod_handler.get_by_uuid(environment.pod_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such pod id') + else: + pod_content = pod.content + + env = Environment(pod=pod_content) + sut_info = env.get_sut_info() + + return result_handler(consts.API_SUCCESS, {'sut': sut_info}) diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py index 0c36a0a26..c3e5ee73e 100644 --- a/api/resources/v2/images.py +++ b/api/resources/v2/images.py @@ -18,8 +18,7 @@ from api.database.v2.handlers import V2ImageHandler from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common.utils import result_handler from yardstick.common.utils import source_env -from yardstick.common.utils import change_obj_to_dict -from yardstick.common.openstack_utils import get_nova_client +from yardstick.common import openstack_utils from yardstick.common.openstack_utils import get_glance_client from yardstick.common import constants as consts @@ -47,39 +46,21 @@ class V2Images(ApiResource): def get(self): try: source_env(consts.OPENRC) - except Exception: + except OSError: return result_handler(consts.API_ERROR, 'source openrc error') - nova_client = get_nova_client() - try: - images_list = nova_client.images.list() - except Exception: + image_list = openstack_utils.list_images() + + if image_list is False: return result_handler(consts.API_ERROR, 'get images error') - else: - images = {i.name: self.get_info(change_obj_to_dict(i)) for i in images_list} + + images = {i.name: format_image_info(i) for i in image_list} return result_handler(consts.API_SUCCESS, {'status': 1, 'images': images}) def post(self): return self._dispatch_post() - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'discription': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result - def load_image(self, args): try: image_name = args['name'] @@ -268,7 +249,7 @@ class V2Images(ApiResource): r = requests.head(url) try: file_size = int(r.headers['content-length']) - except Exception: + except (TypeError, ValueError): return with open(path, 'wb') as f: @@ -303,14 +284,13 @@ class V2Image(ApiResource): except ValueError: return result_handler(consts.API_ERROR, 'no such image id') - nova_client = get_nova_client() - images = nova_client.images.list() + images = openstack_utils.list_images() try: image = next((i for i in images if i.name == image.name)) except StopIteration: pass - return_image = self.get_info(change_obj_to_dict(image)) + return_image = format_image_info(image) return_image['id'] = image_id return result_handler(consts.API_SUCCESS, {'image': return_image}) @@ -349,19 +329,16 @@ class V2Image(ApiResource): return result_handler(consts.API_SUCCESS, {'image': image_id}) - def get_info(self, data): - try: - size = data['OS-EXT-IMG-SIZE:size'] - except KeyError: - size = None - else: - size = float(size) / 1024 / 1024 - - result = { - 'name': data.get('name', ''), - 'description': data.get('description', ''), - 'size': size, - 'status': data.get('status'), - 'time': data.get('updated') - } - return result + +def format_image_info(image): + image_dict = {} + + if image is None: + return image_dict + + image_dict['name'] = image.name + image_dict['size'] = float(image.size) / 1024 / 1024 + image_dict['status'] = image.status.upper() + image_dict['time'] = image.updated_at + + return image_dict diff --git a/api/resources/v2/openrcs.py b/api/resources/v2/openrcs.py index cb506d0e8..4706b856a 100644 --- a/api/resources/v2/openrcs.py +++ b/api/resources/v2/openrcs.py @@ -21,6 +21,7 @@ from yardstick.common import constants as consts from yardstick.common.utils import result_handler from yardstick.common.utils import makedirs from yardstick.common.utils import source_env +from yardstick.common import exceptions as y_exc LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -57,7 +58,7 @@ class V2Openrcs(ApiResource): openrc_data = self._get_openrc_dict() except Exception: LOG.exception('parse openrc failed') - return result_handler(consts.API_ERROR, 'parse openrc failed') + raise y_exc.UploadOpenrcError() openrc_id = str(uuid.uuid4()) self._write_into_database(environment_id, openrc_id, openrc_data) @@ -67,7 +68,7 @@ class V2Openrcs(ApiResource): self._generate_ansible_conf_file(openrc_data) except Exception: LOG.exception('write cloud conf failed') - return result_handler(consts.API_ERROR, 'genarate ansible conf failed') + raise y_exc.UploadOpenrcError() LOG.info('finish writing ansible cloud conf') return result_handler(consts.API_SUCCESS, {'openrc': openrc_data, 'uuid': openrc_id}) @@ -102,7 +103,7 @@ class V2Openrcs(ApiResource): source_env(consts.OPENRC) except Exception: LOG.exception('source openrc failed') - return result_handler(consts.API_ERROR, 'source openrc failed') + raise y_exc.UpdateOpenrcError() LOG.info('source openrc: Done') openrc_id = str(uuid.uuid4()) @@ -113,7 +114,7 @@ class V2Openrcs(ApiResource): self._generate_ansible_conf_file(openrc_vars) except Exception: LOG.exception('write cloud conf failed') - return result_handler(consts.API_ERROR, 'genarate ansible conf failed') + raise y_exc.UpdateOpenrcError() LOG.info('finish writing ansible cloud conf') return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars, 'uuid': openrc_id}) @@ -174,7 +175,7 @@ class V2Openrcs(ApiResource): makedirs(consts.OPENSTACK_CONF_DIR) with open(consts.CLOUDS_CONF, 'w') as f: - yaml.dump(ansible_conf, f, default_flow_style=False) + yaml.safe_dump(ansible_conf, f, default_flow_style=False) class V2Openrc(ApiResource): diff --git a/api/resources/v2/tasks.py b/api/resources/v2/tasks.py index 25a9cf109..17241ed63 100644 --- a/api/resources/v2/tasks.py +++ b/api/resources/v2/tasks.py @@ -38,6 +38,8 @@ class V2Tasks(ApiResource): for t in tasks: result = t['result'] t['result'] = jsonutils.loads(result) if result else None + params = t['params'] + t['params'] = jsonutils.loads(params) if params else None return result_handler(consts.API_SUCCESS, {'tasks': tasks}) @@ -94,6 +96,9 @@ class V2Task(ApiResource): result = task_info['result'] task_info['result'] = jsonutils.loads(result) if result else None + params = task_info['params'] + task_info['params'] = jsonutils.loads(params) if params else None + return result_handler(consts.API_SUCCESS, {'task': task_info}) def delete(self, task_id): @@ -127,7 +132,6 @@ class V2Task(ApiResource): return result_handler(consts.API_SUCCESS, {'task': task_id}) def put(self, task_id): - try: uuid.UUID(task_id) except ValueError: @@ -166,6 +170,21 @@ class V2Task(ApiResource): return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + def add_params(self, args): + task_id = args['task_id'] + try: + params = args['params'] + except KeyError: + return result_handler(consts.API_ERROR, 'params must be provided') + + LOG.info('update params info in task') + + task_handler = V2TaskHandler() + task_update_data = {'params': jsonutils.dumps(params)} + task_handler.update_attr(task_id, task_update_data) + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + def add_case(self, args): task_id = args['task_id'] try: @@ -243,7 +262,8 @@ class V2Task(ApiResource): data = { 'inputfile': ['/tmp/{}.yaml'.format(task.case_name)], - 'task_id': task_id + 'task_id': task_id, + 'task-args': task.params } if task.suite: data.update({'suite': True}) diff --git a/api/server.py b/api/server.py index 37a1ab6a6..914fe8457 100644 --- a/api/server.py +++ b/api/server.py @@ -39,11 +39,13 @@ app.config['MAX_CONTENT_LENGTH'] = 2 * 1024 * 1024 * 1024 Swagger(app) -api = Api(app) +api = Api(app, errors=consts.API_ERRORS) @app.teardown_request def shutdown_session(exception=None): + if exception: + LOG.warning(exception.message) db_session.remove() diff --git a/api/urls.py b/api/urls.py index 4b8e39e8f..9f0abcade 100644 --- a/api/urls.py +++ b/api/urls.py @@ -26,6 +26,7 @@ urlpatterns = [ Url('/api/v2/yardstick/environments', 'v2_environments'), Url('/api/v2/yardstick/environments/action', 'v2_environments'), Url('/api/v2/yardstick/environments/<environment_id>', 'v2_environment'), + Url('/api/v2/yardstick/environments/<environment_id>/sut', 'v2_sut'), Url('/api/v2/yardstick/openrcs', 'v2_openrcs'), Url('/api/v2/yardstick/openrcs/action', 'v2_openrcs'), diff --git a/api/utils/influx.py b/api/utils/influx.py index 9bc6e9abe..8f3604745 100644 --- a/api/utils/influx.py +++ b/api/utils/influx.py @@ -1,53 +1,60 @@ ############################################################################## # Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# Copyright (c) 2019 Intel Corporation # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import absolute_import import logging -import six.moves.configparser as ConfigParser -from six.moves.urllib.parse import urlsplit -from influxdb import InfluxDBClient +from six.moves import configparser as ConfigParser +# NOTE(ralonsoh): pylint E0401 import error +# https://github.com/PyCQA/pylint/issues/1640 +from six.moves.urllib.parse import urlsplit # pylint: disable=relative-import +from influxdb import client as influxdb_client from yardstick.common import constants as consts +from yardstick.common import exceptions +from yardstick import dispatcher -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) -def get_data_db_client(): +def get_data_db_client(db=None): parser = ConfigParser.ConfigParser() try: parser.read(consts.CONF_FILE) - - if parser.get('DEFAULT', 'dispatcher') != 'influxdb': - raise RuntimeError - - return _get_client(parser) + return _get_influxdb_client(parser, db) except ConfigParser.NoOptionError: - logger.error('can not find the key') + logger.error('Can not find the key') raise +def _get_influxdb_client(parser, db=None): + if dispatcher.INFLUXDB not in parser.get('DEFAULT', 'dispatcher'): + raise exceptions.InfluxDBConfigurationMissing() -def _get_client(parser): ip = _get_ip(parser.get('dispatcher_influxdb', 'target')) user = parser.get('dispatcher_influxdb', 'username') password = parser.get('dispatcher_influxdb', 'password') - db_name = parser.get('dispatcher_influxdb', 'db_name') - return InfluxDBClient(ip, consts.INFLUXDB_PORT, user, password, db_name) + if db is None: + db_name = parser.get('dispatcher_influxdb', 'db_name') + else: + db_name = db + + return influxdb_client.InfluxDBClient(ip, consts.INFLUXDB_PORT, user, + password, db_name) def _get_ip(url): return urlsplit(url).hostname -def query(query_sql): +def query(query_sql, db=None): try: - client = get_data_db_client() + client = get_data_db_client(db) logger.debug('Start to query: %s', query_sql) return list(client.query(query_sql).get_points()) except RuntimeError: |