diff options
Diffstat (limited to 'api')
35 files changed, 2670 insertions, 1053 deletions
diff --git a/api/__init__.py b/api/__init__.py index c6cbbf104..c5aefffe8 100644 --- a/api/__init__.py +++ b/api/__init__.py @@ -1,4 +1,67 @@ -from yardstick import _init_logging +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import +import logging + +from flask import request +from flask_restful import Resource +from yardstick import _init_logging +from yardstick.common import constants as consts +from yardstick.common import utils as common_utils _init_logging() +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class ApiResource(Resource): + + def _post_args(self): + data = request.json if request.json else {} + params = common_utils.translate_to_str(data) + action = params.get('action', request.form.get('action', '')) + args = params.get('args', {}) + + try: + args['file'] = request.files['file'] + except KeyError: + pass + + args.update({k: v for k, v in request.form.items()}) + LOG.debug('Input args is: action: %s, args: %s', action, args) + + return action, args + + def _get_args(self): + args = common_utils.translate_to_str(request.args) + LOG.debug('Input args is: args: %s', args) + + return args + + def _dispatch_post(self, **kwargs): + action, args = self._post_args() + args.update(kwargs) + return self._dispatch(args, action) + + def _dispatch(self, args, action): + try: + return getattr(self, action)(args) + except AttributeError: + common_utils.result_handler(consts.API_ERROR, 'No such action') + + +class Url(object): + + def __init__(self, url, target): + super(Url, self).__init__() + self.url = url + self.target = target + +common_utils.import_modules_from_package("api.resources") diff --git a/api/api-prepare.sh b/api/api-prepare.sh deleted file mode 100755 index 7632d9da9..000000000 --- a/api/api-prepare.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -: ${YARDSTICK_REPO_DIR:='/home/opnfv/repos/yardstick'} - -# generate uwsgi config file -mkdir -p /etc/yardstick -uwsgi_config='/etc/yardstick/yardstick.ini' -if [[ ! -e "${uwsgi_config}" ]];then - - cat << EOF > "${uwsgi_config}" -[uwsgi] -master = true -debug = true -chdir = ${YARDSTICK_REPO_DIR}/api -module = server -plugins = python -processes = 10 -threads = 5 -async = true -max-requests = 5000 -chmod-socket = 666 -callable = app_wrapper -enable-threads = true -close-on-exec = 1 -daemonize= /var/log/yardstick/uwsgi.log -socket = /var/run/yardstick.sock -EOF - if [[ "${YARDSTICK_VENV}" ]];then - echo "virtualenv = ${YARDSTICK_VENV}" >> "${uwsgi_config}" - fi -fi - -# nginx config -nginx_config='/etc/nginx/conf.d/yardstick.conf' - -if [[ ! -e "${nginx_config}" ]];then - - cat << EOF > "${nginx_config}" -server { - listen 5000; - server_name localhost; - index index.htm index.html; - location / { - include uwsgi_params; - uwsgi_pass unix:///var/run/yardstick.sock; - } -} -EOF -fi - -# nginx service start when boot -supervisor_config='/etc/supervisor/conf.d/yardstick.conf' - -if [[ ! -e "${supervisor_config}" ]];then - cat << EOF > "${supervisor_config}" -[supervisord] -nodaemon = true - -[program:yardstick_nginx] -user = root -command = service nginx restart -autorestart = true - -[program:yardstick_uwsgi] -user = root -directory = /etc/yardstick -command = uwsgi -i yardstick.ini -autorestart = true -EOF -fi - -# create api log directory -mkdir -p /var/log/yardstick - -# create yardstick.sock for communicating -touch /var/run/yardstick.sock diff --git a/api/base.py b/api/base.py deleted file mode 100644 index 0f1e76a57..000000000 --- a/api/base.py +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import re -import importlib -import logging - -from flask import request -from flask_restful import Resource - -from api.utils import common as common_utils -from yardstick.common import constants as consts - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class ApiResource(Resource): - - def _post_args(self): - data = request.json if request.json else {} - params = common_utils.translate_to_str(data) - action = params.get('action', request.form.get('action', '')) - args = params.get('args', {}) - - try: - args['file'] = request.files['file'] - except KeyError: - pass - - logger.debug('Input args is: action: %s, args: %s', action, args) - - return action, args - - def _get_args(self): - args = common_utils.translate_to_str(request.args) - logger.debug('Input args is: args: %s', args) - - return args - - def _dispatch_post(self): - action, args = self._post_args() - return self._dispatch(args, action) - - def _dispatch_get(self, **kwargs): - args = self._get_args() - args.update(kwargs) - return self._dispatch(args) - - def _dispatch(self, args, action='default'): - module_name = re.sub(r'([A-Z][a-z]*)', r'_\1', - self.__class__.__name__)[1:].lower() - - module_name = 'api.resources.%s' % module_name - resources = importlib.import_module(module_name) - try: - return getattr(resources, action)(args) - except AttributeError: - common_utils.result_handler(consts.API_ERROR, 'No such action') diff --git a/api/database/v2/__init__.py b/api/database/v2/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/database/v2/__init__.py diff --git a/api/database/v2/handlers.py b/api/database/v2/handlers.py new file mode 100644 index 000000000..1bc32bf0e --- /dev/null +++ b/api/database/v2/handlers.py @@ -0,0 +1,200 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from api.database import db_session +from api.database.v2.models import V2Environment +from api.database.v2.models import V2Openrc +from api.database.v2.models import V2Image +from api.database.v2.models import V2Pod +from api.database.v2.models import V2Container +from api.database.v2.models import V2Project +from api.database.v2.models import V2Task + + +class V2EnvironmentHandler(object): + + def insert(self, kwargs): + environment = V2Environment(**kwargs) + db_session.add(environment) + db_session.commit() + return environment + + def list_all(self): + return V2Environment.query.all() + + def get_by_uuid(self, uuid): + environment = V2Environment.query.filter_by(uuid=uuid).first() + if not environment: + raise ValueError + return environment + + def update_attr(self, uuid, attr): + environment = self.get_by_uuid(uuid) + for k, v in attr.items(): + setattr(environment, k, v) + db_session.commit() + + def append_attr(self, uuid, attr): + environment = self.get_by_uuid(uuid) + for k, v in attr.items(): + value = getattr(environment, k) + new = '{},{}'.format(value, v) if value else v + setattr(environment, k, new) + db_session.commit() + + def delete_by_uuid(self, uuid): + environment = self.get_by_uuid(uuid) + db_session.delete(environment) + db_session.commit() + + +class V2OpenrcHandler(object): + + def insert(self, kwargs): + openrc = V2Openrc(**kwargs) + db_session.add(openrc) + db_session.commit() + return openrc + + def get_by_uuid(self, uuid): + openrc = V2Openrc.query.filter_by(uuid=uuid).first() + if not openrc: + raise ValueError + return openrc + + def delete_by_uuid(self, uuid): + openrc = self.get_by_uuid(uuid) + db_session.delete(openrc) + db_session.commit() + + +class V2ImageHandler(object): + + def insert(self, kwargs): + image = V2Image(**kwargs) + db_session.add(image) + db_session.commit() + return image + + def get_by_uuid(self, uuid): + image = V2Image.query.filter_by(uuid=uuid).first() + if not image: + raise ValueError + return image + + +class V2PodHandler(object): + + def insert(self, kwargs): + pod = V2Pod(**kwargs) + db_session.add(pod) + db_session.commit() + return pod + + def get_by_uuid(self, uuid): + pod = V2Pod.query.filter_by(uuid=uuid).first() + if not pod: + raise ValueError + return pod + + def delete_by_uuid(self, uuid): + pod = self.get_by_uuid(uuid) + db_session.delete(pod) + db_session.commit() + + +class V2ContainerHandler(object): + + def insert(self, kwargs): + container = V2Container(**kwargs) + db_session.add(container) + db_session.commit() + return container + + def get_by_uuid(self, uuid): + container = V2Container.query.filter_by(uuid=uuid).first() + if not container: + raise ValueError + return container + + def update_attr(self, uuid, attr): + container = self.get_by_uuid(uuid) + for k, v in attr.items(): + setattr(container, k, v) + db_session.commit() + + def delete_by_uuid(self, uuid): + container = self.get_by_uuid(uuid) + db_session.delete(container) + db_session.commit() + + +class V2ProjectHandler(object): + + def list_all(self): + return V2Project.query.all() + + def insert(self, kwargs): + project = V2Project(**kwargs) + db_session.add(project) + db_session.commit() + return project + + def get_by_uuid(self, uuid): + project = V2Project.query.filter_by(uuid=uuid).first() + if not project: + raise ValueError + return project + + def update_attr(self, uuid, attr): + project = self.get_by_uuid(uuid) + for k, v in attr.items(): + setattr(project, k, v) + db_session.commit() + + def append_attr(self, uuid, attr): + project = self.get_by_uuid(uuid) + for k, v in attr.items(): + value = getattr(project, k) + new = '{},{}'.format(value, v) if value else v + setattr(project, k, new) + db_session.commit() + + def delete_by_uuid(self, uuid): + project = self.get_by_uuid(uuid) + db_session.delete(project) + db_session.commit() + + +class V2TaskHandler(object): + + def list_all(self): + return V2Task.query.all() + + def insert(self, kwargs): + task = V2Task(**kwargs) + db_session.add(task) + db_session.commit() + return task + + def get_by_uuid(self, uuid): + task = V2Task.query.filter_by(uuid=uuid).first() + if not task: + raise ValueError + return task + + def update_attr(self, uuid, attr): + task = self.get_by_uuid(uuid) + for k, v in attr.items(): + setattr(task, k, v) + db_session.commit() + + def delete_by_uuid(self, uuid): + task = self.get_by_uuid(uuid) + db_session.delete(task) + db_session.commit() diff --git a/api/database/v2/models.py b/api/database/v2/models.py new file mode 100644 index 000000000..1e85559cb --- /dev/null +++ b/api/database/v2/models.py @@ -0,0 +1,100 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import +from sqlalchemy import Column +from sqlalchemy import Integer +from sqlalchemy import String +from sqlalchemy import Text +from sqlalchemy import DateTime +from sqlalchemy import Boolean + +from api.database import Base + + +class V2Environment(Base): + __tablename__ = 'v2_environment' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + description = Column(Text) + openrc_id = Column(String(10)) + image_id = Column(String(30)) + container_id = Column(Text) + pod_id = Column(String(10)) + time = Column(DateTime) + + +class V2Openrc(Base): + __tablename__ = 'v2_openrc' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + description = Column(Text) + environment_id = Column(String(30)) + content = Column(Text) + time = Column(DateTime) + + +class V2Image(Base): + __tablename__ = 'v2_image' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + description = Column(Text) + environment_id = Column(String(30)) + size = Column(String(30)) + status = Column(String(30)) + time = Column(DateTime) + + +class V2Container(Base): + __tablename__ = 'v2_container' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + environment_id = Column(String(30)) + status = Column(Integer) + port = Column(Integer) + time = Column(String(30)) + + +class V2Pod(Base): + __tablename__ = 'v2_pod' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + environment_id = Column(String(30)) + content = Column(Text) + time = Column(String(30)) + + +class V2Project(Base): + __tablename__ = 'v2_project' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + description = Column(Text) + time = Column(DateTime) + tasks = Column(Text) + + +class V2Task(Base): + __tablename__ = 'v2_task' + id = Column(Integer, primary_key=True) + uuid = Column(String(30)) + name = Column(String(30)) + description = Column(Text) + project_id = Column(String(30)) + environment_id = Column(String(30)) + time = Column(DateTime) + case_name = Column(String(30)) + suite = Column(Boolean) + content = Column(Text) + result = Column(Text) + error = Column(Text) + status = Column(Integer) diff --git a/api/resources/asynctask.py b/api/resources/asynctask.py deleted file mode 100644 index 39b47c0ee..000000000 --- a/api/resources/asynctask.py +++ /dev/null @@ -1,64 +0,0 @@ -# ############################################################################ -# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# ############################################################################ -import uuid -import logging - -from api.utils.common import result_handler -from api.database.v1.handlers import AsyncTaskHandler -from yardstick.common import constants as consts - -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) - - -def default(args): - return _get_status(args) - - -def _get_status(args): - try: - task_id = args['task_id'] - except KeyError: - return result_handler(consts.API_ERROR, 'task_id must be provided') - - try: - uuid.UUID(task_id) - except ValueError: - return result_handler(consts.API_ERROR, 'invalid task_id') - - asynctask_handler = AsyncTaskHandler() - try: - asynctask = asynctask_handler.get_task_by_taskid(task_id) - except ValueError: - return result_handler(consts.API_ERROR, 'invalid task_id') - - def _unfinished(): - return result_handler(consts.TASK_NOT_DONE, {}) - - def _finished(): - return result_handler(consts.TASK_DONE, {}) - - def _error(): - return result_handler(consts.TASK_FAILED, asynctask.error) - - status = asynctask.status - LOG.debug('Task status is: %s', status) - - if status not in [consts.TASK_NOT_DONE, - consts.TASK_DONE, - consts.TASK_FAILED]: - return result_handler(consts.API_ERROR, 'internal server error') - - switcher = { - consts.TASK_NOT_DONE: _unfinished, - consts.TASK_DONE: _finished, - consts.TASK_FAILED: _error - } - - return switcher.get(status)() diff --git a/api/resources/case_docs.py b/api/resources/case_docs.py deleted file mode 100644 index 289410d2d..000000000 --- a/api/resources/case_docs.py +++ /dev/null @@ -1,30 +0,0 @@ -import os -import logging - -from api.utils.common import result_handler -from yardstick.common import constants as consts - -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) - - -def default(args): - return get_case_docs(args) - - -def get_case_docs(args): - try: - case_name = args['case_name'] - except KeyError: - return result_handler(consts.API_ERROR, 'case_name must be provided') - - docs_path = os.path.join(consts.DOCS_DIR, '{}.rst'.format(case_name)) - - if not os.path.exists(docs_path): - return result_handler(consts.API_ERROR, 'case not exists') - - LOG.info('Reading %s', case_name) - with open(docs_path) as f: - content = f.read() - - return result_handler(consts.API_SUCCESS, {'docs': content}) diff --git a/api/resources/env_action.py b/api/resources/env_action.py deleted file mode 100644 index fed987063..000000000 --- a/api/resources/env_action.py +++ /dev/null @@ -1,427 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import - -import errno -import logging -import os -import subprocess -import threading -import time -import uuid -import glob -import yaml -import collections -from subprocess import PIPE - -from six.moves import configparser -from oslo_serialization import jsonutils -from docker import Client - -from api.database.v1.handlers import AsyncTaskHandler -from api.utils import influx -from api.utils.common import result_handler -from yardstick.common import constants as consts -from yardstick.common import utils as common_utils -from yardstick.common import openstack_utils -from yardstick.common.httpClient import HttpClient - - -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) - -async_handler = AsyncTaskHandler() - - -def create_grafana(args): - task_id = str(uuid.uuid4()) - - thread = threading.Thread(target=_create_grafana, args=(task_id,)) - thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) - - -def _create_grafana(task_id): - _create_task(task_id) - - client = Client(base_url=consts.DOCKER_URL) - - try: - LOG.info('Checking if grafana image exist') - image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) - if not _check_image_exist(client, image): - LOG.info('Grafana image not exist, start pulling') - client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) - - LOG.info('Createing grafana container') - _create_grafana_container(client) - LOG.info('Grafana container is created') - - time.sleep(5) - - LOG.info('Creating data source for grafana') - _create_data_source() - - LOG.info('Creating dashboard for grafana') - _create_dashboard() - - _update_task_status(task_id) - LOG.info('Finished') - except Exception as e: - _update_task_error(task_id, str(e)) - LOG.exception('Create grafana failed') - - -def _create_dashboard(): - url = 'http://admin:admin@%s:3000/api/dashboards/db' % consts.GRAFANA_IP - path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json') - - for i in sorted(glob.iglob(path)): - with open(i) as f: - data = jsonutils.load(f) - try: - HttpClient().post(url, data) - except Exception: - LOG.exception('Create dashboard %s failed', i) - raise - - -def _create_data_source(): - url = 'http://admin:admin@%s:3000/api/datasources' % consts.GRAFANA_IP - data = { - "name": "yardstick", - "type": "influxdb", - "access": "proxy", - "url": "http://%s:8086" % consts.INFLUXDB_IP, - "password": "root", - "user": "root", - "database": "yardstick", - "basicAuth": True, - "basicAuthUser": "admin", - "basicAuthPassword": "admin", - "isDefault": False, - } - try: - HttpClient().post(url, data) - except Exception: - LOG.exception('Create datasources failed') - raise - - -def _create_grafana_container(client): - ports = [3000] - port_bindings = {k: k for k in ports} - restart_policy = {"MaximumRetryCount": 0, "Name": "always"} - host_config = client.create_host_config(port_bindings=port_bindings, - restart_policy=restart_policy) - - LOG.info('Creating container') - container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE, - consts.GRAFANA_TAG), - ports=ports, - detach=True, - tty=True, - host_config=host_config) - LOG.info('Starting container') - client.start(container) - - -def _check_image_exist(client, t): - return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags']) - - -def create_influxdb(args): - task_id = str(uuid.uuid4()) - - thread = threading.Thread(target=_create_influxdb, args=(task_id,)) - thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) - - -def _create_influxdb(task_id): - _create_task(task_id) - - client = Client(base_url=consts.DOCKER_URL) - - try: - LOG.info('Changing output to influxdb') - _change_output_to_influxdb() - - LOG.info('Checking if influxdb image exist') - if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE, - consts.INFLUXDB_TAG)): - LOG.info('Influxdb image not exist, start pulling') - client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG) - - LOG.info('Createing influxdb container') - _create_influxdb_container(client) - LOG.info('Influxdb container is created') - - time.sleep(5) - - LOG.info('Config influxdb') - _config_influxdb() - - _update_task_status(task_id) - - LOG.info('Finished') - except Exception as e: - _update_task_error(task_id, str(e)) - LOG.exception('Creating influxdb failed') - - -def _create_influxdb_container(client): - - ports = [8083, 8086] - port_bindings = {k: k for k in ports} - restart_policy = {"MaximumRetryCount": 0, "Name": "always"} - host_config = client.create_host_config(port_bindings=port_bindings, - restart_policy=restart_policy) - - LOG.info('Creating container') - container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE, - consts.INFLUXDB_TAG), - ports=ports, - detach=True, - tty=True, - host_config=host_config) - LOG.info('Starting container') - client.start(container) - - -def _config_influxdb(): - try: - client = influx.get_data_db_client() - client.create_user(consts.INFLUXDB_USER, - consts.INFLUXDB_PASS, - consts.INFLUXDB_DB_NAME) - client.create_database(consts.INFLUXDB_DB_NAME) - LOG.info('Success to config influxDB') - except Exception: - LOG.exception('Config influxdb failed') - - -def _change_output_to_influxdb(): - common_utils.makedirs(consts.CONF_DIR) - - parser = configparser.ConfigParser() - LOG.info('Reading output sample configuration') - parser.read(consts.CONF_SAMPLE_FILE) - - LOG.info('Set dispatcher to influxdb') - parser.set('DEFAULT', 'dispatcher', 'influxdb') - parser.set('dispatcher_influxdb', 'target', - 'http://%s:8086' % consts.INFLUXDB_IP) - - LOG.info('Writing to %s', consts.CONF_FILE) - with open(consts.CONF_FILE, 'w') as f: - parser.write(f) - - -def prepare_env(args): - task_id = str(uuid.uuid4()) - - thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,)) - thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) - - -def _already_source_openrc(): - """Check if openrc is sourced already""" - return all(os.environ.get(k) for k in ['OS_AUTH_URL', 'OS_USERNAME', - 'OS_PASSWORD', 'EXTERNAL_NETWORK']) - - -def _prepare_env_daemon(task_id): - _create_task(task_id) - - try: - _create_directories() - - rc_file = consts.OPENRC - - LOG.info('Checkout Openrc Environment variable') - if not _already_source_openrc(): - LOG.info('Openrc variable not found in Environment') - if not os.path.exists(rc_file): - LOG.info('Openrc file not found') - installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2') - installer_type = os.environ.get('INSTALLER_TYPE', 'compass') - LOG.info('Getting openrc file from %s', installer_type) - _get_remote_rc_file(rc_file, installer_ip, installer_type) - LOG.info('Source openrc file') - _source_file(rc_file) - LOG.info('Appending external network') - _append_external_network(rc_file) - LOG.info('Openrc file exist, source openrc file') - _source_file(rc_file) - - LOG.info('Cleaning images') - _clean_images() - - LOG.info('Loading images') - _load_images() - - _update_task_status(task_id) - LOG.info('Finished') - except Exception as e: - _update_task_error(task_id, str(e)) - LOG.exception('Prepare env failed') - - -def _create_directories(): - common_utils.makedirs(consts.CONF_DIR) - - -def _source_file(rc_file): - common_utils.source_env(rc_file) - - -def _get_remote_rc_file(rc_file, installer_ip, installer_type): - - os_fetch_script = os.path.join(consts.RELENG_DIR, consts.FETCH_SCRIPT) - - try: - cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, - '-a', installer_ip] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - p.communicate() - - if p.returncode != 0: - LOG.error('Failed to fetch credentials from installer') - except OSError as e: - if e.errno != errno.EEXIST: - raise - - -def _append_external_network(rc_file): - neutron_client = openstack_utils.get_neutron_client() - networks = neutron_client.list_networks()['networks'] - try: - ext_network = next(n['name'] for n in networks if n['router:external']) - except StopIteration: - LOG.warning("Can't find external network") - else: - cmd = 'export EXTERNAL_NETWORK=%s' % ext_network - try: - with open(rc_file, 'a') as f: - f.write(cmd + '\n') - except OSError as e: - if e.errno != errno.EEXIST: - raise - - -def _clean_images(): - cmd = [consts.CLEAN_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) - output = p.communicate()[0] - LOG.debug(output) - - -def _load_images(): - cmd = [consts.LOAD_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) - output = p.communicate()[0] - LOG.debug(output) - - -def _create_task(task_id): - async_handler.insert({'status': 0, 'task_id': task_id}) - - -def _update_task_status(task_id): - async_handler.update_attr(task_id, {'status': 1}) - - -def _update_task_error(task_id, error): - async_handler.update_attr(task_id, {'status': 2, 'error': error}) - - -def update_openrc(args): - try: - openrc_vars = args['openrc'] - except KeyError: - return result_handler(consts.API_ERROR, 'openrc must be provided') - else: - if not isinstance(openrc_vars, collections.Mapping): - return result_handler(consts.API_ERROR, 'args should be a dict') - - lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()] - LOG.debug('Writing: %s', ''.join(lines)) - - LOG.info('Writing openrc: Writing') - common_utils.makedirs(consts.CONF_DIR) - - with open(consts.OPENRC, 'w') as f: - f.writelines(lines) - LOG.info('Writing openrc: Done') - - LOG.info('Source openrc: Sourcing') - try: - _source_file(consts.OPENRC) - except Exception as e: - LOG.exception('Failed to source openrc') - return result_handler(consts.API_ERROR, str(e)) - LOG.info('Source openrc: Done') - - return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars}) - - -def upload_pod_file(args): - try: - pod_file = args['file'] - except KeyError: - return result_handler(consts.API_ERROR, 'file must be provided') - - LOG.info('Checking file') - data = yaml.load(pod_file.read()) - if not isinstance(data, collections.Mapping): - return result_handler(consts.API_ERROR, 'invalid yaml file') - - LOG.info('Writing file') - with open(consts.POD_FILE, 'w') as f: - yaml.dump(data, f, default_flow_style=False) - LOG.info('Writing finished') - - return result_handler(consts.API_SUCCESS, {'pod_info': data}) - - -def update_pod_file(args): - try: - pod_dic = args['pod'] - except KeyError: - return result_handler(consts.API_ERROR, 'pod must be provided') - else: - if not isinstance(pod_dic, collections.Mapping): - return result_handler(consts.API_ERROR, 'pod should be a dict') - - LOG.info('Writing file') - with open(consts.POD_FILE, 'w') as f: - yaml.dump(pod_dic, f, default_flow_style=False) - LOG.info('Writing finished') - - return result_handler(consts.API_SUCCESS, {'pod_info': pod_dic}) - - -def update_hosts(hosts_ip): - if not isinstance(hosts_ip, dict): - return result_handler(consts.API_ERROR, 'Error, args should be a dict') - LOG.info('Writing hosts: Writing') - LOG.debug('Writing: %s', hosts_ip) - cmd = ["sudo", "python", "write_hosts.py"] - p = subprocess.Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, - cwd = os.path.join(consts.REPOS_DIR, "api/resources")) - _, err = p.communicate(jsonutils.dumps(hosts_ip)) - if p.returncode != 0 : - return result_handler(consts.API_ERROR, err) - LOG.info('Writing hosts: Done') - return result_handler(consts.API_SUCCESS, 'success') diff --git a/api/resources/release_action.py b/api/resources/release_action.py deleted file mode 100644 index 9871c1fc3..000000000 --- a/api/resources/release_action.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import uuid -import os -import logging - -from api.utils.common import result_handler -from api.utils.thread import TaskThread -from yardstick.common import constants as consts -from yardstick.benchmark.core import Param -from yardstick.benchmark.core.task import Task - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -def run_test_case(args): - try: - case_name = args['testcase'] - except KeyError: - return result_handler(consts.API_ERROR, 'testcase must be provided') - - testcase = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(case_name)) - - task_id = str(uuid.uuid4()) - - task_args = { - 'inputfile': [testcase], - 'task_id': task_id - } - task_args.update(args.get('opts', {})) - - param = Param(task_args) - task_thread = TaskThread(Task().start, param) - task_thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/results.py b/api/resources/results.py deleted file mode 100644 index 692e00cc6..000000000 --- a/api/resources/results.py +++ /dev/null @@ -1,69 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import logging -import uuid -import json - -from api.utils.common import result_handler -from api.database.v1.handlers import TasksHandler -from yardstick.common import constants as consts - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -def default(args): - return getResult(args) - - -def getResult(args): - try: - task_id = args['task_id'] - except KeyError: - return result_handler(consts.API_ERROR, 'task_id must be provided') - - try: - uuid.UUID(task_id) - except ValueError: - return result_handler(consts.API_ERROR, 'invalid task_id') - - task_handler = TasksHandler() - try: - task = task_handler.get_task_by_taskid(task_id) - except ValueError: - return result_handler(consts.API_ERROR, 'invalid task_id') - - def _unfinished(): - return result_handler(consts.TASK_NOT_DONE, {}) - - def _finished(): - if task.result: - return result_handler(consts.TASK_DONE, json.loads(task.result)) - else: - return result_handler(consts.TASK_DONE, {}) - - def _error(): - return result_handler(consts.TASK_FAILED, task.error) - - status = task.status - logger.debug('Task status is: %s', status) - - if status not in [consts.TASK_NOT_DONE, - consts.TASK_DONE, - consts.TASK_FAILED]: - return result_handler(consts.API_ERROR, 'internal server error') - - switcher = { - consts.TASK_NOT_DONE: _unfinished, - consts.TASK_DONE: _finished, - consts.TASK_FAILED: _error - } - - return switcher.get(status)() diff --git a/api/resources/samples_action.py b/api/resources/samples_action.py deleted file mode 100644 index 10b9980af..000000000 --- a/api/resources/samples_action.py +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import uuid -import os -import logging - -from api.utils.common import result_handler -from api.utils.thread import TaskThread -from yardstick.common import constants as consts -from yardstick.benchmark.core import Param -from yardstick.benchmark.core.task import Task - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -def run_test_case(args): - try: - case_name = args['testcase'] - except KeyError: - return result_handler(consts.API_ERROR, 'testcase must be provided') - - testcase = os.path.join(consts.SAMPLE_CASE_DIR, - '{}.yaml'.format(case_name)) - - task_id = str(uuid.uuid4()) - - task_args = { - 'inputfile': [testcase], - 'task_id': task_id - } - task_args.update(args.get('opts', {})) - - param = Param(task_args) - task_thread = TaskThread(Task().start, param) - task_thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/testcases.py b/api/resources/testcases.py deleted file mode 100644 index 6ee15efb3..000000000 --- a/api/resources/testcases.py +++ /dev/null @@ -1,21 +0,0 @@ -# ############################################################################ -# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# ############################################################################ -from yardstick.benchmark.core.testcase import Testcase -from yardstick.benchmark.core import Param -from api.utils import common as common_utils - - -def default(args): - return listAllTestcases(args) - - -def listAllTestcases(args): - param = Param(args) - testcase_list = Testcase().list_all(param) - return common_utils.result_handler(1, testcase_list) diff --git a/api/resources/testsuites_action.py b/api/resources/testsuites_action.py deleted file mode 100644 index e37eacc3e..000000000 --- a/api/resources/testsuites_action.py +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import uuid -import os -import logging - -from api.utils.common import result_handler -from api.utils.thread import TaskThread -from yardstick.common import constants as consts -from yardstick.benchmark.core import Param -from yardstick.benchmark.core.task import Task - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -def run_test_suite(args): - try: - suite_name = args['testsuite'] - except KeyError: - return result_handler(consts.API_ERROR, 'testsuite must be provided') - - testsuite = os.path.join(consts.TESTSUITE_DIR, - '{}.yaml'.format(suite_name)) - - task_id = str(uuid.uuid4()) - - task_args = { - 'inputfile': [testsuite], - 'task_id': task_id, - 'suite': True - } - task_args.update(args.get('opts', {})) - - param = Param(task_args) - task_thread = TaskThread(Task().start, param) - task_thread.start() - - return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/v1/__init__.py b/api/resources/v1/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/resources/v1/__init__.py diff --git a/api/resources/v1/asynctasks.py b/api/resources/v1/asynctasks.py new file mode 100644 index 000000000..759df214c --- /dev/null +++ b/api/resources/v1/asynctasks.py @@ -0,0 +1,65 @@ +# ############################################################################ +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################ +import uuid +import logging + +from api import ApiResource +from api.database.v1.handlers import AsyncTaskHandler +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V1AsyncTask(ApiResource): + + def get(self): + args = self._get_args() + + try: + task_id = args['task_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'task_id must be provided') + + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + asynctask_handler = AsyncTaskHandler() + try: + asynctask = asynctask_handler.get_task_by_taskid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + def _unfinished(): + return result_handler(consts.TASK_NOT_DONE, {}) + + def _finished(): + return result_handler(consts.TASK_DONE, {}) + + def _error(): + return result_handler(consts.TASK_FAILED, asynctask.error) + + status = asynctask.status + LOG.debug('Task status is: %s', status) + + if status not in [consts.TASK_NOT_DONE, + consts.TASK_DONE, + consts.TASK_FAILED]: + return result_handler(consts.API_ERROR, 'internal server error') + + switcher = { + consts.TASK_NOT_DONE: _unfinished, + consts.TASK_DONE: _finished, + consts.TASK_FAILED: _error + } + + return switcher.get(status)() diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py new file mode 100644 index 000000000..8943db3d1 --- /dev/null +++ b/api/resources/v1/env.py @@ -0,0 +1,439 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import + +import errno +import logging +import os +import subprocess +import threading +import time +import uuid +import glob +import yaml +import collections + +from six.moves import configparser +from oslo_serialization import jsonutils +from docker import Client + +from api.database.v1.handlers import AsyncTaskHandler +from api.utils import influx +from api import ApiResource +from yardstick.common import constants as consts +from yardstick.common import utils +from yardstick.common.utils import result_handler +from yardstick.common import openstack_utils +from yardstick.common.httpClient import HttpClient + + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + +async_handler = AsyncTaskHandler() + + +class V1Env(ApiResource): + + def post(self): + return self._dispatch_post() + + def create_grafana(self, args): + task_id = str(uuid.uuid4()) + + thread = threading.Thread(target=self._create_grafana, args=(task_id,)) + thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) + + def _create_grafana(self, task_id): + self._create_task(task_id) + + client = Client(base_url=consts.DOCKER_URL) + + try: + LOG.info('Checking if grafana image exist') + image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + if not self._check_image_exist(client, image): + LOG.info('Grafana image not exist, start pulling') + client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + + LOG.info('Createing grafana container') + container = self._create_grafana_container(client) + LOG.info('Grafana container is created') + + time.sleep(5) + + container = client.inspect_container(container['Id']) + ip = container['NetworkSettings']['Networks']['bridge']['IPAddress'] + LOG.debug('container ip is: %s', ip) + + LOG.info('Creating data source for grafana') + self._create_data_source(ip) + + LOG.info('Creating dashboard for grafana') + self._create_dashboard(ip) + + self._update_task_status(task_id) + LOG.info('Finished') + except Exception as e: + self._update_task_error(task_id, str(e)) + LOG.exception('Create grafana failed') + + def _create_dashboard(self, ip): + url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, consts.GRAFANA_PORT) + path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json') + + for i in sorted(glob.iglob(path)): + with open(i) as f: + data = jsonutils.load(f) + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create dashboard %s failed', i) + raise + + def _create_data_source(self, ip): + url = 'http://admin:admin@{}:{}/api/datasources'.format(ip, consts.GRAFANA_PORT) + influx_conf = utils.parse_ini_file(consts.CONF_FILE) + + try: + influx_url = influx_conf['dispatcher_influxdb']['target'] + except KeyError: + LOG.exception('influxdb url not set in yardstick.conf') + raise + + data = { + "name": "yardstick", + "type": "influxdb", + "access": "proxy", + "url": influx_url, + "password": "root", + "user": "root", + "database": "yardstick", + "basicAuth": True, + "basicAuthUser": "admin", + "basicAuthPassword": "admin", + "isDefault": False, + } + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create datasources failed') + raise + + def _create_grafana_container(self, client): + ports = [consts.GRAFANA_PORT] + port_bindings = {consts.GRAFANA_PORT: consts.GRAFANA_MAPPING_PORT} + restart_policy = {"MaximumRetryCount": 0, "Name": "always"} + host_config = client.create_host_config(port_bindings=port_bindings, + restart_policy=restart_policy) + + LOG.info('Creating container') + container = client.create_container(image='%s:%s' % + (consts.GRAFANA_IMAGE, + consts.GRAFANA_TAG), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + LOG.info('Starting container') + client.start(container) + return container + + def _check_image_exist(self, client, t): + return any(t in a['RepoTags'][0] + for a in client.images() if a['RepoTags']) + + def create_influxdb(self, args): + task_id = str(uuid.uuid4()) + + thread = threading.Thread(target=self._create_influxdb, args=(task_id,)) + thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) + + def _create_influxdb(self, task_id): + self._create_task(task_id) + + client = Client(base_url=consts.DOCKER_URL) + + try: + LOG.info('Checking if influxdb image exist') + if not self._check_image_exist(client, '%s:%s' % + (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG)): + LOG.info('Influxdb image not exist, start pulling') + client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG) + + LOG.info('Createing influxdb container') + container = self._create_influxdb_container(client) + LOG.info('Influxdb container is created') + + time.sleep(5) + + container = client.inspect_container(container['Id']) + ip = container['NetworkSettings']['Networks']['bridge']['IPAddress'] + LOG.debug('container ip is: %s', ip) + + LOG.info('Changing output to influxdb') + self._change_output_to_influxdb(ip) + + LOG.info('Config influxdb') + self._config_influxdb() + + self._update_task_status(task_id) + + LOG.info('Finished') + except Exception as e: + self._update_task_error(task_id, str(e)) + LOG.exception('Creating influxdb failed') + + def _create_influxdb_container(self, client): + + ports = [consts.INFLUXDB_DASHBOARD_PORT, consts.INFLUXDB_PORT] + port_bindings = {k: k for k in ports} + restart_policy = {"MaximumRetryCount": 0, "Name": "always"} + host_config = client.create_host_config(port_bindings=port_bindings, + restart_policy=restart_policy) + + LOG.info('Creating container') + container = client.create_container(image='%s:%s' % + (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG), + ports=ports, + detach=True, + tty=True, + host_config=host_config) + LOG.info('Starting container') + client.start(container) + return container + + def _config_influxdb(self): + try: + client = influx.get_data_db_client() + client.create_user(consts.INFLUXDB_USER, + consts.INFLUXDB_PASS, + consts.INFLUXDB_DB_NAME) + client.create_database(consts.INFLUXDB_DB_NAME) + LOG.info('Success to config influxDB') + except Exception: + LOG.exception('Config influxdb failed') + + def _change_output_to_influxdb(self, ip): + utils.makedirs(consts.CONF_DIR) + + parser = configparser.ConfigParser() + LOG.info('Reading output sample configuration') + parser.read(consts.CONF_SAMPLE_FILE) + + LOG.info('Set dispatcher to influxdb') + parser.set('DEFAULT', 'dispatcher', 'influxdb') + parser.set('dispatcher_influxdb', 'target', + 'http://{}:{}'.format(ip, consts.INFLUXDB_PORT)) + + LOG.info('Writing to %s', consts.CONF_FILE) + with open(consts.CONF_FILE, 'w') as f: + parser.write(f) + + def prepare_env(self, args): + task_id = str(uuid.uuid4()) + + thread = threading.Thread(target=self._prepare_env_daemon, + args=(task_id,)) + thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) + + def _already_source_openrc(self): + """Check if openrc is sourced already""" + return all(os.environ.get(k) for k in ['OS_AUTH_URL', + 'OS_USERNAME', + 'OS_PASSWORD', + 'EXTERNAL_NETWORK']) + + def _prepare_env_daemon(self, task_id): + self._create_task(task_id) + + try: + self._create_directories() + + rc_file = consts.OPENRC + + LOG.info('Checkout Openrc Environment variable') + if not self._already_source_openrc(): + LOG.info('Openrc variable not found in Environment') + if not os.path.exists(rc_file): + LOG.info('Openrc file not found') + installer_ip = os.environ.get('INSTALLER_IP', + '192.168.200.2') + installer_type = os.environ.get('INSTALLER_TYPE', 'compass') + LOG.info('Getting openrc file from %s', installer_type) + self._get_remote_rc_file(rc_file, + installer_ip, + installer_type) + LOG.info('Source openrc file') + self._source_file(rc_file) + LOG.info('Appending external network') + self._append_external_network(rc_file) + LOG.info('Openrc file exist, source openrc file') + self._source_file(rc_file) + + LOG.info('Cleaning images') + self._clean_images() + + LOG.info('Loading images') + self._load_images() + + self._update_task_status(task_id) + LOG.info('Finished') + except Exception as e: + self._update_task_error(task_id, str(e)) + LOG.exception('Prepare env failed') + + def _create_directories(self): + utils.makedirs(consts.CONF_DIR) + + def _source_file(self, rc_file): + utils.source_env(rc_file) + + def _get_remote_rc_file(self, rc_file, installer_ip, installer_type): + + os_fetch_script = os.path.join(consts.RELENG_DIR, consts.FETCH_SCRIPT) + + try: + cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, + '-a', installer_ip] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p.communicate() + + if p.returncode != 0: + LOG.error('Failed to fetch credentials from installer') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + def _append_external_network(self, rc_file): + neutron_client = openstack_utils.get_neutron_client() + networks = neutron_client.list_networks()['networks'] + try: + ext_network = next(n['name'] + for n in networks if n['router:external']) + except StopIteration: + LOG.warning("Can't find external network") + else: + cmd = 'export EXTERNAL_NETWORK=%s' % ext_network + try: + with open(rc_file, 'a') as f: + f.write(cmd + '\n') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + def _clean_images(self): + cmd = [consts.CLEAN_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) + output = p.communicate()[0] + LOG.debug(output) + + def _load_images(self): + cmd = [consts.LOAD_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR) + output = p.communicate()[0] + LOG.debug(output) + + def _create_task(self, task_id): + async_handler.insert({'status': 0, 'task_id': task_id}) + + def _update_task_status(self, task_id): + async_handler.update_attr(task_id, {'status': 1}) + + def _update_task_error(self, task_id, error): + async_handler.update_attr(task_id, {'status': 2, 'error': error}) + + def update_openrc(self, args): + try: + openrc_vars = args['openrc'] + except KeyError: + return result_handler(consts.API_ERROR, 'openrc must be provided') + else: + if not isinstance(openrc_vars, collections.Mapping): + return result_handler(consts.API_ERROR, 'args should be a dict') + + lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()] + LOG.debug('Writing: %s', ''.join(lines)) + + LOG.info('Writing openrc: Writing') + utils.makedirs(consts.CONF_DIR) + + with open(consts.OPENRC, 'w') as f: + f.writelines(lines) + LOG.info('Writing openrc: Done') + + LOG.info('Source openrc: Sourcing') + try: + self._source_file(consts.OPENRC) + except Exception as e: + LOG.exception('Failed to source openrc') + return result_handler(consts.API_ERROR, str(e)) + LOG.info('Source openrc: Done') + + return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars}) + + def upload_pod_file(self, args): + try: + pod_file = args['file'] + except KeyError: + return result_handler(consts.API_ERROR, 'file must be provided') + + LOG.info('Checking file') + data = yaml.load(pod_file.read()) + if not isinstance(data, collections.Mapping): + return result_handler(consts.API_ERROR, 'invalid yaml file') + + LOG.info('Writing file') + with open(consts.POD_FILE, 'w') as f: + yaml.dump(data, f, default_flow_style=False) + LOG.info('Writing finished') + + return result_handler(consts.API_SUCCESS, {'pod_info': data}) + + def update_pod_file(self, args): + try: + pod_dic = args['pod'] + except KeyError: + return result_handler(consts.API_ERROR, 'pod must be provided') + else: + if not isinstance(pod_dic, collections.Mapping): + return result_handler(consts.API_ERROR, 'pod should be a dict') + + LOG.info('Writing file') + with open(consts.POD_FILE, 'w') as f: + yaml.dump(pod_dic, f, default_flow_style=False) + LOG.info('Writing finished') + + return result_handler(consts.API_SUCCESS, {'pod_info': pod_dic}) + + def update_hosts(self, hosts_ip): + if not isinstance(hosts_ip, collections.Mapping): + return result_handler(consts.API_ERROR, 'args should be a dict') + LOG.info('Writing hosts: Writing') + LOG.debug('Writing: %s', hosts_ip) + cmd = ["sudo", "python", "write_hosts.py"] + p = subprocess.Popen(cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=os.path.join(consts.REPOS_DIR, + "api/resources")) + _, err = p.communicate(jsonutils.dumps(hosts_ip)) + if p.returncode != 0: + return result_handler(consts.API_ERROR, err) + LOG.info('Writing hosts: Done') + return result_handler(consts.API_SUCCESS, 'success') diff --git a/api/resources/v1/results.py b/api/resources/v1/results.py new file mode 100644 index 000000000..0493b43b6 --- /dev/null +++ b/api/resources/v1/results.py @@ -0,0 +1,78 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import +import logging +import uuid +import json +import os + +from flasgger.utils import swag_from + +from api import ApiResource +from api.database.v1.handlers import TasksHandler +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler +from api.swagger import models + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +ResultModel = models.ResultModel + + +class V1Result(ApiResource): + + @swag_from(os.path.join(consts.REPOS_DIR, 'api/swagger/docs/results.yaml')) + def get(self): + args = self._get_args() + + try: + task_id = args['task_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'task_id must be provided') + + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + task_handler = TasksHandler() + try: + task = task_handler.get_task_by_taskid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + def _unfinished(): + return result_handler(consts.TASK_NOT_DONE, {}) + + def _finished(): + if task.result: + return result_handler(consts.TASK_DONE, json.loads(task.result)) + else: + return result_handler(consts.TASK_DONE, {}) + + def _error(): + return result_handler(consts.TASK_FAILED, task.error) + + status = task.status + LOG.debug('Task status is: %s', status) + + if status not in [consts.TASK_NOT_DONE, + consts.TASK_DONE, + consts.TASK_FAILED]: + return result_handler(consts.API_ERROR, 'internal server error') + + switcher = { + consts.TASK_NOT_DONE: _unfinished, + consts.TASK_DONE: _finished, + consts.TASK_FAILED: _error + } + + return switcher.get(status)() diff --git a/api/resources/v1/testcases.py b/api/resources/v1/testcases.py new file mode 100644 index 000000000..f15947245 --- /dev/null +++ b/api/resources/v1/testcases.py @@ -0,0 +1,115 @@ +# ############################################################################ +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# ############################################################################ + +from __future__ import absolute_import +import uuid +import os +import logging + +from flasgger.utils import swag_from + +from yardstick.benchmark.core.testcase import Testcase +from yardstick.benchmark.core.task import Task +from yardstick.benchmark.core import Param +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler +from api.utils.thread import TaskThread +from api import ApiResource +from api.swagger import models +from api.database.v1.handlers import TasksHandler + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V1Testcase(ApiResource): + + def get(self): + param = Param({}) + testcase_list = Testcase().list_all(param) + return result_handler(consts.API_SUCCESS, testcase_list) + + +class V1CaseDocs(ApiResource): + + def get(self, case_name): + docs_path = os.path.join(consts.DOCS_DIR, '{}.rst'.format(case_name)) + + if not os.path.exists(docs_path): + return result_handler(consts.API_ERROR, 'case not exists') + + LOG.info('Reading %s', case_name) + with open(docs_path) as f: + content = f.read() + + return result_handler(consts.API_SUCCESS, {'docs': content}) + + +TestCaseActionModel = models.TestCaseActionModel +TestCaseActionArgsModel = models.TestCaseActionArgsModel +TestCaseActionArgsOptsModel = models.TestCaseActionArgsOptsModel +TestCaseActionArgsOptsTaskArgModel = models.TestCaseActionArgsOptsTaskArgModel + + +class V1ReleaseCase(ApiResource): + + @swag_from(os.path.join(consts.REPOS_DIR, + 'api/swagger/docs/release_action.yaml')) + def post(self): + return self._dispatch_post() + + def run_test_case(self, args): + try: + name = args['testcase'] + except KeyError: + return result_handler(consts.API_ERROR, 'testcase must be provided') + + testcase = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(name)) + + task_id = str(uuid.uuid4()) + + task_args = { + 'inputfile': [testcase], + 'task_id': task_id + } + task_args.update(args.get('opts', {})) + + param = Param(task_args) + task_thread = TaskThread(Task().start, param, TasksHandler()) + task_thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) + + +class V1SampleCase(ApiResource): + + def post(self): + return self._dispatch_post() + + def run_test_case(self, args): + try: + name = args['testcase'] + except KeyError: + return result_handler(consts.API_ERROR, 'testcase must be provided') + + testcase = os.path.join(consts.SAMPLE_CASE_DIR, '{}.yaml'.format(name)) + + task_id = str(uuid.uuid4()) + + task_args = { + 'inputfile': [testcase], + 'task_id': task_id + } + task_args.update(args.get('opts', {})) + + param = Param(task_args) + task_thread = TaskThread(Task().start, param, TasksHandler()) + task_thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/v1/testsuites.py b/api/resources/v1/testsuites.py new file mode 100644 index 000000000..5f72c2ea6 --- /dev/null +++ b/api/resources/v1/testsuites.py @@ -0,0 +1,64 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import +import uuid +import os +import logging + +from flasgger.utils import swag_from + +from api import ApiResource +from api.utils.thread import TaskThread +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler +from yardstick.benchmark.core import Param +from yardstick.benchmark.core.task import Task +from api.swagger import models + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +TestSuiteActionModel = models.TestSuiteActionModel +TestSuiteActionArgsModel = models.TestSuiteActionArgsModel +TestSuiteActionArgsOptsModel = models.TestSuiteActionArgsOptsModel +TestSuiteActionArgsOptsTaskArgModel = \ + models.TestSuiteActionArgsOptsTaskArgModel + + +class V1Testsuite(ApiResource): + + @swag_from(os.path.join(consts.REPOS_DIR, + 'api/swagger/docs/testsuites_action.yaml')) + def post(self): + return self._dispatch_post() + + def run_test_suite(self, args): + try: + name = args['testsuite'] + except KeyError: + return result_handler(consts.API_ERROR, + 'testsuite must be provided') + + testsuite = os.path.join(consts.TESTSUITE_DIR, '{}.yaml'.format(name)) + + task_id = str(uuid.uuid4()) + + task_args = { + 'inputfile': [testsuite], + 'task_id': task_id, + 'suite': True + } + task_args.update(args.get('opts', {})) + + param = Param(task_args) + task_thread = TaskThread(Task().start, param) + task_thread.start() + + return result_handler(consts.API_SUCCESS, {'task_id': task_id}) diff --git a/api/resources/v2/__init__.py b/api/resources/v2/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/api/resources/v2/__init__.py diff --git a/api/resources/v2/containers.py b/api/resources/v2/containers.py new file mode 100644 index 000000000..66dc94120 --- /dev/null +++ b/api/resources/v2/containers.py @@ -0,0 +1,383 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import absolute_import + +import logging +import threading +import time +import uuid +import os +import glob + +from six.moves import configparser +from oslo_serialization import jsonutils +from docker import Client + +from api import ApiResource +from api.utils import influx +from api.database.v2.handlers import V2ContainerHandler +from api.database.v2.handlers import V2EnvironmentHandler +from yardstick.common import constants as consts +from yardstick.common import utils +from yardstick.common.utils import result_handler +from yardstick.common.utils import get_free_port +from yardstick.common.httpClient import HttpClient + + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + +environment_handler = V2EnvironmentHandler() +container_handler = V2ContainerHandler() + + +class V2Containers(ApiResource): + + def post(self): + return self._dispatch_post() + + def create_influxdb(self, args): + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + container_info = environment.container_id + container_info = jsonutils.loads(container_info) if container_info else {} + + if container_info.get('influxdb'): + return result_handler(consts.API_ERROR, 'influxdb container already exist') + + name = 'influxdb-{}'.format(environment_id[:8]) + port = get_free_port(consts.SERVER_IP) + container_id = str(uuid.uuid4()) + LOG.info('%s will launch on : %s', name, port) + + LOG.info('launch influxdb background') + args = (name, port, container_id) + thread = threading.Thread(target=self._create_influxdb, args=args) + thread.start() + + LOG.info('record container in database') + container_init_data = { + 'uuid': container_id, + 'environment_id': environment_id, + 'name': name, + 'port': port, + 'status': 0 + } + container_handler.insert(container_init_data) + + LOG.info('update container in environment') + container_info['influxdb'] = container_id + environment_info = {'container_id': jsonutils.dumps(container_info)} + environment_handler.update_attr(environment_id, environment_info) + + return result_handler(consts.API_SUCCESS, {'uuid': container_id}) + + def _check_image_exist(self, client, t): + return any(t in a['RepoTags'][0] + for a in client.images() if a['RepoTags']) + + def _create_influxdb(self, name, port, container_id): + client = Client(base_url=consts.DOCKER_URL) + + try: + LOG.info('Checking if influxdb image exist') + if not self._check_image_exist(client, '%s:%s' % + (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG)): + LOG.info('Influxdb image not exist, start pulling') + client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG) + + LOG.info('Createing influxdb container') + container = self._create_influxdb_container(client, name, port) + LOG.info('Influxdb container is created') + + time.sleep(5) + + container = client.inspect_container(container['Id']) + ip = container['NetworkSettings']['Networks']['bridge']['IPAddress'] + LOG.debug('container ip is: %s', ip) + + LOG.info('Changing output to influxdb') + self._change_output_to_influxdb(ip) + + LOG.info('Config influxdb') + self._config_influxdb() + + container_handler.update_attr(container_id, {'status': 1}) + + LOG.info('Finished') + except Exception: + container_handler.update_attr(container_id, {'status': 2}) + LOG.exception('Creating influxdb failed') + + def _create_influxdb_container(self, client, name, port): + + ports = [port] + port_bindings = {8086: port} + restart_policy = {"MaximumRetryCount": 0, "Name": "always"} + host_config = client.create_host_config(port_bindings=port_bindings, + restart_policy=restart_policy) + + LOG.info('Creating container') + container = client.create_container(image='%s:%s' % + (consts.INFLUXDB_IMAGE, + consts.INFLUXDB_TAG), + ports=ports, + name=name, + detach=True, + tty=True, + host_config=host_config) + LOG.info('Starting container') + client.start(container) + return container + + def _config_influxdb(self): + try: + client = influx.get_data_db_client() + client.create_user(consts.INFLUXDB_USER, + consts.INFLUXDB_PASS, + consts.INFLUXDB_DB_NAME) + client.create_database(consts.INFLUXDB_DB_NAME) + LOG.info('Success to config influxDB') + except Exception: + LOG.exception('Config influxdb failed') + + def _change_output_to_influxdb(self, ip): + utils.makedirs(consts.CONF_DIR) + + parser = configparser.ConfigParser() + LOG.info('Reading output sample configuration') + parser.read(consts.CONF_SAMPLE_FILE) + + LOG.info('Set dispatcher to influxdb') + parser.set('DEFAULT', 'dispatcher', 'influxdb') + parser.set('dispatcher_influxdb', 'target', + 'http://{}:{}'.format(ip, 8086)) + + LOG.info('Writing to %s', consts.CONF_FILE) + with open(consts.CONF_FILE, 'w') as f: + parser.write(f) + + def create_grafana(self, args): + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + container_info = environment.container_id + container_info = jsonutils.loads(container_info) if container_info else {} + + if not container_info.get('influxdb'): + return result_handler(consts.API_ERROR, 'influxdb not set') + + if container_info.get('grafana'): + return result_handler(consts.API_ERROR, 'grafana container already exists') + + name = 'grafana-{}'.format(environment_id[:8]) + port = get_free_port(consts.SERVER_IP) + container_id = str(uuid.uuid4()) + + args = (name, port, container_id) + thread = threading.Thread(target=self._create_grafana, args=args) + thread.start() + + container_init_data = { + 'uuid': container_id, + 'environment_id': environment_id, + 'name': name, + 'port': port, + 'status': 0 + } + container_handler.insert(container_init_data) + + container_info['grafana'] = container_id + environment_info = {'container_id': jsonutils.dumps(container_info)} + environment_handler.update_attr(environment_id, environment_info) + + return result_handler(consts.API_SUCCESS, {'uuid': container_id}) + + def _create_grafana(self, name, port, container_id): + client = Client(base_url=consts.DOCKER_URL) + + try: + LOG.info('Checking if grafana image exist') + image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + if not self._check_image_exist(client, image): + LOG.info('Grafana image not exist, start pulling') + client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG) + + LOG.info('Createing grafana container') + container = self._create_grafana_container(client, name, port) + LOG.info('Grafana container is created') + + time.sleep(5) + + container = client.inspect_container(container['Id']) + ip = container['NetworkSettings']['Networks']['bridge']['IPAddress'] + LOG.debug('container ip is: %s', ip) + + LOG.info('Creating data source for grafana') + self._create_data_source(ip) + + LOG.info('Creating dashboard for grafana') + self._create_dashboard(ip) + + container_handler.update_attr(container_id, {'status': 1}) + LOG.info('Finished') + except Exception: + container_handler.update_attr(container_id, {'status': 2}) + LOG.exception('Create grafana failed') + + def _create_dashboard(self, ip): + url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, 3000) + path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json') + + for i in sorted(glob.iglob(path)): + with open(i) as f: + data = jsonutils.load(f) + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create dashboard %s failed', i) + raise + + def _create_data_source(self, ip): + url = 'http://admin:admin@{}:{}/api/datasources'.format(ip, 3000) + + influx_conf = utils.parse_ini_file(consts.CONF_FILE) + try: + influx_url = influx_conf['dispatcher_influxdb']['target'] + except KeyError: + LOG.exception('influxdb url not set in yardstick.conf') + raise + + data = { + "name": "yardstick", + "type": "influxdb", + "access": "proxy", + "url": influx_url, + "password": "root", + "user": "root", + "database": "yardstick", + "basicAuth": True, + "basicAuthUser": "admin", + "basicAuthPassword": "admin", + "isDefault": False, + } + try: + HttpClient().post(url, data) + except Exception: + LOG.exception('Create datasources failed') + raise + + def _create_grafana_container(self, client, name, port): + ports = [3000] + port_bindings = {3000: port} + restart_policy = {"MaximumRetryCount": 0, "Name": "always"} + host_config = client.create_host_config(port_bindings=port_bindings, + restart_policy=restart_policy) + + LOG.info('Creating container') + container = client.create_container(image='%s:%s' % + (consts.GRAFANA_IMAGE, + consts.GRAFANA_TAG), + name=name, + ports=ports, + detach=True, + tty=True, + host_config=host_config) + LOG.info('Starting container') + client.start(container) + return container + + +class V2Container(ApiResource): + + def get(self, container_id): + try: + uuid.UUID(container_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid container id') + + try: + container = container_handler.get_by_uuid(container_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such container id') + + name = container.name + client = Client(base_url=consts.DOCKER_URL) + info = client.inspect_container(name) + + data = { + 'name': name, + 'status': info.get('State', {}).get('Status', 'error'), + 'time': info.get('Created'), + 'port': container.port + } + + return result_handler(consts.API_SUCCESS, {'container': data}) + + def delete(self, container_id): + try: + uuid.UUID(container_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid container id') + + try: + container = container_handler.get_by_uuid(container_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such container id') + + environment_id = container.environment_id + + client = Client(base_url=consts.DOCKER_URL) + LOG.info('delete container: %s', container.name) + try: + client.remove_container(container.name, force=True) + except Exception: + LOG.exception('delete container failed') + return result_handler(consts.API_ERROR, 'delete container failed') + + LOG.info('delete container in database') + container_handler.delete_by_uuid(container_id) + + LOG.info('update container in environment') + environment = environment_handler.get_by_uuid(environment_id) + container_info = jsonutils.loads(environment.container_id) + key = next((k for k, v in container_info.items() if v == container_id)) + container_info.pop(key) + environment_delete_data = { + 'container_id': jsonutils.dumps(container_info) + } + environment_handler.update_attr(environment_id, environment_delete_data) + + return result_handler(consts.API_SUCCESS, {'container': container_id}) diff --git a/api/resources/v2/environments.py b/api/resources/v2/environments.py new file mode 100644 index 000000000..f021a3c5a --- /dev/null +++ b/api/resources/v2/environments.py @@ -0,0 +1,125 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import logging + +from oslo_serialization import jsonutils +from docker import Client + +from api import ApiResource +from api.database.v2.handlers import V2EnvironmentHandler +from api.database.v2.handlers import V2OpenrcHandler +from api.database.v2.handlers import V2PodHandler +from api.database.v2.handlers import V2ContainerHandler +from yardstick.common.utils import result_handler +from yardstick.common.utils import change_obj_to_dict +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Environments(ApiResource): + + def get(self): + environment_handler = V2EnvironmentHandler() + environments = [change_obj_to_dict(e) for e in environment_handler.list_all()] + + for e in environments: + container_info = e['container_id'] + e['container_id'] = jsonutils.loads(container_info) if container_info else {} + + data = { + 'environments': environments + } + + return result_handler(consts.API_SUCCESS, data) + + def post(self): + return self._dispatch_post() + + def create_environment(self, args): + try: + name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'name must be provided') + + env_id = str(uuid.uuid4()) + + environment_handler = V2EnvironmentHandler() + + env_init_data = { + 'name': name, + 'uuid': env_id + } + environment_handler.insert(env_init_data) + + return result_handler(consts.API_SUCCESS, {'uuid': env_id}) + + +class V2Environment(ApiResource): + + def get(self, environment_id): + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + environment = change_obj_to_dict(environment) + container_id = environment['container_id'] + environment['container_id'] = jsonutils.loads(container_id) if container_id else {} + return result_handler(consts.API_SUCCESS, {'environment': environment}) + + def delete(self, environment_id): + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + if environment.openrc_id: + LOG.info('delete openrc: %s', environment.openrc_id) + openrc_handler = V2OpenrcHandler() + openrc_handler.delete_by_uuid(environment.openrc_id) + + if environment.pod_id: + LOG.info('delete pod: %s', environment.pod_id) + pod_handler = V2PodHandler() + pod_handler.delete_by_uuid(environment.pod_id) + + if environment.container_id: + LOG.info('delete containers') + container_info = jsonutils.loads(environment.container_id) + + container_handler = V2ContainerHandler() + client = Client(base_url=consts.DOCKER_URL) + for k, v in container_info.items(): + LOG.info('start delete: %s', k) + container = container_handler.get_by_uuid(v) + LOG.debug('container name: %s', container.name) + try: + client.remove_container(container.name, force=True) + except Exception: + LOG.exception('remove container failed') + container_handler.delete_by_uuid(v) + + environment_handler.delete_by_uuid(environment_id) + + return result_handler(consts.API_SUCCESS, {'environment': environment_id}) diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py new file mode 100644 index 000000000..8359e105b --- /dev/null +++ b/api/resources/v2/images.py @@ -0,0 +1,82 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import subprocess +import threading + +from api import ApiResource +from yardstick.common.utils import result_handler +from yardstick.common.utils import source_env +from yardstick.common.utils import change_obj_to_dict +from yardstick.common.openstack_utils import get_nova_client +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Images(ApiResource): + + def get(self): + try: + source_env(consts.OPENRC) + except: + return result_handler(consts.API_ERROR, 'source openrc error') + + nova_client = get_nova_client() + try: + images_list = nova_client.images.list() + except: + return result_handler(consts.API_ERROR, 'get images error') + else: + images = [self.get_info(change_obj_to_dict(i)) for i in images_list] + status = 1 if all(i['status'] == 'ACTIVE' for i in images) else 0 + if not images: + status = 0 + + return result_handler(consts.API_SUCCESS, {'status': status, 'images': images}) + + def post(self): + return self._dispatch_post() + + def get_info(self, data): + result = { + 'name': data.get('name', ''), + 'size': data.get('OS-EXT-IMG-SIZE:size', ''), + 'status': data.get('status', ''), + 'time': data.get('updated', '') + } + return result + + def load_image(self, args): + thread = threading.Thread(target=self._load_images) + thread.start() + return result_handler(consts.API_SUCCESS, {}) + + def _load_images(self): + LOG.info('source openrc') + source_env(consts.OPENRC) + + LOG.info('clean images') + cmd = [consts.CLEAN_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=consts.REPOS_DIR) + _, err = p.communicate() + if p.returncode != 0: + LOG.error('clean image failed: %s', err) + + LOG.info('load images') + cmd = [consts.LOAD_IMAGES_SCRIPT] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=consts.REPOS_DIR) + _, err = p.communicate() + if p.returncode != 0: + LOG.error('load image failed: %s', err) + + LOG.info('Done') diff --git a/api/resources/v2/openrcs.py b/api/resources/v2/openrcs.py new file mode 100644 index 000000000..cb506d0e8 --- /dev/null +++ b/api/resources/v2/openrcs.py @@ -0,0 +1,219 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import logging +import re +import os + +import yaml +from oslo_serialization import jsonutils + +from api import ApiResource +from api.database.v2.handlers import V2OpenrcHandler +from api.database.v2.handlers import V2EnvironmentHandler +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler +from yardstick.common.utils import makedirs +from yardstick.common.utils import source_env + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Openrcs(ApiResource): + + def post(self): + return self._dispatch_post() + + def upload_openrc(self, args): + try: + upload_file = args['file'] + except KeyError: + return result_handler(consts.API_ERROR, 'file must be provided') + + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + LOG.info('writing openrc: %s', consts.OPENRC) + makedirs(consts.CONF_DIR) + upload_file.save(consts.OPENRC) + source_env(consts.OPENRC) + + LOG.info('parsing openrc') + try: + openrc_data = self._get_openrc_dict() + except Exception: + LOG.exception('parse openrc failed') + return result_handler(consts.API_ERROR, 'parse openrc failed') + + openrc_id = str(uuid.uuid4()) + self._write_into_database(environment_id, openrc_id, openrc_data) + + LOG.info('writing ansible cloud conf') + try: + self._generate_ansible_conf_file(openrc_data) + except Exception: + LOG.exception('write cloud conf failed') + return result_handler(consts.API_ERROR, 'genarate ansible conf failed') + LOG.info('finish writing ansible cloud conf') + + return result_handler(consts.API_SUCCESS, {'openrc': openrc_data, 'uuid': openrc_id}) + + def update_openrc(self, args): + try: + openrc_vars = args['openrc'] + except KeyError: + return result_handler(consts.API_ERROR, 'openrc must be provided') + + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + LOG.info('writing openrc: %s', consts.OPENRC) + makedirs(consts.CONF_DIR) + + lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()] + LOG.debug('writing: %s', ''.join(lines)) + with open(consts.OPENRC, 'w') as f: + f.writelines(lines) + LOG.info('writing openrc: Done') + + LOG.info('source openrc: %s', consts.OPENRC) + try: + source_env(consts.OPENRC) + except Exception: + LOG.exception('source openrc failed') + return result_handler(consts.API_ERROR, 'source openrc failed') + LOG.info('source openrc: Done') + + openrc_id = str(uuid.uuid4()) + self._write_into_database(environment_id, openrc_id, openrc_vars) + + LOG.info('writing ansible cloud conf') + try: + self._generate_ansible_conf_file(openrc_vars) + except Exception: + LOG.exception('write cloud conf failed') + return result_handler(consts.API_ERROR, 'genarate ansible conf failed') + LOG.info('finish writing ansible cloud conf') + + return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars, 'uuid': openrc_id}) + + def _write_into_database(self, environment_id, openrc_id, openrc_data): + LOG.info('writing openrc to database') + openrc_handler = V2OpenrcHandler() + openrc_init_data = { + 'uuid': openrc_id, + 'environment_id': environment_id, + 'content': jsonutils.dumps(openrc_data) + } + openrc_handler.insert(openrc_init_data) + + LOG.info('binding openrc to environment: %s', environment_id) + environment_handler = V2EnvironmentHandler() + environment_handler.update_attr(environment_id, {'openrc_id': openrc_id}) + + def _get_openrc_dict(self): + with open(consts.OPENRC) as f: + content = f.readlines() + + result = {} + for line in content: + m = re.search(r'(\ .*)=(.*)', line) + if m: + try: + value = os.environ[m.group(1).strip()] + except KeyError: + pass + else: + result.update({m.group(1).strip(): value}) + + return result + + def _generate_ansible_conf_file(self, openrc_data): + ansible_conf = { + 'clouds': { + 'opnfv': { + 'auth': { + } + } + } + } + black_list = ['OS_IDENTITY_API_VERSION', 'OS_IMAGE_API_VERSION'] + + for k, v in openrc_data.items(): + if k.startswith('OS') and k not in black_list: + key = k[3:].lower() + ansible_conf['clouds']['opnfv']['auth'][key] = v + + try: + value = openrc_data['OS_IDENTITY_API_VERSION'] + except KeyError: + pass + else: + ansible_conf['clouds']['opnfv']['identity_api_version'] = value + + makedirs(consts.OPENSTACK_CONF_DIR) + with open(consts.CLOUDS_CONF, 'w') as f: + yaml.dump(ansible_conf, f, default_flow_style=False) + + +class V2Openrc(ApiResource): + + def get(self, openrc_id): + try: + uuid.UUID(openrc_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid openrc id') + + LOG.info('Geting openrc: %s', openrc_id) + openrc_handler = V2OpenrcHandler() + try: + openrc = openrc_handler.get_by_uuid(openrc_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such openrc id') + + LOG.info('load openrc content') + content = jsonutils.loads(openrc.content) + + return result_handler(consts.API_ERROR, {'openrc': content}) + + def delete(self, openrc_id): + try: + uuid.UUID(openrc_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid openrc id') + + LOG.info('Geting openrc: %s', openrc_id) + openrc_handler = V2OpenrcHandler() + try: + openrc = openrc_handler.get_by_uuid(openrc_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such openrc id') + + LOG.info('update openrc in environment') + environment_handler = V2EnvironmentHandler() + environment_handler.update_attr(openrc.environment_id, {'openrc_id': None}) + + openrc_handler.delete_by_uuid(openrc_id) + + return result_handler(consts.API_SUCCESS, {'openrc': openrc_id}) diff --git a/api/resources/v2/pods.py b/api/resources/v2/pods.py new file mode 100644 index 000000000..f2316d353 --- /dev/null +++ b/api/resources/v2/pods.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import yaml +import logging + +from oslo_serialization import jsonutils + +from api import ApiResource +from api.database.v2.handlers import V2PodHandler +from api.database.v2.handlers import V2EnvironmentHandler +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler +from yardstick.common.task_template import TaskTemplate + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Pods(ApiResource): + + def post(self): + return self._dispatch_post() + + def upload_pod_file(self, args): + try: + upload_file = args['file'] + except KeyError: + return result_handler(consts.API_ERROR, 'file must be provided') + + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + LOG.info('writing pod file: %s', consts.POD_FILE) + upload_file.save(consts.POD_FILE) + + with open(consts.POD_FILE) as f: + data = yaml.safe_load(TaskTemplate.render(f.read())) + LOG.debug('pod content is: %s', data) + + LOG.info('create pod in database') + pod_id = str(uuid.uuid4()) + pod_handler = V2PodHandler() + pod_init_data = { + 'uuid': pod_id, + 'environment_id': environment_id, + 'content': jsonutils.dumps(data) + } + pod_handler.insert(pod_init_data) + + LOG.info('update pod in environment') + environment_handler = V2EnvironmentHandler() + environment_handler.update_attr(environment_id, {'pod_id': pod_id}) + + return result_handler(consts.API_SUCCESS, {'uuid': pod_id, 'pod': data}) + + +class V2Pod(ApiResource): + + def get(self, pod_id): + try: + uuid.UUID(pod_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid pod id') + + pod_handler = V2PodHandler() + try: + pod = pod_handler.get_by_uuid(pod_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such pod') + + content = jsonutils.loads(pod.content) + + return result_handler(consts.API_SUCCESS, {'pod': content}) + + def delete(self, pod_id): + try: + uuid.UUID(pod_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid pod id') + + pod_handler = V2PodHandler() + try: + pod = pod_handler.get_by_uuid(pod_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such pod') + + LOG.info('update pod in environment') + environment_handler = V2EnvironmentHandler() + environment_handler.update_attr(pod.environment_id, {'pod_id': None}) + + LOG.info('delete pod in database') + pod_handler.delete_by_uuid(pod_id) + + return result_handler(consts.API_SUCCESS, {'pod': pod_id}) diff --git a/api/resources/v2/projects.py b/api/resources/v2/projects.py new file mode 100644 index 000000000..2ff61d0fe --- /dev/null +++ b/api/resources/v2/projects.py @@ -0,0 +1,105 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import logging + +from datetime import datetime + +from api import ApiResource +from api.database.v2.handlers import V2ProjectHandler +from api.database.v2.handlers import V2TaskHandler +from yardstick.common.utils import result_handler +from yardstick.common.utils import change_obj_to_dict +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Projects(ApiResource): + + def get(self): + project_handler = V2ProjectHandler() + projects = [change_obj_to_dict(p) for p in project_handler.list_all()] + + for p in projects: + tasks = p['tasks'] + p['tasks'] = tasks.split(',') if tasks else [] + + return result_handler(consts.API_SUCCESS, {'projects': projects}) + + def post(self): + return self._dispatch_post() + + def create_project(self, args): + try: + name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'name must be provided') + + project_id = str(uuid.uuid4()) + create_time = datetime.now() + project_handler = V2ProjectHandler() + + project_init_data = { + 'uuid': project_id, + 'name': name, + 'time': create_time + } + project_handler.insert(project_init_data) + + return result_handler(consts.API_SUCCESS, {'uuid': project_id}) + + +class V2Project(ApiResource): + + def get(self, project_id): + try: + uuid.UUID(project_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid project id') + + project_handler = V2ProjectHandler() + try: + project = project_handler.get_by_uuid(project_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such project id') + + project_info = change_obj_to_dict(project) + tasks = project_info['tasks'] + project_info['tasks'] = tasks.split(',') if tasks else [] + + return result_handler(consts.API_SUCCESS, {'project': project_info}) + + def delete(self, project_id): + try: + uuid.UUID(project_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid project id') + + project_handler = V2ProjectHandler() + try: + project = project_handler.get_by_uuid(project_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such project id') + + if project.tasks: + LOG.info('delete related task') + task_handler = V2TaskHandler() + for task_id in project.tasks.split(','): + LOG.debug('delete task: %s', task_id) + try: + task_handler.delete_by_uuid(task_id) + except ValueError: + LOG.exception('no such task id: %s', task_id) + + LOG.info('delete project in database') + project_handler.delete_by_uuid(project_id) + + return result_handler(consts.API_SUCCESS, {'project': project_id}) diff --git a/api/resources/v2/tasks.py b/api/resources/v2/tasks.py new file mode 100644 index 000000000..885a190c6 --- /dev/null +++ b/api/resources/v2/tasks.py @@ -0,0 +1,254 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import uuid +import logging +from datetime import datetime + +from oslo_serialization import jsonutils + +from api import ApiResource +from api.database.v2.handlers import V2TaskHandler +from api.database.v2.handlers import V2ProjectHandler +from api.database.v2.handlers import V2EnvironmentHandler +from api.utils.thread import TaskThread +from yardstick.common.utils import result_handler +from yardstick.common.utils import change_obj_to_dict +from yardstick.common import constants as consts +from yardstick.benchmark.core.task import Task +from yardstick.benchmark.core import Param + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Tasks(ApiResource): + + def get(self): + task_handler = V2TaskHandler() + tasks = [change_obj_to_dict(t) for t in task_handler.list_all()] + + for t in tasks: + result = t['result'] + t['result'] = jsonutils.loads(result) if result else None + + return result_handler(consts.API_SUCCESS, {'tasks': tasks}) + + def post(self): + return self._dispatch_post() + + def create_task(self, args): + try: + name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'name must be provided') + + try: + project_id = args['project_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'project_id must be provided') + + task_id = str(uuid.uuid4()) + create_time = datetime.now() + task_handler = V2TaskHandler() + + LOG.info('create task in database') + task_init_data = { + 'uuid': task_id, + 'project_id': project_id, + 'name': name, + 'time': create_time, + 'status': -1 + } + task_handler.insert(task_init_data) + + LOG.info('create task in project') + project_handler = V2ProjectHandler() + project_handler.append_attr(project_id, {'tasks': task_id}) + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + + +class V2Task(ApiResource): + + def get(self, task_id): + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task id') + + task_handler = V2TaskHandler() + try: + task = task_handler.get_by_uuid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such task id') + + task_info = change_obj_to_dict(task) + result = task_info['result'] + task_info['result'] = jsonutils.loads(result) if result else None + + return result_handler(consts.API_SUCCESS, {'task': task_info}) + + def delete(self, task_id): + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task id') + + task_handler = V2TaskHandler() + try: + project_id = task_handler.get_by_uuid(task_id).project_id + except ValueError: + return result_handler(consts.API_ERROR, 'no such task id') + + LOG.info('delete task in database') + task_handler.delete_by_uuid(task_id) + + project_handler = V2ProjectHandler() + project = project_handler.get_by_uuid(project_id) + + if project.tasks: + LOG.info('update tasks in project') + new_task_list = project.tasks.split(',') + new_task_list.remove(task_id) + if new_task_list: + new_tasks = ','.join(new_task_list) + else: + new_tasks = None + project_handler.update_attr(project_id, {'tasks': new_tasks}) + + return result_handler(consts.API_SUCCESS, {'task': task_id}) + + def put(self, task_id): + + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task id') + + task_handler = V2TaskHandler() + try: + task_handler.get_by_uuid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such task id') + + return self._dispatch_post(task_id=task_id) + + def add_environment(self, args): + + task_id = args['task_id'] + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment id') + + LOG.info('update environment_id in task') + task_handler = V2TaskHandler() + task_handler.update_attr(task_id, {'environment_id': environment_id}) + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + + def add_case(self, args): + task_id = args['task_id'] + try: + name = args['case_name'] + except KeyError: + return result_handler(consts.API_ERROR, 'case_name must be provided') + + try: + content = args['case_content'] + except KeyError: + return result_handler(consts.API_ERROR, 'case_content must be provided') + + LOG.info('update case info in task') + task_handler = V2TaskHandler() + task_update_data = { + 'case_name': name, + 'content': content, + 'suite': False + } + task_handler.update_attr(task_id, task_update_data) + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + + def add_suite(self, args): + task_id = args['task_id'] + try: + name = args['suite_name'] + except KeyError: + return result_handler(consts.API_ERROR, 'suite_name must be provided') + + try: + content = args['suite_content'] + except KeyError: + return result_handler(consts.API_ERROR, 'suite_content must be provided') + + LOG.info('update suite info in task') + task_handler = V2TaskHandler() + task_update_data = { + 'case_name': name, + 'content': content, + 'suite': True + } + task_handler.update_attr(task_id, task_update_data) + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) + + def run(self, args): + try: + task_id = args['task_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'task_id must be provided') + + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task id') + + task_handler = V2TaskHandler() + try: + task = task_handler.get_by_uuid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such task id') + + if not task.environment_id: + return result_handler(consts.API_ERROR, 'environment not set') + + if not task.case_name or not task.content: + return result_handler(consts.API_ERROR, 'case not set') + + if task.status == 0: + return result_handler(consts.API_ERROR, 'task is already running') + + with open('/tmp/{}.yaml'.format(task.case_name), 'w') as f: + f.write(task.content) + + data = { + 'inputfile': ['/tmp/{}.yaml'.format(task.case_name)], + 'task_id': task_id + } + if task.suite: + data.update({'suite': True}) + + LOG.info('start task thread') + param = Param(data) + task_thread = TaskThread(Task().start, param, task_handler) + task_thread.start() + + return result_handler(consts.API_SUCCESS, {'uuid': task_id}) diff --git a/api/resources/v2/testcases.py b/api/resources/v2/testcases.py new file mode 100644 index 000000000..316ef2664 --- /dev/null +++ b/api/resources/v2/testcases.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging +import errno +import os + +import jinja2schema + +from api import ApiResource +from yardstick.common.utils import result_handler +from yardstick.common import constants as consts +from yardstick.benchmark.core import Param +from yardstick.benchmark.core.testcase import Testcase + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Testcases(ApiResource): + + def get(self): + param = Param({}) + testcase_list = Testcase().list_all(param) + return result_handler(consts.API_SUCCESS, {'testcases': testcase_list}) + + def post(self): + return self._dispatch_post() + + def upload_case(self, args): + try: + upload_file = args['file'] + except KeyError: + return result_handler(consts.API_ERROR, 'file must be provided') + + case_name = os.path.join(consts.TESTCASE_DIR, upload_file.filename) + + LOG.info('save case file') + upload_file.save(case_name) + + return result_handler(consts.API_SUCCESS, {'testcase': upload_file.filename}) + + +class V2Testcase(ApiResource): + + def get(self, case_name): + case_path = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(case_name)) + + try: + with open(case_path) as f: + data = f.read() + except IOError as e: + if e.errno == errno.ENOENT: + return result_handler(consts.API_ERROR, 'case does not exist') + + options = {k: {'description': '', 'type': v.__class__.__name__} + for k, v in jinja2schema.infer(data).items()} + + return result_handler(consts.API_SUCCESS, {'testcase': data, 'args': options}) + + def delete(self, case_name): + case_path = os.path.join(consts.TESTCASE_DIR, '{}.yaml'.format(case_name)) + + try: + os.remove(case_path) + except IOError as e: + if e.errno == errno.ENOENT: + return result_handler(consts.API_ERROR, 'case does not exist') + + return result_handler(consts.API_SUCCESS, {'testcase': case_name}) diff --git a/api/resources/v2/testsuites.py b/api/resources/v2/testsuites.py new file mode 100644 index 000000000..56ad47375 --- /dev/null +++ b/api/resources/v2/testsuites.py @@ -0,0 +1,89 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import errno +import logging + +import yaml + +from api import ApiResource +from yardstick.common.utils import result_handler +from yardstick.common import constants as consts +from yardstick.benchmark.core.testsuite import Testsuite +from yardstick.benchmark.core import Param + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +class V2Testsuites(ApiResource): + + def get(self): + param = Param({}) + testsuite_list = Testsuite().list_all(param) + + data = { + 'testsuites': testsuite_list + } + + return result_handler(consts.API_SUCCESS, data) + + def post(self): + return self._dispatch_post() + + def create_suite(self, args): + try: + suite_name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'name must be provided') + + try: + testcases = args['testcases'] + except KeyError: + return result_handler(consts.API_ERROR, 'testcases must be provided') + + testcases = [{'file_name': '{}.yaml'.format(t)} for t in testcases] + + suite = os.path.join(consts.TESTSUITE_DIR, '{}.yaml'.format(suite_name)) + suite_content = { + 'schema': 'yardstick:suite:0.1', + 'name': suite_name, + 'test_cases_dir': 'tests/opnfv/test_cases/', + 'test_cases': testcases + } + + LOG.info('write test suite') + with open(suite, 'w') as f: + yaml.dump(suite_content, f, default_flow_style=False) + + return result_handler(consts.API_SUCCESS, {'suite': suite_name}) + + +class V2Testsuite(ApiResource): + + def get(self, suite_name): + suite_path = os.path.join(consts.TESTSUITE_DIR, '{}.yaml'.format(suite_name)) + try: + with open(suite_path) as f: + data = f.read() + except IOError as e: + if e.errno == errno.ENOENT: + return result_handler(consts.API_ERROR, 'suite does not exist') + + return result_handler(consts.API_SUCCESS, {'testsuite': data}) + + def delete(self, suite_name): + suite_path = os.path.join(consts.TESTSUITE_DIR, '{}.yaml'.format(suite_name)) + try: + os.remove(suite_path) + except IOError as e: + if e.errno == errno.ENOENT: + return result_handler(consts.API_ERROR, 'suite does not exist') + + return result_handler(consts.API_SUCCESS, {'testsuite': suite_name}) diff --git a/api/server.py b/api/server.py index d39c44544..158b8a508 100644 --- a/api/server.py +++ b/api/server.py @@ -10,6 +10,7 @@ from __future__ import absolute_import import inspect import logging +import socket from six.moves import filter from flasgger import Swagger @@ -21,9 +22,17 @@ from api.database import db_session from api.database import engine from api.database.v1 import models from api.urls import urlpatterns +from api import ApiResource from yardstick import _init_logging +from yardstick.common import utils +from yardstick.common import constants as consts -logger = logging.getLogger(__name__) +try: + from urlparse import urljoin +except ImportError: + from urllib.parse import urljoin + +LOG = logging.getLogger(__name__) app = Flask(__name__) @@ -37,8 +46,10 @@ def shutdown_session(exception=None): db_session.remove() -for u in urlpatterns: - api.add_resource(u.resource, u.url, endpoint=u.endpoint) +def get_resource(resource_name): + name = ''.join(resource_name.split('_')) + return next((r for r in utils.itersubclasses(ApiResource) + if r.__name__.lower() == name)) def init_db(): @@ -51,7 +62,7 @@ def init_db(): return False subclses = filter(func, inspect.getmembers(models, inspect.isclass)) - logger.debug('Import models: %s', [a[1] for a in subclses]) + LOG.debug('Import models: %s', [a[1] for a in subclses]) Base.metadata.create_all(bind=engine) @@ -60,9 +71,21 @@ def app_wrapper(*args, **kwargs): return app(*args, **kwargs) +def get_endpoint(url): + ip = socket.gethostbyname(socket.gethostname()) + return urljoin('http://{}:{}'.format(ip, consts.API_PORT), url) + + +for u in urlpatterns: + try: + api.add_resource(get_resource(u.target), u.url, endpoint=get_endpoint(u.url)) + except StopIteration: + LOG.error('url resource not found: %s', u.url) + + if __name__ == '__main__': _init_logging() - logger.setLevel(logging.DEBUG) - logger.info('Starting server') + LOG.setLevel(logging.DEBUG) + LOG.info('Starting server') init_db() app.run(host='0.0.0.0') diff --git a/api/urls.py b/api/urls.py index 13c6c7675..3fef91af8 100644 --- a/api/urls.py +++ b/api/urls.py @@ -8,17 +8,52 @@ ############################################################################## from __future__ import absolute_import -from api import views -from api.utils.common import Url +from api import Url urlpatterns = [ - Url('/yardstick/asynctask', views.Asynctask, 'asynctask'), - Url('/yardstick/testcases', views.Testcases, 'testcases'), - Url('/yardstick/testcases/release/action', views.ReleaseAction, 'release'), - Url('/yardstick/testcases/samples/action', views.SamplesAction, 'samples'), - Url('/yardstick/testcases/<case_name>/docs', views.CaseDocs, 'casedocs'), - Url('/yardstick/testsuites/action', views.TestsuitesAction, 'testsuites'), - Url('/yardstick/results', views.Results, 'results'), - Url('/yardstick/env/action', views.EnvAction, 'env') + Url('/yardstick/asynctask', 'v1_async_task'), + Url('/yardstick/testcases', 'v1_test_case'), + Url('/yardstick/testcases/release/action', 'v1_release_case'), + Url('/yardstick/testcases/samples/action', 'v1_sample_case'), + Url('/yardstick/testcases/<case_name>/docs', 'v1_case_docs'), + Url('/yardstick/testsuites/action', 'v1_test_suite'), + Url('/yardstick/results', 'v1_result'), + Url('/yardstick/env/action', 'v1_env'), + + # api v2 + Url('/api/v2/yardstick/environments', 'v2_environments'), + Url('/api/v2/yardstick/environments/action', 'v2_environments'), + Url('/api/v2/yardstick/environments/<environment_id>', 'v2_environment'), + + Url('/api/v2/yardstick/openrcs', 'v2_openrcs'), + Url('/api/v2/yardstick/openrcs/action', 'v2_openrcs'), + Url('/api/v2/yardstick/openrcs/<openrc_id>', 'v2_openrc'), + + Url('/api/v2/yardstick/pods', 'v2_pods'), + Url('/api/v2/yardstick/pods/action', 'v2_pods'), + Url('/api/v2/yardstick/pods/<pod_id>', 'v2_pod'), + + Url('/api/v2/yardstick/images', 'v2_images'), + Url('/api/v2/yardstick/images/action', 'v2_images'), + + Url('/api/v2/yardstick/containers', 'v2_containers'), + Url('/api/v2/yardstick/containers/action', 'v2_containers'), + Url('/api/v2/yardstick/containers/<container_id>', 'v2_container'), + + Url('/api/v2/yardstick/projects', 'v2_projects'), + Url('/api/v2/yardstick/projects/action', 'v2_projects'), + Url('/api/v2/yardstick/projects/<project_id>', 'v2_project'), + + Url('/api/v2/yardstick/tasks', 'v2_tasks'), + Url('/api/v2/yardstick/tasks/action', 'v2_tasks'), + Url('/api/v2/yardstick/tasks/<task_id>', 'v2_task'), + + Url('/api/v2/yardstick/testcases', 'v2_testcases'), + Url('/api/v2/yardstick/testcases/action', 'v2_testcases'), + Url('/api/v2/yardstick/testcases/<case_name>', 'v2_testcase'), + + Url('/api/v2/yardstick/testsuites', 'v2_testsuites'), + Url('/api/v2/yardstick/testsuites/action', 'v2_testsuites'), + Url('/api/v2/yardstick/testsuites/<suite_name>', 'v2_testsuite') ] diff --git a/api/utils/common.py b/api/utils/common.py deleted file mode 100644 index eda9c17dd..000000000 --- a/api/utils/common.py +++ /dev/null @@ -1,44 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import collections -import logging - -from flask import jsonify -import six - -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.DEBUG) - - -def translate_to_str(obj): - if isinstance(obj, collections.Mapping): - return {str(k): translate_to_str(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [translate_to_str(ele) for ele in obj] - elif isinstance(obj, six.text_type): - return str(obj) - return obj - - -def result_handler(status, data): - result = { - 'status': status, - 'result': data - } - return jsonify(result) - - -class Url(object): - - def __init__(self, url, resource, endpoint): - super(Url, self).__init__() - self.url = url - self.resource = resource - self.endpoint = endpoint diff --git a/api/utils/thread.py b/api/utils/thread.py index 2106548f5..20bd07a12 100644 --- a/api/utils/thread.py +++ b/api/utils/thread.py @@ -1,37 +1,53 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## import threading +import os import logging from oslo_serialization import jsonutils -from api.database.v1.handlers import TasksHandler from yardstick.common import constants as consts -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) class TaskThread(threading.Thread): - def __init__(self, target, args): + def __init__(self, target, args, handler): super(TaskThread, self).__init__(target=target, args=args) self.target = target self.args = args + self.handler = handler def run(self): - task_handler = TasksHandler() - data = {'task_id': self.args.task_id, 'status': consts.TASK_NOT_DONE} - task_handler.insert(data) + if self.handler.__class__.__name__.lower().startswith('v2'): + self.handler.update_attr(self.args.task_id, {'status': consts.TASK_NOT_DONE}) + else: + update_data = {'task_id': self.args.task_id, 'status': consts.TASK_NOT_DONE} + self.handler.insert(update_data) - logger.info('Starting run task') + LOG.info('Starting run task') try: data = self.target(self.args) except Exception as e: - logger.exception('Task Failed') + LOG.exception('Task Failed') update_data = {'status': consts.TASK_FAILED, 'error': str(e)} - task_handler.update_attr(self.args.task_id, update_data) + self.handler.update_attr(self.args.task_id, update_data) else: - logger.info('Task Finished') - logger.debug('Result: %s', data) - - data['result'] = jsonutils.dumps(data.get('result', {})) - task_handler.update_attr(self.args.task_id, data) + LOG.info('Task Finished') + LOG.debug('Result: %s', data) + + if self.handler.__class__.__name__.lower().startswith('v2'): + new_data = {'status': consts.TASK_DONE, 'result': jsonutils.dumps(data['result'])} + self.handler.update_attr(self.args.task_id, new_data) + os.remove(self.args.inputfile[0]) + else: + data['result'] = jsonutils.dumps(data.get('result', {})) + self.handler.update_attr(self.args.task_id, data) diff --git a/api/views.py b/api/views.py deleted file mode 100644 index 9c9ca4ef9..000000000 --- a/api/views.py +++ /dev/null @@ -1,82 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -from __future__ import absolute_import -import logging -import os - -from flasgger.utils import swag_from - -from api.base import ApiResource -from api.swagger import models - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -TestCaseActionModel = models.TestCaseActionModel -TestCaseActionArgsModel = models.TestCaseActionArgsModel -TestCaseActionArgsOptsModel = models.TestCaseActionArgsOptsModel -TestCaseActionArgsOptsTaskArgModel = models.TestCaseActionArgsOptsTaskArgModel - - -class Asynctask(ApiResource): - def get(self): - return self._dispatch_get() - - -class Testcases(ApiResource): - def get(self): - return self._dispatch_get() - - -class ReleaseAction(ApiResource): - @swag_from(os.getcwd() + '/swagger/docs/release_action.yaml') - def post(self): - return self._dispatch_post() - - -class SamplesAction(ApiResource): - - def post(self): - return self._dispatch_post() - - -TestSuiteActionModel = models.TestSuiteActionModel -TestSuiteActionArgsModel = models.TestSuiteActionArgsModel -TestSuiteActionArgsOptsModel = models.TestSuiteActionArgsOptsModel -TestSuiteActionArgsOptsTaskArgModel = \ - models.TestSuiteActionArgsOptsTaskArgModel - - -class TestsuitesAction(ApiResource): - @swag_from(os.getcwd() + '/swagger/docs/testsuites_action.yaml') - def post(self): - return self._dispatch_post() - - -ResultModel = models.ResultModel - - -class Results(ApiResource): - - @swag_from(os.getcwd() + '/swagger/docs/results.yaml') - def get(self): - return self._dispatch_get() - - -class EnvAction(ApiResource): - - def post(self): - return self._dispatch_post() - - -class CaseDocs(ApiResource): - - def get(self, case_name): - return self._dispatch_get(case_name=case_name) |