diff options
Diffstat (limited to 'api/resources')
| -rw-r--r-- | api/resources/v1/env.py | 12 | ||||
| -rw-r--r-- | api/resources/v1/tasks.py | 50 | ||||
| -rw-r--r-- | api/resources/v2/environments.py | 8 | ||||
| -rw-r--r-- | api/resources/v2/images.py | 341 | ||||
| -rw-r--r-- | api/resources/v2/pods.py | 3 | ||||
| -rw-r--r-- | api/resources/v2/testcases.py | 8 |
6 files changed, 386 insertions, 36 deletions
diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py index 8367fa9eb..04cc659c7 100644 --- a/api/resources/v1/env.py +++ b/api/resources/v1/env.py @@ -31,7 +31,7 @@ from yardstick.common import utils from yardstick.common.utils import result_handler from yardstick.common import openstack_utils from yardstick.common.httpClient import HttpClient - +from yardstick.common.yaml_loader import yaml_load LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -88,13 +88,13 @@ class V1Env(ApiResource): def _create_dashboard(self, ip): url = 'http://admin:admin@{}:{}/api/dashboards/db'.format(ip, consts.GRAFANA_PORT) - path = os.path.join(consts.REPOS_DIR, 'dashboard', '*dashboard.json') + path = os.path.join(consts.REPOS_DIR, 'dashboard', 'opnfv_yardstick_tc*.json') for i in sorted(glob.iglob(path)): with open(i) as f: data = jsonutils.load(f) try: - HttpClient().post(url, data) + HttpClient().post(url, {"dashboard": data}) except Exception: LOG.exception('Create dashboard %s failed', i) raise @@ -120,10 +120,10 @@ class V1Env(ApiResource): "basicAuth": True, "basicAuthUser": "admin", "basicAuthPassword": "admin", - "isDefault": False, + "isDefault": True, } try: - HttpClient().post(url, data) + HttpClient().post(url, data, timeout=10) except Exception: LOG.exception('Create datasources failed') raise @@ -393,7 +393,7 @@ class V1Env(ApiResource): return result_handler(consts.API_ERROR, 'file must be provided') LOG.info('Checking file') - data = yaml.safe_load(pod_file.read()) + data = yaml_load(pod_file.read()) if not isinstance(data, collections.Mapping): return result_handler(consts.API_ERROR, 'invalid yaml file') diff --git a/api/resources/v1/tasks.py b/api/resources/v1/tasks.py new file mode 100644 index 000000000..52455fbf5 --- /dev/null +++ b/api/resources/v1/tasks.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import errno +import uuid + +from api import ApiResource +from api.database.v1.handlers import TasksHandler +from yardstick.common import constants as consts +from yardstick.common.utils import result_handler + + +class V1TaskLog(ApiResource): + def get(self, task_id): + + try: + uuid.UUID(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + task_handler = TasksHandler() + try: + task = task_handler.get_task_by_taskid(task_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid task_id') + + index = int(self._get_args().get('index', 0)) + + try: + with open(os.path.join(consts.TASK_LOG_DIR, '{}.log'.format(task_id))) as f: + f.seek(index) + data = f.readlines() + index = f.tell() + except OSError as e: + if e.errno == errno.ENOENT: + return result_handler(consts.API_ERROR, 'log file does not exist') + return result_handler(consts.API_ERROR, 'error with log file') + + return_data = { + 'index': index, + 'data': data + } + + return result_handler(task.status, return_data) diff --git a/api/resources/v2/environments.py b/api/resources/v2/environments.py index f021a3c5a..158e98be7 100644 --- a/api/resources/v2/environments.py +++ b/api/resources/v2/environments.py @@ -35,6 +35,9 @@ class V2Environments(ApiResource): container_info = e['container_id'] e['container_id'] = jsonutils.loads(container_info) if container_info else {} + image_id = e['image_id'] + e['image_id'] = image_id.split(',') if image_id else [] + data = { 'environments': environments } @@ -78,8 +81,13 @@ class V2Environment(ApiResource): return result_handler(consts.API_ERROR, 'no such environment id') environment = change_obj_to_dict(environment) + container_id = environment['container_id'] environment['container_id'] = jsonutils.loads(container_id) if container_id else {} + + image_id = environment['image_id'] + environment['image_id'] = image_id.split(',') if image_id else [] + return result_handler(consts.API_SUCCESS, {'environment': environment}) def delete(self, environment_id): diff --git a/api/resources/v2/images.py b/api/resources/v2/images.py index 8359e105b..0c36a0a26 100644 --- a/api/resources/v2/images.py +++ b/api/resources/v2/images.py @@ -7,76 +7,361 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import logging -import subprocess +import os +import uuid import threading +import requests +import datetime from api import ApiResource +from api.database.v2.handlers import V2ImageHandler +from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common.utils import result_handler from yardstick.common.utils import source_env from yardstick.common.utils import change_obj_to_dict from yardstick.common.openstack_utils import get_nova_client +from yardstick.common.openstack_utils import get_glance_client from yardstick.common import constants as consts LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) +IMAGE_MAP = { + 'yardstick-image': { + 'path': os.path.join(consts.IMAGE_DIR, 'yardstick-image.img'), + 'url': 'http://artifacts.opnfv.org/yardstick/images/yardstick-image.img' + }, + 'Ubuntu-16.04': { + 'path': os.path.join(consts.IMAGE_DIR, 'xenial-server-cloudimg-amd64-disk1.img'), + 'url': 'cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img' + }, + 'cirros-0.3.5': { + 'path': os.path.join(consts.IMAGE_DIR, 'cirros-0.3.5-x86_64-disk.img'), + 'url': 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img' + } +} + class V2Images(ApiResource): def get(self): try: source_env(consts.OPENRC) - except: + except Exception: return result_handler(consts.API_ERROR, 'source openrc error') nova_client = get_nova_client() try: images_list = nova_client.images.list() - except: + except Exception: return result_handler(consts.API_ERROR, 'get images error') else: - images = [self.get_info(change_obj_to_dict(i)) for i in images_list] - status = 1 if all(i['status'] == 'ACTIVE' for i in images) else 0 - if not images: - status = 0 + images = {i.name: self.get_info(change_obj_to_dict(i)) for i in images_list} - return result_handler(consts.API_SUCCESS, {'status': status, 'images': images}) + return result_handler(consts.API_SUCCESS, {'status': 1, 'images': images}) def post(self): return self._dispatch_post() def get_info(self, data): + try: + size = data['OS-EXT-IMG-SIZE:size'] + except KeyError: + size = None + else: + size = float(size) / 1024 / 1024 + result = { 'name': data.get('name', ''), - 'size': data.get('OS-EXT-IMG-SIZE:size', ''), - 'status': data.get('status', ''), - 'time': data.get('updated', '') + 'discription': data.get('description', ''), + 'size': size, + 'status': data.get('status'), + 'time': data.get('updated') } return result def load_image(self, args): - thread = threading.Thread(target=self._load_images) + try: + image_name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'image name must provided') + + if image_name not in IMAGE_MAP: + return result_handler(consts.API_ERROR, 'wrong image name') + + thread = threading.Thread(target=self._do_load_image, args=(image_name,)) thread.start() + return result_handler(consts.API_SUCCESS, {'image': image_name}) + + def upload_image(self, args): + try: + image_file = args['file'] + except KeyError: + return result_handler(consts.API_ERROR, 'file must be provided') + + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment') + + file_path = os.path.join(consts.IMAGE_DIR, image_file.filename) + LOG.info('saving file') + image_file.save(file_path) + + LOG.info('loading image') + self._load_image(image_file.filename, file_path) + + LOG.info('creating image in DB') + image_handler = V2ImageHandler() + image_id = str(uuid.uuid4()) + image_init_data = { + 'uuid': image_id, + 'name': image_file.filename, + 'environment_id': environment_id + } + image_handler.insert(image_init_data) + + LOG.info('update image in environment') + if environment.image_id: + image_list = environment.image_id.split(',') + image_list.append(image_id) + new_image_id = ','.join(image_list) + else: + new_image_id = image_id + + environment_handler.update_attr(environment_id, {'image_id': new_image_id}) + + return result_handler(consts.API_SUCCESS, {'uuid': image_id}) + + def upload_image_by_url(self, args): + try: + url = args['url'] + except KeyError: + return result_handler(consts.API_ERROR, 'url must be provided') + + try: + environment_id = args['environment_id'] + except KeyError: + return result_handler(consts.API_ERROR, 'environment_id must be provided') + + try: + uuid.UUID(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid environment id') + + environment_handler = V2EnvironmentHandler() + try: + environment = environment_handler.get_by_uuid(environment_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such environment') + + thread = threading.Thread(target=self._do_upload_image_by_url, args=(url,)) + thread.start() + + file_name = url.split('/')[-1] + + LOG.info('creating image in DB') + image_handler = V2ImageHandler() + image_id = str(uuid.uuid4()) + image_init_data = { + 'uuid': image_id, + 'name': file_name, + 'environment_id': environment_id + } + image_handler.insert(image_init_data) + + LOG.info('update image in environment') + if environment.image_id: + image_list = environment.image_id.split(',') + image_list.append(image_id) + new_image_id = ','.join(image_list) + else: + new_image_id = image_id + + environment_handler.update_attr(environment_id, {'image_id': new_image_id}) + + return result_handler(consts.API_SUCCESS, {'uuid': image_id}) + + def delete_image(self, args): + try: + image_name = args['name'] + except KeyError: + return result_handler(consts.API_ERROR, 'image name must provided') + + if image_name not in IMAGE_MAP: + return result_handler(consts.API_ERROR, 'wrong image name') + + glance_client = get_glance_client() + try: + image = next((i for i in glance_client.images.list() if i.name == image_name)) + except StopIteration: + return result_handler(consts.API_ERROR, 'can not find image') + + glance_client.images.delete(image.id) + return result_handler(consts.API_SUCCESS, {}) - def _load_images(self): + def _do_upload_image_by_url(self, url): + file_name = url.split('/')[-1] + path = os.path.join(consts.IMAGE_DIR, file_name) + + LOG.info('download image') + self._download_image(url, path) + + LOG.info('loading image') + self._load_image(file_name, path) + + def _do_load_image(self, image_name): + if not os.path.exists(IMAGE_MAP[image_name]['path']): + self._download_image(IMAGE_MAP[image_name]['url'], + IMAGE_MAP[image_name]['path']) + + self._load_image(image_name, IMAGE_MAP[image_name]['path']) + + def _load_image(self, image_name, image_path): LOG.info('source openrc') source_env(consts.OPENRC) - LOG.info('clean images') - cmd = [consts.CLEAN_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - cwd=consts.REPOS_DIR) - _, err = p.communicate() - if p.returncode != 0: - LOG.error('clean image failed: %s', err) - - LOG.info('load images') - cmd = [consts.LOAD_IMAGES_SCRIPT] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - cwd=consts.REPOS_DIR) - _, err = p.communicate() - if p.returncode != 0: - LOG.error('load image failed: %s', err) + LOG.info('load image') + glance_client = get_glance_client() + image = glance_client.images.create(name=image_name, + visibility='public', + disk_format='qcow2', + container_format='bare') + with open(image_path, 'rb') as f: + glance_client.images.upload(image.id, f) LOG.info('Done') + + def _download_image(self, url, path): + start = datetime.datetime.now().replace(microsecond=0) + + LOG.info('download image from: %s', url) + self._download_file(url, path) + + end = datetime.datetime.now().replace(microsecond=0) + LOG.info('download image success, total: %s s', end - start) + + def _download_handler(self, start, end, url, filename): + + headers = {'Range': 'bytes=%d-%d' % (start, end)} + r = requests.get(url, headers=headers, stream=True) + + with open(filename, "r+b") as fp: + fp.seek(start) + fp.tell() + fp.write(r.content) + + def _download_file(self, url, path, num_thread=5): + + r = requests.head(url) + try: + file_size = int(r.headers['content-length']) + except Exception: + return + + with open(path, 'wb') as f: + f.truncate(file_size) + + thread_list = [] + part = file_size // num_thread + for i in range(num_thread): + start = part * i + end = start + part if i != num_thread - 1 else file_size + + kwargs = {'start': start, 'end': end, 'url': url, 'filename': path} + t = threading.Thread(target=self._download_handler, kwargs=kwargs) + t.setDaemon(True) + t.start() + thread_list.append(t) + + for t in thread_list: + t.join() + + +class V2Image(ApiResource): + def get(self, image_id): + try: + uuid.UUID(image_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid image id') + + image_handler = V2ImageHandler() + try: + image = image_handler.get_by_uuid(image_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such image id') + + nova_client = get_nova_client() + images = nova_client.images.list() + try: + image = next((i for i in images if i.name == image.name)) + except StopIteration: + pass + + return_image = self.get_info(change_obj_to_dict(image)) + return_image['id'] = image_id + + return result_handler(consts.API_SUCCESS, {'image': return_image}) + + def delete(self, image_id): + try: + uuid.UUID(image_id) + except ValueError: + return result_handler(consts.API_ERROR, 'invalid image id') + + image_handler = V2ImageHandler() + try: + image = image_handler.get_by_uuid(image_id) + except ValueError: + return result_handler(consts.API_ERROR, 'no such image id') + + LOG.info('delete image in openstack') + glance_client = get_glance_client() + try: + image_o = next((i for i in glance_client.images.list() if i.name == image.name)) + except StopIteration: + return result_handler(consts.API_ERROR, 'can not find image') + + glance_client.images.delete(image_o.id) + + LOG.info('delete image in environment') + environment_id = image.environment_id + environment_handler = V2EnvironmentHandler() + environment = environment_handler.get_by_uuid(environment_id) + image_list = environment.image_id.split(',') + image_list.remove(image_id) + environment_handler.update_attr(environment_id, {'image_id': ','.join(image_list)}) + + LOG.info('delete image in DB') + image_handler.delete_by_uuid(image_id) + + return result_handler(consts.API_SUCCESS, {'image': image_id}) + + def get_info(self, data): + try: + size = data['OS-EXT-IMG-SIZE:size'] + except KeyError: + size = None + else: + size = float(size) / 1024 / 1024 + + result = { + 'name': data.get('name', ''), + 'description': data.get('description', ''), + 'size': size, + 'status': data.get('status'), + 'time': data.get('updated') + } + return result diff --git a/api/resources/v2/pods.py b/api/resources/v2/pods.py index f2316d353..d98238ca1 100644 --- a/api/resources/v2/pods.py +++ b/api/resources/v2/pods.py @@ -18,6 +18,7 @@ from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common import constants as consts from yardstick.common.utils import result_handler from yardstick.common.task_template import TaskTemplate +from yardstick.common.yaml_loader import yaml_load LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -48,7 +49,7 @@ class V2Pods(ApiResource): upload_file.save(consts.POD_FILE) with open(consts.POD_FILE) as f: - data = yaml.safe_load(TaskTemplate.render(f.read())) + data = yaml_load(TaskTemplate.render(f.read())) LOG.debug('pod content is: %s', data) LOG.info('create pod in database') diff --git a/api/resources/v2/testcases.py b/api/resources/v2/testcases.py index 316ef2664..0edbf6285 100644 --- a/api/resources/v2/testcases.py +++ b/api/resources/v2/testcases.py @@ -60,7 +60,13 @@ class V2Testcase(ApiResource): options = {k: {'description': '', 'type': v.__class__.__name__} for k, v in jinja2schema.infer(data).items()} - + # [('segmentation_id', < scalar >), ('image', < string >), ('provider', < scalar >), + # ('physical_network', < string >), ('packetsize', < number >)] + for k, v in options.items(): + if k == 'segmentation_id': + options[k]['type'] = 'Number' + if k == 'provider': + options[k]['type'] = 'String' return result_handler(consts.API_SUCCESS, {'testcase': data, 'args': options}) def delete(self, case_name): |