aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorchenjiankun <chenjiankun1@huawei.com>2017-06-30 02:32:55 +0000
committerchenjiankun <chenjiankun1@huawei.com>2017-06-30 05:06:26 +0000
commit1b9eeb504ed9416c74e1caf8975b51ee6af86b29 (patch)
treeebffc173b165e3d29a0f95aa2b687b163c67288a
parent9ba594fa3f34d551d34abd7582281ce081924c67 (diff)
Change prepareYardstickEnv to prepare_env and add log info
JIRA: YARDSTICK-697 Currently the action in env like prepareYardstickEnv, createGrafanaContainer, createInfluxDBContainer, this is not python style. So I do some change: prepareYardstickEnv->prepare_env createGrafanaContainer->create_grafana createInfluxDBContainer->create_influxdb And also add some log info in env_action. Change-Id: Ia91576b975f3de76b96e312779fda4911e7cac24 Signed-off-by: chenjiankun <chenjiankun1@huawei.com>
-rw-r--r--api/base.py5
-rw-r--r--api/database/v1/handlers.py28
-rw-r--r--api/resources/asynctask.py51
-rw-r--r--api/resources/env_action.py128
-rw-r--r--api/utils/common.py13
-rw-r--r--tests/unit/apiserver/resources/test_env_action.py2
-rw-r--r--tests/unit/apiserver/utils/test_common.py25
-rw-r--r--tests/unit/cmd/commands/test_env.py2
-rw-r--r--yardstick/cmd/commands/env.py6
9 files changed, 140 insertions, 120 deletions
diff --git a/api/base.py b/api/base.py
index 6fa2777ce..bd967bbae 100644
--- a/api/base.py
+++ b/api/base.py
@@ -15,6 +15,7 @@ from flask import request
from flask_restful import Resource
from api.utils import common as common_utils
+from yardstick.common import constants as consts
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@@ -59,5 +60,5 @@ class ApiResource(Resource):
resources = importlib.import_module(module_name)
try:
return getattr(resources, action)(args)
- except NameError:
- common_utils.error_handler('Wrong action')
+ except AttributeError:
+ common_utils.result_handler(consts.API_ERROR, 'No such action')
diff --git a/api/database/v1/handlers.py b/api/database/v1/handlers.py
index f7c448cda..938015d82 100644
--- a/api/database/v1/handlers.py
+++ b/api/database/v1/handlers.py
@@ -19,14 +19,6 @@ class TasksHandler(object):
db_session.commit()
return task
- def update_status(self, task, status):
- task.status = status
- db_session.commit()
-
- def update_error(self, task, error):
- task.error = error
- db_session.commit()
-
def get_task_by_taskid(self, task_id):
task = Tasks.query.filter_by(task_id=task_id).first()
if not task:
@@ -35,7 +27,7 @@ class TasksHandler(object):
return task
def update_attr(self, task_id, attr):
- task = self.get_task_by_taskid(task_id)
+ task = self.get_task_by_taskid(task_id)
for k, v in attr.items():
setattr(task, k, v)
@@ -49,14 +41,16 @@ class AsyncTaskHandler(object):
db_session.commit()
return task
- def update_status(self, task, status):
- task.status = status
- db_session.commit()
-
- def update_error(self, task, error):
- task.error = error
- db_session.commit()
-
def get_task_by_taskid(self, task_id):
task = AsyncTasks.query.filter_by(task_id=task_id).first()
+ if not task:
+ raise ValueError
+
return task
+
+ def update_attr(self, task_id, attr):
+ task = self.get_task_by_taskid(task_id)
+
+ for k, v in attr.items():
+ setattr(task, k, v)
+ db_session.commit()
diff --git a/api/resources/asynctask.py b/api/resources/asynctask.py
index 1f70501d2..39b47c0ee 100644
--- a/api/resources/asynctask.py
+++ b/api/resources/asynctask.py
@@ -7,9 +7,14 @@
# http://www.apache.org/licenses/LICENSE-2.0
# ############################################################################
import uuid
+import logging
-from api.utils import common as common_utils
-from api.database.v1.models import AsyncTasks
+from api.utils.common import result_handler
+from api.database.v1.handlers import AsyncTaskHandler
+from yardstick.common import constants as consts
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.DEBUG)
def default(args):
@@ -19,17 +24,41 @@ def default(args):
def _get_status(args):
try:
task_id = args['task_id']
- uuid.UUID(task_id)
except KeyError:
- message = 'measurement and task_id must be provided'
- return common_utils.error_handler(message)
+ return result_handler(consts.API_ERROR, 'task_id must be provided')
- asynctask = AsyncTasks.query.filter_by(task_id=task_id).first()
+ try:
+ uuid.UUID(task_id)
+ except ValueError:
+ return result_handler(consts.API_ERROR, 'invalid task_id')
+ asynctask_handler = AsyncTaskHandler()
try:
- status = asynctask.status
- error = asynctask.error if asynctask.error else []
+ asynctask = asynctask_handler.get_task_by_taskid(task_id)
+ except ValueError:
+ return result_handler(consts.API_ERROR, 'invalid task_id')
+
+ def _unfinished():
+ return result_handler(consts.TASK_NOT_DONE, {})
+
+ def _finished():
+ return result_handler(consts.TASK_DONE, {})
+
+ def _error():
+ return result_handler(consts.TASK_FAILED, asynctask.error)
+
+ status = asynctask.status
+ LOG.debug('Task status is: %s', status)
+
+ if status not in [consts.TASK_NOT_DONE,
+ consts.TASK_DONE,
+ consts.TASK_FAILED]:
+ return result_handler(consts.API_ERROR, 'internal server error')
+
+ switcher = {
+ consts.TASK_NOT_DONE: _unfinished,
+ consts.TASK_DONE: _finished,
+ consts.TASK_FAILED: _error
+ }
- return common_utils.result_handler(status, error)
- except AttributeError:
- return common_utils.error_handler('no such task')
+ return switcher.get(status)()
diff --git a/api/resources/env_action.py b/api/resources/env_action.py
index 4eda73368..2ea64ef1a 100644
--- a/api/resources/env_action.py
+++ b/api/resources/env_action.py
@@ -32,17 +32,19 @@ from yardstick.common import openstack_utils
from yardstick.common.httpClient import HttpClient
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.DEBUG)
+async_handler = AsyncTaskHandler()
-def createGrafanaContainer(args):
+
+def create_grafana(args):
task_id = str(uuid.uuid4())
thread = threading.Thread(target=_create_grafana, args=(task_id,))
thread.start()
- return result_handler('success', {'task_id': task_id})
+ return result_handler(consts.API_SUCCESS, {'task_id': task_id})
def _create_grafana(task_id):
@@ -51,22 +53,29 @@ def _create_grafana(task_id):
client = Client(base_url=consts.DOCKER_URL)
try:
+ LOG.info('Checking if grafana image exist')
image = '{}:{}'.format(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
if not _check_image_exist(client, image):
+ LOG.info('Grafana image not exist, start pulling')
client.pull(consts.GRAFANA_IMAGE, consts.GRAFANA_TAG)
+ LOG.info('Createing grafana container')
_create_grafana_container(client)
+ LOG.info('Grafana container is created')
time.sleep(5)
+ LOG.info('Creating data source for grafana')
_create_data_source()
+ LOG.info('Creating dashboard for grafana')
_create_dashboard()
_update_task_status(task_id)
+ LOG.info('Finished')
except Exception as e:
_update_task_error(task_id, str(e))
- logger.exception('Error: %s', e)
+ LOG.exception('Create grafana failed')
def _create_dashboard():
@@ -76,7 +85,11 @@ def _create_dashboard():
for i in sorted(glob.iglob(path)):
with open(i) as f:
data = jsonutils.load(f)
- HttpClient().post(url, data)
+ try:
+ HttpClient().post(url, data)
+ except Exception:
+ LOG.exception('Create dashboard %s failed', i)
+ raise
def _create_data_source():
@@ -94,7 +107,11 @@ def _create_data_source():
"basicAuthPassword": "admin",
"isDefault": False,
}
- HttpClient().post(url, data)
+ try:
+ HttpClient().post(url, data)
+ except Exception:
+ LOG.exception('Create datasources failed')
+ raise
def _create_grafana_container(client):
@@ -104,12 +121,14 @@ def _create_grafana_container(client):
host_config = client.create_host_config(port_bindings=port_bindings,
restart_policy=restart_policy)
+ LOG.info('Creating container')
container = client.create_container(image='%s:%s' % (consts.GRAFANA_IMAGE,
consts.GRAFANA_TAG),
ports=ports,
detach=True,
tty=True,
host_config=host_config)
+ LOG.info('Starting container')
client.start(container)
@@ -117,13 +136,13 @@ def _check_image_exist(client, t):
return any(t in a['RepoTags'][0] for a in client.images() if a['RepoTags'])
-def createInfluxDBContainer(args):
+def create_influxdb(args):
task_id = str(uuid.uuid4())
thread = threading.Thread(target=_create_influxdb, args=(task_id,))
thread.start()
- return result_handler('success', {'task_id': task_id})
+ return result_handler(consts.API_SUCCESS, {'task_id': task_id})
def _create_influxdb(task_id):
@@ -132,22 +151,30 @@ def _create_influxdb(task_id):
client = Client(base_url=consts.DOCKER_URL)
try:
+ LOG.info('Changing output to influxdb')
_change_output_to_influxdb()
+ LOG.info('Checking if influxdb image exist')
if not _check_image_exist(client, '%s:%s' % (consts.INFLUXDB_IMAGE,
consts.INFLUXDB_TAG)):
+ LOG.info('Influxdb image not exist, start pulling')
client.pull(consts.INFLUXDB_IMAGE, tag=consts.INFLUXDB_TAG)
+ LOG.info('Createing influxdb container')
_create_influxdb_container(client)
+ LOG.info('Influxdb container is created')
time.sleep(5)
+ LOG.info('Config influxdb')
_config_influxdb()
_update_task_status(task_id)
+
+ LOG.info('Finished')
except Exception as e:
_update_task_error(task_id, str(e))
- logger.debug('Error: %s', e)
+ LOG.exception('Creating influxdb failed')
def _create_influxdb_container(client):
@@ -158,12 +185,14 @@ def _create_influxdb_container(client):
host_config = client.create_host_config(port_bindings=port_bindings,
restart_policy=restart_policy)
+ LOG.info('Creating container')
container = client.create_container(image='%s:%s' % (consts.INFLUXDB_IMAGE,
consts.INFLUXDB_TAG),
ports=ports,
detach=True,
tty=True,
host_config=host_config)
+ LOG.info('Starting container')
client.start(container)
@@ -174,32 +203,35 @@ def _config_influxdb():
consts.INFLUXDB_PASS,
consts.INFLUXDB_DB_NAME)
client.create_database(consts.INFLUXDB_DB_NAME)
- logger.info('Success to config influxDB')
- except Exception as e:
- logger.debug('Failed to config influxDB: %s', e)
+ LOG.info('Success to config influxDB')
+ except Exception:
+ LOG.exception('Config influxdb failed')
def _change_output_to_influxdb():
common_utils.makedirs(consts.CONF_DIR)
parser = configparser.ConfigParser()
+ LOG.info('Reading output sample configuration')
parser.read(consts.CONF_SAMPLE_FILE)
+ LOG.info('Set dispatcher to influxdb')
parser.set('DEFAULT', 'dispatcher', 'influxdb')
parser.set('dispatcher_influxdb', 'target',
'http://%s:8086' % consts.INFLUXDB_IP)
+ LOG.info('Writing to %s', consts.CONF_FILE)
with open(consts.CONF_FILE, 'w') as f:
parser.write(f)
-def prepareYardstickEnv(args):
+def prepare_env(args):
task_id = str(uuid.uuid4())
thread = threading.Thread(target=_prepare_env_daemon, args=(task_id,))
thread.start()
- return result_handler('success', {'task_id': task_id})
+ return result_handler(consts.API_SUCCESS, {'task_id': task_id})
def _already_source_openrc():
@@ -216,23 +248,33 @@ def _prepare_env_daemon(task_id):
rc_file = consts.OPENRC
+ LOG.info('Checkout Openrc Environment variable')
if not _already_source_openrc():
+ LOG.info('Openrc variable not found in Environment')
if not os.path.exists(rc_file):
+ LOG.info('Openrc file not found')
installer_ip = os.environ.get('INSTALLER_IP', '192.168.200.2')
installer_type = os.environ.get('INSTALLER_TYPE', 'compass')
+ LOG.info('Getting openrc file from %s', installer_type)
_get_remote_rc_file(rc_file, installer_ip, installer_type)
+ LOG.info('Source openrc file')
_source_file(rc_file)
+ LOG.info('Appending external network')
_append_external_network(rc_file)
+ LOG.info('Openrc file exist, source openrc file')
_source_file(rc_file)
+ LOG.info('Cleaning images')
_clean_images()
+ LOG.info('Loading images')
_load_images()
_update_task_status(task_id)
+ LOG.info('Finished')
except Exception as e:
_update_task_error(task_id, str(e))
- logger.debug('Error: %s', e)
+ LOG.exception('Prepare env failed')
def _create_directories():
@@ -254,7 +296,7 @@ def _get_remote_rc_file(rc_file, installer_ip, installer_type):
p.communicate()
if p.returncode != 0:
- logger.debug('Failed to fetch credentials from installer')
+ LOG.error('Failed to fetch credentials from installer')
except OSError as e:
if e.errno != errno.EEXIST:
raise
@@ -266,7 +308,7 @@ def _append_external_network(rc_file):
try:
ext_network = next(n['name'] for n in networks if n['router:external'])
except StopIteration:
- logger.warning("Can't find external network")
+ LOG.warning("Can't find external network")
else:
cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
try:
@@ -281,38 +323,26 @@ def _clean_images():
cmd = [consts.CLEAN_IMAGES_SCRIPT]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
output = p.communicate()[0]
- logger.debug('The result is: %s', output)
+ LOG.debug(output)
def _load_images():
cmd = [consts.LOAD_IMAGES_SCRIPT]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=consts.REPOS_DIR)
output = p.communicate()[0]
- logger.debug('The result is: %s', output)
+ LOG.debug(output)
def _create_task(task_id):
- async_handler = AsyncTaskHandler()
- task_dict = {
- 'task_id': task_id,
- 'status': 0
- }
- async_handler.insert(task_dict)
+ async_handler.insert({'status': 0, 'task_id': task_id})
def _update_task_status(task_id):
- async_handler = AsyncTaskHandler()
-
- task = async_handler.get_task_by_taskid(task_id)
- async_handler.update_status(task, 1)
+ async_handler.update_attr(task_id, {'status': 1})
def _update_task_error(task_id, error):
- async_handler = AsyncTaskHandler()
-
- task = async_handler.get_task_by_taskid(task_id)
- async_handler.update_status(task, 2)
- async_handler.update_error(task, error)
+ async_handler.update_attr(task_id, {'status': 2, 'error': error})
def update_openrc(args):
@@ -325,22 +355,22 @@ def update_openrc(args):
return result_handler(consts.API_ERROR, 'args should be a dict')
lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()]
- logger.debug('Writing: %s', ''.join(lines))
+ LOG.debug('Writing: %s', ''.join(lines))
- logger.info('Writing openrc: Writing')
+ LOG.info('Writing openrc: Writing')
common_utils.makedirs(consts.CONF_DIR)
with open(consts.OPENRC, 'w') as f:
f.writelines(lines)
- logger.info('Writing openrc: Done')
+ LOG.info('Writing openrc: Done')
- logger.info('Source openrc: Sourcing')
+ LOG.info('Source openrc: Sourcing')
try:
_source_file(consts.OPENRC)
except Exception as e:
- logger.exception('Failed to source openrc')
+ LOG.exception('Failed to source openrc')
return result_handler(consts.API_ERROR, str(e))
- logger.info('Source openrc: Done')
+ LOG.info('Source openrc: Done')
return result_handler(consts.API_SUCCESS, {'openrc': openrc_vars})
@@ -351,15 +381,15 @@ def upload_pod_file(args):
except KeyError:
return result_handler(consts.API_ERROR, 'file must be provided')
- logger.info('Checking file')
+ LOG.info('Checking file')
data = yaml.load(pod_file.read())
if not isinstance(data, collections.Mapping):
return result_handler(consts.API_ERROR, 'invalid yaml file')
- logger.info('Writing file')
+ LOG.info('Writing file')
with open(consts.POD_FILE, 'w') as f:
yaml.dump(data, f, default_flow_style=False)
- logger.info('Writing finished')
+ LOG.info('Writing finished')
return result_handler(consts.API_SUCCESS, {'pod_info': data})
@@ -373,10 +403,10 @@ def update_pod_file(args):
if not isinstance(pod_dic, collections.Mapping):
return result_handler(consts.API_ERROR, 'pod should be a dict')
- logger.info('Writing file')
+ LOG.info('Writing file')
with open(consts.POD_FILE, 'w') as f:
yaml.dump(pod_dic, f, default_flow_style=False)
- logger.info('Writing finished')
+ LOG.info('Writing finished')
return result_handler(consts.API_SUCCESS, {'pod_info': pod_dic})
@@ -384,11 +414,11 @@ def update_pod_file(args):
def update_hosts(hosts_ip):
if not isinstance(hosts_ip, dict):
return result_handler(consts.API_ERROR, 'Error, args should be a dict')
- logger.info('Writing hosts: Writing')
+ LOG.info('Writing hosts: Writing')
hosts_list = ['\n{} {}'.format(ip, host_name)
for host_name, ip in hosts_ip.items()]
- logger.debug('Writing: %s', hosts_list)
+ LOG.debug('Writing: %s', hosts_list)
with open(consts.ETC_HOSTS, 'a') as f:
f.writelines(hosts_list)
- logger.info('Writing hosts: Done')
+ LOG.info('Writing hosts: Done')
return result_handler(consts.API_SUCCESS, 'success')
diff --git a/api/utils/common.py b/api/utils/common.py
index 8398b8f60..eda9c17dd 100644
--- a/api/utils/common.py
+++ b/api/utils/common.py
@@ -13,8 +13,8 @@ import logging
from flask import jsonify
import six
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.DEBUG)
def translate_to_str(obj):
@@ -27,15 +27,6 @@ def translate_to_str(obj):
return obj
-def error_handler(message):
- logger.debug(message)
- result = {
- 'status': 'error',
- 'message': message
- }
- return jsonify(result)
-
-
def result_handler(status, data):
result = {
'status': status,
diff --git a/tests/unit/apiserver/resources/test_env_action.py b/tests/unit/apiserver/resources/test_env_action.py
index c7ae10a20..d61092dbc 100644
--- a/tests/unit/apiserver/resources/test_env_action.py
+++ b/tests/unit/apiserver/resources/test_env_action.py
@@ -18,7 +18,7 @@ class EnvTestCase(APITestCase):
def test_create_grafana(self):
url = 'yardstick/env/action'
- data = {'action': 'createGrafanaContainer'}
+ data = {'action': 'create_grafana'}
resp = self._post(url, data)
time.sleep(1)
diff --git a/tests/unit/apiserver/utils/test_common.py b/tests/unit/apiserver/utils/test_common.py
index acf6e41b1..ad81cb76b 100644
--- a/tests/unit/apiserver/utils/test_common.py
+++ b/tests/unit/apiserver/utils/test_common.py
@@ -33,31 +33,6 @@ class TranslateToStrTestCase(unittest.TestCase):
self.assertEqual(result, output_str)
-class GetCommandListTestCase(unittest.TestCase):
-
- def test_get_command_list_no_opts(self):
- command_list = ['a']
- opts = {}
- args = 'b'
- output_list = common.get_command_list(command_list, opts, args)
-
- result_list = ['a', 'b']
- self.assertEqual(result_list, output_list)
-
- def test_get_command_list_with_opts_args(self):
- command_list = ['a']
- opts = {
- 'b': 'c',
- 'task-args': 'd'
- }
- args = 'e'
-
- output_list = common.get_command_list(command_list, opts, args)
-
- result_list = ['a', 'e', '--b', '--task-args', 'd']
- self.assertEqual(result_list, output_list)
-
-
def main():
unittest.main()
diff --git a/tests/unit/cmd/commands/test_env.py b/tests/unit/cmd/commands/test_env.py
index c6e0e1d20..73cd5af47 100644
--- a/tests/unit/cmd/commands/test_env.py
+++ b/tests/unit/cmd/commands/test_env.py
@@ -42,7 +42,7 @@ class EnvCommandTestCase(unittest.TestCase):
@mock.patch('yardstick.cmd.commands.env.HttpClient.post')
def test_start_async_task(self, post_mock):
- data = {'action': 'createGrafanaContainer'}
+ data = {'action': 'create_grafana'}
EnvCommand()._start_async_task(data)
self.assertTrue(post_mock.called)
diff --git a/yardstick/cmd/commands/env.py b/yardstick/cmd/commands/env.py
index d5aef7faf..dbef303b0 100644
--- a/yardstick/cmd/commands/env.py
+++ b/yardstick/cmd/commands/env.py
@@ -26,21 +26,21 @@ class EnvCommand(object):
"""
def do_influxdb(self, args):
- data = {'action': 'createInfluxDBContainer'}
+ data = {'action': 'create_influxdb'}
task_id = self._start_async_task(data)
start = '* creating influxDB'
self._check_status(task_id, start)
def do_grafana(self, args):
- data = {'action': 'createGrafanaContainer'}
+ data = {'action': 'create_grafana'}
task_id = self._start_async_task(data)
start = '* creating grafana'
self._check_status(task_id, start)
def do_prepare(self, args):
- data = {'action': 'prepareYardstickEnv'}
+ data = {'action': 'prepare_env'}
task_id = self._start_async_task(data)
start = '* preparing yardstick environment'