Add API and command support for yardstick env prepare

JIRA: YARDSTICK-406

Change-Id: Icf837a6f34a22158203566a43a6446fc269c096f
Signed-off-by: chenjiankun <chenjiankun1@huawei.com>

7 files changed, 201 insertions, 8 deletions

diff --git a/api/actions/env.py b/api/actions/env.py
index 40d81c648..5ecb851ef 100644
--- a/api/actions/env.py
+++ b/api/actions/env.py
@@ -8,8 +8,11 @@
 ##############################################################################
 import logging
 import threading
+import subprocess
 import time
 import json
+import os
+import errno
 
 from docker import Client
 
@@ -18,6 +21,7 @@ from yardstick.common import utils as yardstick_utils
 from yardstick.common.httpClient import HttpClient
 from api import conf as api_conf
 from api.utils import influx
+from api.utils.common import result_handler
 
 logger = logging.getLogger(__name__)
 
@@ -25,6 +29,7 @@ logger = logging.getLogger(__name__)
 def createGrafanaContainer(args):
     thread = threading.Thread(target=_create_grafana)
     thread.start()
+    return result_handler('success', [])
 
 
 def _create_grafana():
@@ -91,6 +96,7 @@ def _check_image_exist(client, t):
 def createInfluxDBContainer(args):
     thread = threading.Thread(target=_create_influxdb)
     thread.start()
+    return result_handler('success', [])
 
 
 def _create_influxdb():
@@ -138,8 +144,8 @@ def _config_influxdb():
 
 
 def _config_output_file():
-    yardstick_utils.makedirs('/etc/yardstick')
-    with open('/etc/yardstick/yardstick.conf', 'w') as f:
+    yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
+    with open(config.YARDSTICK_CONFIG_FILE, 'w') as f:
         f.write("""\
 [DEFAULT]
 debug = True
@@ -160,3 +166,90 @@ username = root
 password = root
 """
                 % api_conf.GATEWAY_IP)
+
+
+def prepareYardstickEnv(args):
+    thread = threading.Thread(target=_prepare_env_daemon)
+    thread.start()
+    return result_handler('success', [])
+
+
+def _prepare_env_daemon():
+
+    installer_ip = os.environ.get('INSTALLER_IP', 'undefined')
+    installer_type = os.environ.get('INSTALLER_TYPE', 'undefined')
+
+    _check_variables(installer_ip, installer_type)
+
+    _create_directories()
+
+    rc_file = config.OPENSTACK_RC_FILE
+
+    _get_remote_rc_file(rc_file, installer_ip, installer_type)
+
+    _source_file(rc_file)
+
+    _append_external_network(rc_file)
+
+    _load_images()
+
+
+def _check_variables(installer_ip, installer_type):
+
+    if installer_ip == 'undefined':
+        raise SystemExit('Missing INSTALLER_IP')
+
+    if installer_type == 'undefined':
+        raise SystemExit('Missing INSTALLER_TYPE')
+    elif installer_type not in config.INSTALLERS:
+        raise SystemExit('INSTALLER_TYPE is not correct')
+
+
+def _create_directories():
+    yardstick_utils.makedirs(config.YARDSTICK_CONFIG_DIR)
+
+
+def _source_file(rc_file):
+    yardstick_utils.source_env(rc_file)
+
+
+def _get_remote_rc_file(rc_file, installer_ip, installer_type):
+
+    os_fetch_script = os.path.join(config.RELENG_DIR, config.OS_FETCH_SCRIPT)
+
+    try:
+        cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
+               '-a', installer_ip]
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+        p.communicate()[0]
+
+        if p.returncode != 0:
+            logger.debug('Failed to fetch credentials from installer')
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+
+def _append_external_network(rc_file):
+    neutron_client = yardstick_utils.get_neutron_client()
+    networks = neutron_client.list_networks()['networks']
+    try:
+        ext_network = next(n['name'] for n in networks if n['router:external'])
+    except StopIteration:
+        logger.warning("Can't find external network")
+    else:
+        cmd = 'export EXTERNAL_NETWORK=%s' % ext_network
+        try:
+            with open(rc_file, 'a') as f:
+                f.write(cmd + '\n')
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+
+def _load_images():
+    cmd = [config.LOAD_IMAGES_SCRIPT]
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                         cwd=config.YARDSTICK_REPOS_DIR)
+    output = p.communicate()[0]
+    logger.debug('The result is: %s', output)
diff --git a/setup.py b/setup.py
index 6a89d6368..54595b648 100755
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,8 @@ setup(
             'yardstick/nodes/*/*.yaml'
         ],
         'tests': [
-            'opnfv/*/*.yaml'
+            'opnfv/*/*.yaml',
+            'ci/*.sh'
         ]
     },
     url="https://www.opnfv.org",
diff --git a/yardstick/cmd/commands/env.py b/yardstick/cmd/commands/env.py
index ed46b84c4..098379ae1 100644
--- a/yardstick/cmd/commands/env.py
+++ b/yardstick/cmd/commands/env.py
@@ -6,17 +6,34 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+import logging
+
 from yardstick.common.httpClient import HttpClient
+from yardstick.common import constants
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
 
 
 class EnvCommand(object):
+    '''
 
+        Set of commands to prepare environment
+    '''
     def do_influxdb(self, args):
-        url = 'http://localhost:5000/yardstick/env/action'
+        url = constants.YARDSTICK_ENV_ACTION_API
         data = {'action': 'createInfluxDBContainer'}
         HttpClient().post(url, data)
+        logger.debug('Now creating and configing influxdb')
 
     def do_grafana(self, args):
-        url = 'http://localhost:5000/yardstick/env/action'
+        url = constants.YARDSTICK_ENV_ACTION_API
         data = {'action': 'createGrafanaContainer'}
         HttpClient().post(url, data)
+        logger.debug('Now creating and configing grafana')
+
+    def do_prepare(self, args):
+        url = constants.YARDSTICK_ENV_ACTION_API
+        data = {'action': 'prepareYardstickEnv'}
+        HttpClient().post(url, data)
+        logger.debug('Now preparing environment')
diff --git a/yardstick/cmd/commands/task.py b/yardstick/cmd/commands/task.py
index 47fb2ee60..9524778ba 100644
--- a/yardstick/cmd/commands/task.py
+++ b/yardstick/cmd/commands/task.py
@@ -17,12 +17,15 @@ import ipaddress
 import time
 import logging
 import uuid
+import errno
 from itertools import ifilter
 
 from yardstick.benchmark.contexts.base import Context
 from yardstick.benchmark.runners import base as base_runner
 from yardstick.common.task_template import TaskTemplate
 from yardstick.common.utils import cliargs
+from yardstick.common.utils import source_env
+from yardstick.common import constants
 
 output_file_default = "/tmp/yardstick.out"
 test_cases_dir_default = "tests/opnfv/test_cases/"
@@ -58,6 +61,8 @@ class TaskCommands(object):
 
         self.task_id = kwargs.get('task_id', str(uuid.uuid4()))
 
+        check_environment()
+
         total_start_time = time.time()
         parser = TaskParser(args.inputfile[0])
 
@@ -483,3 +488,14 @@ def parse_task_args(src_name, args):
               % {"src": src_name, "src_type": type(kw)})
         raise TypeError()
     return kw
+
+
+def check_environment():
+    auth_url = os.environ.get('OS_AUTH_URL', None)
+    if not auth_url:
+        try:
+            source_env(constants.OPENSTACK_RC_FILE)
+        except IOError as e:
+            if e.errno != errno.EEXIST:
+                raise
+            LOG.debug('OPENRC file not found')
diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py
index 07d869755..d541ead15 100644
--- a/yardstick/common/constants.py
+++ b/yardstick/common/constants.py
@@ -1,6 +1,4 @@
-CONFIG_SAMPLE = '/etc/yardstick/config.yaml'
-
-RELENG_DIR = 'releng.dir'
+import os
 
 DOCKER_URL = 'unix://var/run/docker.sock'
 
@@ -14,3 +12,27 @@ INFLUXDB_TAG = '0.13'
 
 GRAFANA_IMAGE = 'grafana/grafana'
 GRAFANA_TAGS = '3.1.1'
+
+dirname = os.path.dirname
+abspath = os.path.abspath
+sep = os.path.sep
+
+INSTALLERS = ['apex', 'compass', 'fuel', 'joid']
+
+YARDSTICK_ROOT_PATH = dirname(dirname(dirname(abspath(__file__)))) + sep
+
+YARDSTICK_REPOS_DIR = '/home/opnfv/repos/yardstick'
+
+YARDSTICK_CONFIG_DIR = '/etc/yardstick/'
+
+YARDSTICK_CONFIG_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'config.yaml')
+
+RELENG_DIR = '/home/opnfv/repos/releng'
+
+OS_FETCH_SCRIPT = 'utils/fetch_os_creds.sh'
+
+LOAD_IMAGES_SCRIPT = 'tests/ci/load_images.sh'
+
+OPENSTACK_RC_FILE = os.path.join(YARDSTICK_CONFIG_DIR, 'openstack.creds')
+
+YARDSTICK_ENV_ACTION_API = 'http://localhost:5000/yardstick/env/action'
diff --git a/yardstick/common/httpClient.py b/yardstick/common/httpClient.py
index b6959b400..ab2e9a379 100644
--- a/yardstick/common/httpClient.py
+++ b/yardstick/common/httpClient.py
@@ -23,5 +23,8 @@ class HttpClient(object):
             response = requests.post(url, data=data, headers=headers)
             result = response.json()
             logger.debug('The result is: %s', result)
+
+            return result
         except Exception as e:
             logger.debug('Failed: %s', e)
+            raise
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index afbe4e8ec..3ecb0ae20 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -19,10 +19,19 @@ import os
 import sys
 import yaml
 import errno
+import subprocess
+import logging
+
 from oslo_utils import importutils
+from keystoneauth1 import identity
+from keystoneauth1 import session
+from neutronclient.v2_0 import client
 
 import yardstick
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 
 # Decorator for cli-args
 def cliargs(*args, **kwargs):
@@ -100,3 +109,35 @@ def makedirs(d):
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
+
+
+def execute_command(cmd):
+    exec_msg = "Executing command: '%s'" % cmd
+    logger.debug(exec_msg)
+
+    output = subprocess.check_output(cmd.split()).split(os.linesep)
+
+    return output
+
+
+def source_env(env_file):
+    p = subprocess.Popen(". %s; env" % env_file, stdout=subprocess.PIPE,
+                         shell=True)
+    output = p.communicate()[0]
+    env = dict((line.split('=', 1) for line in output.splitlines()))
+    os.environ.update(env)
+    return env
+
+
+def get_openstack_session():
+    auth = identity.Password(auth_url=os.environ.get('OS_AUTH_URL'),
+                             username=os.environ.get('OS_USERNAME'),
+                             password=os.environ.get('OS_PASSWORD'),
+                             tenant_name=os.environ.get('OS_TENANT_NAME'))
+    return session.Session(auth=auth)
+
+
+def get_neutron_client():
+    sess = get_openstack_session()
+    neutron_client = client.Client(session=sess)
+    return neutron_client