Yardstick virtualenv support

JIRA: YARDSTICK-620

Currently we recommend using docker to run yardstick.
And it is hard to use virtualenv to install yardstick.
So I modify install.sh in yardstick root path.
It will support using virtualenv to install yardstick(including API) in
linux.
In this patch, I make yardstick support read yardstick configuration
have priority over constants.

Change-Id: I9ea1241b228532a6497451e6c8f232173ddb783e
Signed-off-by: chenjiankun <chenjiankun1@huawei.com>
(cherry picked from commit b74d875134b988a26441d559a9e700aaa68d6a0c)

8 files changed, 88 insertions, 82 deletions

diff --git a/yardstick/__init__.py b/yardstick/__init__.py
index e19be36fd..2f5ae9feb 100644
--- a/yardstick/__init__.py
+++ b/yardstick/__init__.py
@@ -20,8 +20,8 @@ from yardstick.common import utils as yardstick_utils
 # without having to install apexlake.
 sys.path.append(os.path.dirname(apexlake.__file__))
 
-yardstick_utils.makedirs(constants.YARDSTICK_LOG_DIR)
-LOG_FILE = os.path.join(constants.YARDSTICK_LOG_DIR, 'yardstick.log')
+yardstick_utils.makedirs(constants.LOG_DIR)
+LOG_FILE = os.path.join(constants.LOG_DIR, 'yardstick.log')
 LOG_FORMATTER = ('%(asctime)s '
                  '%(name)s %(filename)s:%(lineno)d '
                  '%(levelname)s %(message)s')
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py
index 571a769eb..604df80d1 100644
--- a/yardstick/benchmark/contexts/heat.py
+++ b/yardstick/benchmark/contexts/heat.py
@@ -25,7 +25,7 @@ from yardstick.benchmark.contexts.model import PlacementGroup, ServerGroup
 from yardstick.benchmark.contexts.model import Server
 from yardstick.benchmark.contexts.model import update_scheduler_hints
 from yardstick.orchestrator.heat import HeatTemplate, get_short_key_uuid
-from yardstick.definitions import YARDSTICK_ROOT_PATH
+from yardstick.common.constants import YARDSTICK_ROOT_PATH
 
 LOG = logging.getLogger(__name__)
 
diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py
index 8bf915609..a4a2cfc57 100644
--- a/yardstick/benchmark/contexts/node.py
+++ b/yardstick/benchmark/contexts/node.py
@@ -19,7 +19,7 @@ import pkg_resources
 
 from yardstick import ssh
 from yardstick.benchmark.contexts.base import Context
-from yardstick.common.constants import YARDSTICK_ROOT_PATH
+from yardstick.common import constants as consts
 
 LOG = logging.getLogger(__name__)
 
@@ -57,7 +57,7 @@ class NodeContext(Context):
         except IOError as ioerror:
             if ioerror.errno == errno.ENOENT:
                 self.file_path = \
-                    os.path.join(YARDSTICK_ROOT_PATH, self.file_path)
+                    os.path.join(consts.YARDSTICK_ROOT_PATH, self.file_path)
                 cfg = self.read_config_file()
             else:
                 raise
@@ -108,8 +108,7 @@ class NodeContext(Context):
 
     def _do_ansible_job(self, path):
         cmd = 'ansible-playbook -i inventory.ini %s' % path
-        base = '/home/opnfv/repos/yardstick/ansible'
-        p = subprocess.Popen(cmd, shell=True, cwd=base)
+        p = subprocess.Popen(cmd, shell=True, cwd=consts.ANSIBLE_DIR)
         p.communicate()
 
     def _get_server(self, attr_name):
@@ -164,7 +163,7 @@ class NodeContext(Context):
 
     def _execute_local_script(self, info):
         script, options = self._get_script(info)
-        script = os.path.join(YARDSTICK_ROOT_PATH, script)
+        script = os.path.join(consts.YARDSTICK_ROOT_PATH, script)
         cmd = ['bash', script, options]
 
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py
index 674e57f54..78eaac7ee 100644
--- a/yardstick/benchmark/contexts/standalone.py
+++ b/yardstick/benchmark/contexts/standalone.py
@@ -20,7 +20,7 @@ import collections
 import yaml
 
 from yardstick.benchmark.contexts.base import Context
-from yardstick.definitions import YARDSTICK_ROOT_PATH
+from yardstick.common.constants import YARDSTICK_ROOT_PATH
 
 LOG = logging.getLogger(__name__)
 
diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index 40122764c..3a151dbba 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -503,7 +503,7 @@ def check_environment():
     auth_url = os.environ.get('OS_AUTH_URL', None)
     if not auth_url:
         try:
-            source_env(constants.OPENSTACK_RC_FILE)
+            source_env(constants.OPENRC)
         except IOError as e:
             if e.errno != errno.EEXIST:
                 raise
diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py
index 6550cc8fd..33266e233 100644
--- a/yardstick/common/constants.py
+++ b/yardstick/common/constants.py
@@ -9,61 +9,78 @@
 from __future__ import absolute_import
 import os
 
-DOCKER_URL = 'unix://var/run/docker.sock'
-
-# database config
-USER = 'root'
-PASSWORD = 'root'
-DATABASE = 'yardstick'
-
-INFLUXDB_IMAGE = 'tutum/influxdb'
-INFLUXDB_TAG = '0.13'
+from yardstick.common.utils import get_param
 
-GRAFANA_IMAGE = 'grafana/grafana'
-GRAFANA_TAGS = '3.1.1'
 
 dirname = os.path.dirname
 abspath = os.path.abspath
 join = os.path.join
 sep = os.path.sep
 
-INSTALLERS = ['apex', 'compass', 'fuel', 'joid']
-
+try:
+    SERVER_IP = get_param('api.server_ip')
+except KeyError:
+    try:
+        from pyroute2 import IPDB
+    except ImportError:
+        SERVER_IP = '172.17.0.1'
+    else:
+        with IPDB() as ip:
+            SERVER_IP = ip.routes['default'].gateway
+
+# dir
+CONF_DIR = get_param('dir.conf', '/etc/yardstick')
+REPOS_DIR = get_param('dir.repos', '/home/opnfv/repos/yardstick')
+RELENG_DIR = get_param('dir.releng', '/home/opnfv/repos/releng')
+LOG_DIR = get_param('dir.log', '/tmp/yardstick/')
 YARDSTICK_ROOT_PATH = dirname(dirname(dirname(abspath(__file__)))) + sep
-
+CONF_SAMPLE_DIR = join(REPOS_DIR, 'etc/yardstick/')
+ANSIBLE_DIR = join(REPOS_DIR, 'ansible')
+SAMPLE_CASE_DIR = join(REPOS_DIR, 'samples')
 TESTCASE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_cases/')
-
 TESTSUITE_DIR = join(YARDSTICK_ROOT_PATH, 'tests/opnfv/test_suites/')
 
-YARDSTICK_REPOS_DIR = '/home/opnfv/repos/yardstick'
-
-YARDSTICK_LOG_DIR = '/tmp/yardstick/'
-
-YARDSTICK_CONFIG_DIR = '/etc/yardstick/'
-
-YARDSTICK_CONFIG_FILE = join(YARDSTICK_CONFIG_DIR, 'yardstick.conf')
-
-YARDSTICK_CONFIG_SAMPLE_DIR = join(YARDSTICK_ROOT_PATH, 'etc/yardstick/')
-
-YARDSTICK_CONFIG_SAMPLE_FILE = join(YARDSTICK_CONFIG_SAMPLE_DIR,
-                                    'yardstick.conf.sample')
-
-RELENG_DIR = '/home/opnfv/repos/releng'
-
-OS_FETCH_SCRIPT = 'utils/fetch_os_creds.sh'
-
-CLEAN_IMAGES_SCRIPT = 'tests/ci/clean_images.sh'
-
-LOAD_IMAGES_SCRIPT = 'tests/ci/load_images.sh'
-
-OPENSTACK_RC_FILE = join(YARDSTICK_CONFIG_DIR, 'openstack.creds')
+# file
+OPENRC = get_param('file.openrc', '/etc/yardstick/yardstick.conf')
+CONF_FILE = join(CONF_DIR, 'yardstick.conf')
+CONF_SAMPLE_FILE = join(CONF_SAMPLE_DIR, 'yardstick.conf.sample')
+FETCH_SCRIPT = get_param('file.fetch_script', 'utils/fetch_os_creds.sh')
+FETCH_SCRIPT = join(RELENG_DIR, FETCH_SCRIPT)
+CLEAN_IMAGES_SCRIPT = get_param('file.clean_image_script',
+                                'tests/ci/clean_images.sh')
+CLEAN_IMAGES_SCRIPT = join(REPOS_DIR, CLEAN_IMAGES_SCRIPT)
+LOAD_IMAGES_SCRIPT = get_param('file.load_image_script',
+                               'tests/ci/load_images.sh')
+LOAD_IMAGES_SCRIPT = join(REPOS_DIR, LOAD_IMAGES_SCRIPT)
+DEFAULT_OUTPUT_FILE = get_param('file.output_file', '/tmp/yardstick.out')
+DEFAULT_HTML_FILE = get_param('file.html_file', '/tmp/yardstick.htm')
+
+# influxDB
+INFLUXDB_IP = get_param('influxdb.ip', SERVER_IP)
+INFLUXDB_PORT = get_param('influxdb.port', 8086)
+INFLUXDB_USER = get_param('influxdb.username', 'root')
+INFLUXDB_PASS = get_param('influxdb.password', 'root')
+INFLUXDB_DB_NAME = get_param('influxdb.db_name', 'yardstick')
+INFLUXDB_IMAGE = get_param('influxdb.image', 'tutum/influxdb')
+INFLUXDB_TAG = get_param('influxdb.tag', '0.13')
+
+# grafana
+GRAFANA_IP = get_param('grafana.ip', SERVER_IP)
+GRAFANA_PORT = get_param('grafana.port', 3000)
+GRAFANA_USER = get_param('grafana.username', 'admin')
+GRAFANA_PASS = get_param('grafana.password', 'admin')
+GRAFANA_IMAGE = get_param('grafana.image', 'grafana/grafana')
+GRAFANA_TAG = get_param('grafana.tag', '3.1.1')
+
+# api
+DOCKER_URL = 'unix://var/run/docker.sock'
+INSTALLERS = ['apex', 'compass', 'fuel', 'joid']
+SQLITE = 'sqlite:////tmp/yardstick.db'
 
 BASE_URL = 'http://localhost:5000'
 ENV_ACTION_API = BASE_URL + '/yardstick/env/action'
 ASYNC_TASK_API = BASE_URL + '/yardstick/asynctask'
 
-SQLITE = 'sqlite:////tmp/yardstick.db'
-
-DEFAULT_OUTPUT_FILE = '/tmp/yardstick.out'
-
-DEFAULT_HTML_FILE = '/tmp/yardstick.htm'
+# general
+TESTCASE_PRE = 'opnfv_yardstick_'
+TESTSUITE_PRE = 'opnfv_'
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index d2be8000e..7035f3374 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -84,26 +84,30 @@ def import_modules_from_package(package):
             try_append_module(module_name, sys.modules)
 
 
-def get_para_from_yaml(file_path, args):
-
-    def func(a, b):
-        if a is None:
-            return None
-        return a.get(b)
-
-    if os.path.exists(file_path):
+def parse_yaml(file_path):
+    try:
         with open(file_path) as f:
             value = yaml.safe_load(f)
-            value = reduce(func, args.split('.'), value)
+    except IOError:
+        return {}
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+    else:
+        return value
 
-            if value is None:
-                print('parameter not found')
-                return None
 
-            return value
-    else:
-        print('file not exist')
-        return None
+def get_param(key, default=''):
+
+    conf_file = os.environ.get('CONF_FILE', '/etc/yardstick/yardstick.yaml')
+
+    conf = parse_yaml(conf_file)
+    try:
+        return reduce(lambda a, b: a[b], key.split('.'), conf)
+    except KeyError:
+        if not default:
+            raise
+        return default
 
 
 def makedirs(d):
diff --git a/yardstick/definitions.py b/yardstick/definitions.py
deleted file mode 100644
index 64a4a80d4..000000000
--- a/yardstick/definitions.py
+++ /dev/null
@@ -1,14 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-from __future__ import absolute_import
-import os
-
-dirname = os.path.dirname
-YARDSTICK_ROOT_PATH = dirname(dirname(os.path.abspath(__file__)))
-YARDSTICK_ROOT_PATH += os.path.sep