Fix exception when running HA tests without pod.yaml

1. use volumes '-v' to map files/directories which may be non-existing
2. use mounts '--mount' to map files/directories which couldn't be non-existing

JIRA: DOVETAIL-789

Change-Id: I2184e5baed3d1491a2df4d3a1a77a11e3e9b4fc8
Signed-off-by: xudan <xudan16@huawei.com>

10 files changed, 108 insertions, 28 deletions

diff --git a/dovetail/container.py b/dovetail/container.py
index ec9b1fb2..b2a9428f 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -58,23 +58,28 @@ class Container(object):
         kwargs = dt_utils.get_value_from_dict('opts', project_cfg)
         shell = dt_utils.get_value_from_dict('shell', project_cfg)
         if not shell:
-            return None
+            return None, "Lacking of key word 'shell' in config file."
         env_list = dt_utils.get_value_from_dict('envs', project_cfg)
         if env_list:
             kwargs['environment'] = \
                 [env for env in env_list if env is not None]
         volume_list = dt_utils.get_value_from_dict('volumes', project_cfg)
         kwargs['volumes'] = [vol for vol in volume_list if vol is not None]
+
+        kwargs['mounts'], msg = dt_utils.get_mount_list(project_cfg)
+        if not kwargs['mounts']:
+            return None, msg
+
         kwargs['extra_hosts'] = dt_utils.get_hosts_info(self.logger)
 
         try:
             self.container = self.client.containers.run(
                 docker_image, shell, **kwargs)
         except (docker.errors.ContainerError, docker.errors.ImageNotFound,
-                docker.errors.APIError):
-            return None
+                docker.errors.APIError) as e:
+            return None, e
 
-        return self.container.id
+        return self.container.id, 'Successfully to create container.'
 
     def get_image_id(self, image_name):
         try:
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index 97367db9..266bdc20 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -77,9 +77,10 @@ class DockerRunner(Runner):
                 self.logger.error("Failed to pull the image.")
                 return
 
-        container_id = container.create(docker_image)
+        container_id, msg = container.create(docker_image)
         if not container_id:
             self.logger.error('Failed to create container.')
+            self.logger.error(msg)
             return
 
         self.logger.debug('container id: {}'.format(container_id))
diff --git a/dovetail/tests/unit/test_container.py b/dovetail/tests/unit/test_container.py
index 01c1d8fd..86da9d3c 100644
--- a/dovetail/tests/unit/test_container.py
+++ b/dovetail/tests/unit/test_container.py
@@ -409,6 +409,7 @@ class ContainerTesting(unittest.TestCase):
         container_id = 'container_id'
         mock_utils.get_value_from_dict.side_effect = [
             {'key': 'value'}, 'shell', 'envs', ['volume_one', 'volume_two']]
+        mock_utils.get_mount_list.side_effect = [['mount', 'list'], 'success']
         mock_utils.get_hosts_info.return_value = 'host_info'
         container_obj = Mock()
         container_obj.id = container_id
@@ -417,7 +418,7 @@ class ContainerTesting(unittest.TestCase):
         mock_config.dovetail_config = {'bottlenecks': project_config}
 
         expected = container_id
-        result = self.container.create(docker_image)
+        result, msg = self.container.create(docker_image)
 
         mock_utils.get_value_from_dict.assert_has_calls([
             call('opts', project_config),
@@ -426,6 +427,7 @@ class ContainerTesting(unittest.TestCase):
             call('volumes', project_config)])
         mock_utils.get_hosts_info.assert_called_once_with(self.logger)
         self.assertEqual(expected, result)
+        self.assertEqual('Successfully to create container.', msg)
 
     @patch('dovetail.container.dt_utils')
     @patch('dovetail.container.dt_cfg')
@@ -435,12 +437,32 @@ class ContainerTesting(unittest.TestCase):
         mock_utils.get_value_from_dict.side_effect = ['opts', None]
         mock_utils.get_hosts_info.return_value = 'host_info'
 
-        result = self.container.create(docker_image)
+        result, msg = self.container.create(docker_image)
 
         mock_utils.get_value_from_dict.assert_has_calls([
             call('opts', 'value'),
             call('shell', 'value')])
         self.assertEqual(None, result)
+        self.assertEqual("Lacking of key word 'shell' in config file.", msg)
+
+    @patch('dovetail.container.dt_utils')
+    @patch('dovetail.container.dt_cfg')
+    def test_create_mounts_none(self, mock_config, mock_utils):
+        docker_image = 'docker_image'
+        project_config = {}
+        mock_config.dovetail_config = {'bottlenecks': project_config}
+        mock_utils.get_value_from_dict.side_effect = [
+            {'key': 'value'}, 'shell', ['envs'], ['volume_one']]
+        mock_utils.get_mount_list.side_effect = [[None, 'error']]
+        mock_utils.get_hosts_info.return_value = 'host_info'
+
+        result, msg = self.container.create(docker_image)
+
+        mock_utils.get_value_from_dict.assert_has_calls([
+            call('opts', project_config), call('shell', project_config),
+            call('envs', project_config), call('volumes', project_config)])
+        self.assertEqual(None, result)
+        self.assertEqual('error', msg)
 
     @patch('dovetail.container.dt_utils')
     @patch('dovetail.container.dt_cfg')
@@ -448,13 +470,14 @@ class ContainerTesting(unittest.TestCase):
         docker_image = 'docker_image'
         mock_utils.get_value_from_dict.side_effect = [
             {'key': 'value'}, 'shell', ['envs'], ['volume_one']]
+        mock_utils.get_mount_list.side_effect = [['mount', 'list'], 'success']
         mock_utils.get_hosts_info.return_value = 'host_info'
         mock_utils.check_https_enabled.return_value = True
         self.client.containers.run.side_effect = \
             docker.errors.ImageNotFound('error')
         project_config = {}
         mock_config.dovetail_config = {'bottlenecks': project_config}
-        result = self.container.create(docker_image)
+        result, msg = self.container.create(docker_image)
 
         mock_utils.get_value_from_dict.assert_has_calls([
             call('opts', project_config),
@@ -463,3 +486,4 @@ class ContainerTesting(unittest.TestCase):
             call('volumes', project_config)])
         mock_utils.get_hosts_info.assert_called_once_with(self.logger)
         self.assertEqual(None, result)
+        self.assertEqual('error', str(docker.errors.ImageNotFound('error')))
diff --git a/dovetail/tests/unit/test_test_runner.py b/dovetail/tests/unit/test_test_runner.py
index 3cb27536..232de7b1 100644
--- a/dovetail/tests/unit/test_test_runner.py
+++ b/dovetail/tests/unit/test_test_runner.py
@@ -107,7 +107,7 @@ class TestRunnerTesting(unittest.TestCase):
         docker_img_obj = Mock()
         container_obj.get_docker_image.return_value = docker_img_obj
         container_obj.pull_image.return_value = True
-        container_obj.create.return_value = False
+        container_obj.create.return_value = [None, 'error']
         mock_container.return_value = container_obj
 
         docker_runner.run()
@@ -116,8 +116,8 @@ class TestRunnerTesting(unittest.TestCase):
         container_obj.get_docker_image.assert_called_once_with()
         container_obj.pull_image.assert_called_once_with(docker_img_obj)
         container_obj.create.assert_called_once_with(docker_img_obj)
-        docker_runner.logger.error.assert_called_once_with(
-            'Failed to create container.')
+        docker_runner.logger.error.assert_has_calls([
+            call('Failed to create container.'), call('error')])
 
     @patch('dovetail.test_runner.dt_utils')
     @patch('dovetail.test_runner.dt_cfg')
@@ -137,7 +137,8 @@ class TestRunnerTesting(unittest.TestCase):
         container_obj.get_docker_image.return_value = docker_img_obj
         container_obj.pull_image.return_value = True
         container_id = '12345'
-        container_obj.create.return_value = container_id
+        container_msg = 'Successfully to create container.'
+        container_obj.create.return_value = [container_id, container_msg]
         mock_container.return_value = container_obj
         self.testcase.pre_condition.return_value = ['cmd']
         self.testcase.prepare_cmd.return_value = False
@@ -180,7 +181,8 @@ class TestRunnerTesting(unittest.TestCase):
         container_obj.get_docker_image.return_value = docker_img_obj
         container_obj.pull_image.return_value = True
         container_id = '12345'
-        container_obj.create.return_value = container_id
+        container_msg = 'Successfully to create container.'
+        container_obj.create.return_value = [container_id, container_msg]
         mock_container.return_value = container_obj
         self.testcase.pre_condition.return_value = ['cmd']
         self.testcase.prepare_cmd.return_value = True
diff --git a/dovetail/tests/unit/utils/test_dovetail_utils.py b/dovetail/tests/unit/utils/test_dovetail_utils.py
index 7ec177d1..7d1fddc1 100644
--- a/dovetail/tests/unit/utils/test_dovetail_utils.py
+++ b/dovetail/tests/unit/utils/test_dovetail_utils.py
@@ -1380,3 +1380,23 @@ class DovetailUtilsTesting(unittest.TestCase):
         logger.debug.assert_not_called()
         logger.exception.assert_called_once_with(
             "The results cannot be pushed to DB.")
+
+    def test_get_mount_list_error_mount(self):
+        project_cfg = {'mounts': ['aaa']}
+        res, msg = dovetail_utils.get_mount_list(project_cfg)
+        self.assertEqual(None, res)
+        self.assertEqual('Error mount aaa.', msg)
+
+    def test_get_mount_list_keyerror_exception(self):
+        project_cfg = {'mounts': ['aaa=a,bbb=b', '']}
+        res, msg = dovetail_utils.get_mount_list(project_cfg)
+        self.assertEqual(None, res)
+        self.assertEqual("'target'", str(msg))
+
+    def test_get_mount_list(self):
+        project_cfg = {'mounts': ['target=a,source=b', '']}
+        res, msg = dovetail_utils.get_mount_list(project_cfg)
+        expected = [{'Source': 'b', 'Type': 'bind', 'ReadOnly': False,
+                     'Target': 'a'}]
+        self.assertEqual(expected, res)
+        self.assertEqual('Successfully to get mount list.', msg)
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index 306dacd1..1c4aca9d 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -21,6 +21,7 @@ from distutils.version import LooseVersion
 import yaml
 import python_hosts
 import docker
+from docker.types import Mount
 
 from dovetail import constants
 from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg
@@ -432,3 +433,26 @@ def push_results_to_db(case_name, details, start_date, stop_date, logger):
     except Exception:
         logger.exception('The results cannot be pushed to DB.')
         return False
+
+
+def get_mount_list(project_cfg):
+    mount_list = []
+    mounts = get_value_from_dict('mounts', project_cfg)
+    for mount in mounts:
+        if mount:
+            param_dict = {}
+            for param in mount.split(','):
+                key_word = param.split('=')
+
+                if len(key_word) != 2:
+                    return None, 'Error mount {}.'.format(mount)
+
+                param_dict[key_word[0]] = key_word[1]
+            try:
+                mount_list.append(Mount(target=param_dict['target'],
+                                        source=param_dict['source'],
+                                        type='bind'))
+            except Exception as e:
+                return None, e
+
+    return mount_list, 'Successfully to get mount list.'
diff --git a/etc/conf/bottlenecks_config.yml b/etc/conf/bottlenecks_config.yml
index bd5e106a..d36f9bbf 100644
--- a/etc/conf/bottlenecks_config.yml
+++ b/etc/conf/bottlenecks_config.yml
@@ -17,7 +17,7 @@
 {% set build_tag = build_tag or '' %}
 {% set cacert_volume = '' %}
 {% if cacert %}
-    {% set cacert_volume = cacert + ':' + cacert %}
+    {% set cacert_volume = 'source=' + cacert + ',target=' + cacert %}
 {% endif %}
 {% set openrc_file = '/tmp/admin_rc.sh' %}
 {% set result_dir = '/home/opnfv/bottlenecks/results' %}
@@ -40,12 +40,13 @@ bottlenecks:
     - 'CI_DEBUG={{debug}}'
     - 'BUILD_TAG={{build_tag}}-{{testcase}}'
   volumes:
-    - '/var/run/docker.sock:/var/run/docker.sock'
     - '{{dovetail_home}}/results/bottlenecks:/tmp'
-    - '{{dovetail_home}}/pre_config/env_config.sh:{{openrc_file}}'
-    - {{cacert_volume}}
     - '{{dovetail_home}}/images:{{images_dir}}'
     - '{{dovetail_home}}/results:{{result_dir}}'
+  mounts:
+    - 'source=/var/run/docker.sock,target=/var/run/docker.sock'
+    - 'source={{dovetail_home}}/pre_config/env_config.sh,target={{openrc_file}}'
+    - {{cacert_volume}}
   pre_condition:
     - 'cp {{images_dir}}/ubuntu-16.04-server-cloudimg-amd64-disk1.img {{image_file}}'
   cmds:
diff --git a/etc/conf/functest-k8s_config.yml b/etc/conf/functest-k8s_config.yml
index 52631673..a01f5715 100644
--- a/etc/conf/functest-k8s_config.yml
+++ b/etc/conf/functest-k8s_config.yml
@@ -33,9 +33,10 @@ functest-k8s:
     - 'CI_DEBUG={{debug}}'
     - 'BUILD_TAG={{build_tag}}-{{testcase}}'
   volumes:
-    - '{{dovetail_home}}/pre_config/k8.creds:{{openrc_file}}'
-    - '{{dovetail_home}}/pre_config/admin.conf:{{kube_file}}'
     - '{{dovetail_home}}/results/:{{result_dir}}'
+  mounts:
+    - 'source={{dovetail_home}}/pre_config/k8.creds,target={{openrc_file}}'
+    - 'source={{dovetail_home}}/pre_config/admin.conf,target={{kube_file}}'
   pre_condition:
     - 'echo test for precondition in functest'
   cmds:
diff --git a/etc/conf/functest_config.yml b/etc/conf/functest_config.yml
index 06def4d4..91fd68e2 100644
--- a/etc/conf/functest_config.yml
+++ b/etc/conf/functest_config.yml
@@ -22,7 +22,7 @@
 {% set build_tag = build_tag or '' %}
 {% set cacert_volume = '' %}
 {% if cacert %}
-    {% set cacert_volume = cacert + ':' + cacert %}
+    {% set cacert_volume = 'source=' + cacert + ',target=' + cacert %}
 {% endif %}
 {% set openrc_file = '/home/opnfv/functest/conf/env_file' %}
 {% set result_dir = '/home/opnfv/functest/results' %}
@@ -47,13 +47,14 @@ functest:
     - 'CI_DEBUG={{debug}}'
     - 'BUILD_TAG={{build_tag}}-{{testcase}}'
   volumes:
-    - '{{dovetail_home}}/pre_config/env_config.sh:{{openrc_file}}'
-    - {{cacert_volume}}
-    - '{{dovetail_home}}/pre_config:/home/opnfv/pre_config'
     - '{{dovetail_home}}/userconfig:{{userconfig_dir}}'
     - '{{dovetail_home}}/patches:{{patches_dir}}'
     - '{{dovetail_home}}/results:{{result_dir}}'
     - '{{dovetail_home}}/images:{{images_dir}}'
+  mounts:
+    - 'source={{dovetail_home}}/pre_config/env_config.sh,target={{openrc_file}}'
+    - 'source={{dovetail_home}}/pre_config,target=/home/opnfv/pre_config'
+    - {{cacert_volume}}
   patches_dir: {{patches_dir}}
   pre_condition:
     - 'echo test for precondition in functest'
diff --git a/etc/conf/yardstick_config.yml b/etc/conf/yardstick_config.yml
index e4758c8e..21716f8f 100644
--- a/etc/conf/yardstick_config.yml
+++ b/etc/conf/yardstick_config.yml
@@ -23,7 +23,7 @@
 {% set build_tag = build_tag or '' %}
 {% set cacert_volume = '' %}
 {% if cacert %}
-    {% set cacert_volume = cacert + ':' + cacert %}
+    {% set cacert_volume = 'source=' + cacert + ',target=' + cacert %}
 {% endif %}
 {% set openrc_file = '/etc/yardstick/openstack.creds' %}
 {% set pod_file = '/etc/yardstick/pod.yaml' %}
@@ -43,12 +43,13 @@ yardstick:
     - 'CI_DEBUG={{debug}}'
     - 'BUILD_TAG={{build_tag}}-{{testcase}}"'
   volumes:
-    - '{{dovetail_home}}/pre_config/env_config.sh:{{openrc_file}}'
-    - {{cacert_volume}}
-    - '{{dovetail_home}}/pre_config/pod.yaml:{{pod_file}}'
     - '{{dovetail_home}}/images:/home/opnfv/images'
     - '{{dovetail_home}}/results:{{result_dir}}'
-    - '{{dovetail_home}}/pre_config:{{dovetail_home}}/pre_config'
+  mounts:
+    - 'source={{dovetail_home}}/pre_config,target={{dovetail_home}}/pre_config'
+    - 'source={{dovetail_home}}/pre_config/env_config.sh,target={{openrc_file}}'
+    - 'source={{dovetail_home}}/pre_config/pod.yaml,target={{pod_file}}'
+    - {{cacert_volume}}
   pre_condition:
     - 'echo this is pre_condition'
   cmds: