Add "volumes" parameter in Kubernetes context

This new parameter, "volumes", will allow the user to automatically create
new volumes. Example of Kubernetes context definition:

  context:
    type: Kubernetes
    servers:
      host:
        image: ...
        commands: ...
        volumes:
          - name: volume1              # mandatory
            <volume type definition>   # mandatory

The volume type and the definition must be one of the supported ones in
Kubernetes [1].

[1] https://kubernetes.io/docs/concepts/storage/volumes/#types-of-volumes

JIRA: YARDSTICK-1152

Change-Id: I44a91c605f047de4f286407e28fb5aa2e921b00a
Signed-off-by: Rodolfo Alonso Hernandez <rodolfo.alonso.hernandez@intel.com>

6 files changed, 118 insertions, 40 deletions

diff --git a/yardstick/benchmark/contexts/kubernetes.py b/yardstick/benchmark/contexts/kubernetes.py
index 4bea991ea..6b00c0374 100644
--- a/yardstick/benchmark/contexts/kubernetes.py
+++ b/yardstick/benchmark/contexts/kubernetes.py
@@ -15,7 +15,7 @@ import pkg_resources
 import paramiko
 
 from yardstick.benchmark.contexts.base import Context
-from yardstick.orchestrator.kubernetes import KubernetesTemplate
+from yardstick.orchestrator import kubernetes
 from yardstick.common import kubernetes_utils as k8s_utils
 from yardstick.common import utils
 
@@ -39,11 +39,8 @@ class KubernetesContext(Context):
     def init(self, attrs):
         super(KubernetesContext, self).init(attrs)
 
-        template_cfg = attrs.get('servers', {})
-        self.template = KubernetesTemplate(self.name, template_cfg)
-
+        self.template = kubernetes.KubernetesTemplate(self.name, attrs)
         self.ssh_key = '{}-key'.format(self.name)
-
         self.key_path = self._get_key_path()
         self.public_key_path = '{}.pub'.format(self.key_path)
 
diff --git a/yardstick/common/exceptions.py b/yardstick/common/exceptions.py
index 966b15cf8..5f362b356 100644
--- a/yardstick/common/exceptions.py
+++ b/yardstick/common/exceptions.py
@@ -195,6 +195,10 @@ class WaitTimeout(YardstickException):
     message = 'Wait timeout while waiting for condition'
 
 
+class KubernetesTemplateInvalidVolumeType(YardstickException):
+    message = 'No valid "volume" types present in %(volume)s'
+
+
 class ScenarioCreateNetworkError(YardstickException):
     message = 'Create Neutron Network Scenario failed'
 
diff --git a/yardstick/common/kubernetes_utils.py b/yardstick/common/kubernetes_utils.py
index d60c9b23a..ee8e8edcd 100644
--- a/yardstick/common/kubernetes_utils.py
+++ b/yardstick/common/kubernetes_utils.py
@@ -199,3 +199,9 @@ def get_pod_list(namespace='default'):      # pragma: no cover
 def get_pod_by_name(name):  # pragma: no cover
     pod_list = get_pod_list()
     return next((n for n in pod_list.items if n.metadata.name.startswith(name)), None)
+
+
+def get_volume_types():
+    """Return the "volume" types supported by the current API"""
+    return [vtype for vtype in client.V1Volume.attribute_map.values()
+            if vtype != 'name']
diff --git a/yardstick/orchestrator/kubernetes.py b/yardstick/orchestrator/kubernetes.py
index ac3a09ed1..ae6c9459b 100644
--- a/yardstick/orchestrator/kubernetes.py
+++ b/yardstick/orchestrator/kubernetes.py
@@ -7,9 +7,9 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from __future__ import absolute_import
-from __future__ import print_function
+import copy
 
+from yardstick.common import exceptions
 from yardstick.common import utils
 from yardstick.common import kubernetes_utils as k8s_utils
 
@@ -24,8 +24,7 @@ class KubernetesObject(object):
         self.args = kwargs.get('args', [])
         self.ssh_key = kwargs.get('ssh_key', 'yardstick_key')
         self.node_selector = kwargs.get('nodeSelector', {})
-
-        self.volumes = []
+        self._volumes = kwargs.get('volumes', [])
 
         self.template = {
             "apiVersion": "v1",
@@ -53,7 +52,6 @@ class KubernetesObject(object):
         self._change_value_according_name(name)
         self._add_containers()
         self._add_node_selector()
-        self._add_ssh_key_volume()
         self._add_volumes()
 
     def get_template(self):
@@ -97,21 +95,32 @@ class KubernetesObject(object):
                              self.node_selector)
 
     def _add_volumes(self):
+        """Add "volume" items to container specs, including the SSH one"""
+        volume_items = [self._create_volume(vol) for vol in self._volumes]
+        volume_items.append(self._create_ssh_key_volume())
         utils.set_dict_value(self.template,
                              'spec.template.spec.volumes',
-                             self.volumes)
+                             volume_items)
 
-    def _add_volume(self, volume):
-        self.volumes.append(volume)
+    def _create_ssh_key_volume(self):
+        """Create a "volume" item of type "configMap" for the SSH key"""
+        return {'name': self.ssh_key,
+                'configMap': {'name': self.ssh_key}}
 
-    def _add_ssh_key_volume(self):
-        key_volume = {
-            "configMap": {
-                "name": self.ssh_key
-            },
-            "name": self.ssh_key
-        }
-        self._add_volume(key_volume)
+    @staticmethod
+    def _create_volume(volume):
+        """Create a "volume" item"""
+        volume = copy.deepcopy(volume)
+        name = volume.pop('name')
+        for key in (k for k in volume if k in k8s_utils.get_volume_types()):
+            type_name = key
+            type_data = volume[key]
+            break
+        else:
+            raise exceptions.KubernetesTemplateInvalidVolumeType(volume=volume)
+
+        return {'name': name,
+                type_name: type_data}
 
 
 class ServiceObject(object):
@@ -145,15 +154,22 @@ class ServiceObject(object):
 
 class KubernetesTemplate(object):
 
-    def __init__(self, name, template_cfg):
+    def __init__(self, name, context_cfg):
+        """KubernetesTemplate object initialization
+
+        :param name: (str) name of the Kubernetes context
+        :param context_cfg: (dict) context definition
+        """
+        context_cfg = copy.deepcopy(context_cfg)
+        servers_cfg = context_cfg.pop('servers', {})
         self.name = name
         self.ssh_key = '{}-key'.format(name)
 
-        self.rcs = [self._get_rc_name(rc) for rc in template_cfg]
+        self.rcs = [self._get_rc_name(rc) for rc in servers_cfg]
         self.k8s_objs = [KubernetesObject(self._get_rc_name(rc),
                                           ssh_key=self.ssh_key,
                                           **cfg)
-                         for rc, cfg in template_cfg.items()]
+                         for rc, cfg in servers_cfg.items()]
         self.service_objs = [ServiceObject(s) for s in self.rcs]
 
         self.pods = []
diff --git a/yardstick/tests/unit/benchmark/contexts/test_kubernetes.py b/yardstick/tests/unit/benchmark/contexts/test_kubernetes.py
index 0e11a53e1..0d698c5bd 100644
--- a/yardstick/tests/unit/benchmark/contexts/test_kubernetes.py
+++ b/yardstick/tests/unit/benchmark/contexts/test_kubernetes.py
@@ -12,9 +12,10 @@ import unittest
 
 from yardstick.benchmark.contexts import base
 from yardstick.benchmark.contexts import kubernetes
+from yardstick.orchestrator import kubernetes as orchestrator_kubernetes
 
 
-context_cfg = {
+CONTEXT_CFG = {
     'type': 'Kubernetes',
     'name': 'k8s',
     'task_id': '1234567890',
@@ -22,14 +23,14 @@ context_cfg = {
         'host': {
             'image': 'openretriever/yardstick',
             'command': '/bin/bash',
-            'args': ['-c', 'chmod 700 ~/.ssh; chmod 600 ~/.ssh/*; \
-service ssh restart;while true ; do sleep 10000; done']
+            'args': ['-c', 'chmod 700 ~/.ssh; chmod 600 ~/.ssh/*; '
+                     'service ssh restart;while true ; do sleep 10000; done']
         },
         'target': {
             'image': 'openretriever/yardstick',
             'command': '/bin/bash',
-            'args': ['-c', 'chmod 700 ~/.ssh; chmod 600 ~/.ssh/*; \
-service ssh restart;while true ; do sleep 10000; done']
+            'args': ['-c', 'chmod 700 ~/.ssh; chmod 600 ~/.ssh/*; '
+                     'service ssh restart;while true ; do sleep 10000; done']
         }
     }
 }
@@ -42,7 +43,7 @@ class KubernetesTestCase(unittest.TestCase):
     def setUp(self):
         self.k8s_context = kubernetes.KubernetesContext()
         self.addCleanup(self._remove_contexts)
-        self.k8s_context.init(context_cfg)
+        self.k8s_context.init(CONTEXT_CFG)
 
     @staticmethod
     def _remove_contexts():
@@ -68,7 +69,8 @@ class KubernetesTestCase(unittest.TestCase):
 
     @mock.patch.object(kubernetes.KubernetesContext, '_create_services')
     @mock.patch.object(kubernetes.KubernetesContext, '_wait_until_running')
-    @mock.patch.object(kubernetes.KubernetesTemplate, 'get_rc_pods')
+    @mock.patch.object(orchestrator_kubernetes.KubernetesTemplate,
+                       'get_rc_pods')
     @mock.patch.object(kubernetes.KubernetesContext, '_create_rcs')
     @mock.patch.object(kubernetes.KubernetesContext, '_set_ssh_key')
     def test_deploy(self,
@@ -170,3 +172,13 @@ class KubernetesTestCase(unittest.TestCase):
     def test_delete_services(self, mock_delete):
         self.k8s_context._delete_services()
         mock_delete.assert_called()
+
+    def test_init(self):
+        self.k8s_context._delete_context()
+        with mock.patch.object(orchestrator_kubernetes, 'KubernetesTemplate',
+                return_value='fake_template') as mock_k8stemplate:
+            self.k8s_context = kubernetes.KubernetesContext()
+            self.k8s_context.init(CONTEXT_CFG)
+        mock_k8stemplate.assert_called_once_with(self.k8s_context.name,
+                                                 CONTEXT_CFG)
+        self.assertEqual('fake_template', self.k8s_context.template)
diff --git a/yardstick/tests/unit/orchestrator/test_kubernetes.py b/yardstick/tests/unit/orchestrator/test_kubernetes.py
index 58971f515..e3e5516ca 100644
--- a/yardstick/tests/unit/orchestrator/test_kubernetes.py
+++ b/yardstick/tests/unit/orchestrator/test_kubernetes.py
@@ -7,15 +7,15 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-# Unittest for yardstick.benchmark.orchestrator.heat
-import unittest
 import mock
 
-from yardstick.orchestrator.kubernetes import KubernetesObject
-from yardstick.orchestrator.kubernetes import KubernetesTemplate
+from yardstick.common import exceptions
+from yardstick.common import kubernetes_utils
+from yardstick.orchestrator import kubernetes
+from yardstick.tests.unit import base
 
 
-class GetTemplateTestCase(unittest.TestCase):
+class GetTemplateTestCase(base.BaseUnitTestCase):
 
     def test_get_template(self):
         output_t = {
@@ -73,14 +73,15 @@ service ssh restart;while true ; do sleep 10000; done"
             'args': ['-c', 'chmod 700 ~/.ssh; chmod 600 ~/.ssh/*; \
 service ssh restart;while true ; do sleep 10000; done'],
             'ssh_key': 'k8s-86096c30-key',
-            'nodeSelector': {'kubernetes.io/hostname': 'node-01'}
+            'nodeSelector': {'kubernetes.io/hostname': 'node-01'},
+            'volumes': []
         }
         name = 'host-k8s-86096c30'
-        output_r = KubernetesObject(name, **input_s).get_template()
+        output_r = kubernetes.KubernetesObject(name, **input_s).get_template()
         self.assertEqual(output_r, output_t)
 
 
-class GetRcPodsTestCase(unittest.TestCase):
+class GetRcPodsTestCase(base.BaseUnitTestCase):
 
     @mock.patch('yardstick.orchestrator.kubernetes.k8s_utils.get_pod_list')
     def test_get_rc_pods(self, mock_get_pod_list):
@@ -98,7 +99,49 @@ service ssh restart;while true ; do sleep 10000; done']
 service ssh restart;while true ; do sleep 10000; done']
             }
         }
-        k8s_template = KubernetesTemplate('k8s-86096c30', servers)
+        k8s_template = kubernetes.KubernetesTemplate('k8s-86096c30', servers)
         mock_get_pod_list.return_value.items = []
         pods = k8s_template.get_rc_pods()
         self.assertEqual(pods, [])
+
+
+class KubernetesObjectTestCase(base.BaseUnitTestCase):
+
+    def test__add_volumes(self):
+        volume1 = {'name': 'fake_sshkey',
+                   'configMap': {'name': 'fake_sshkey'}}
+        volume2 = {'name': 'volume2',
+                   'configMap': 'data'}
+        k8s_obj = kubernetes.KubernetesObject('name', ssh_key='fake_sshkey',
+                                              volumes=[volume2])
+        k8s_obj._add_volumes()
+        volumes = k8s_obj.template['spec']['template']['spec']['volumes']
+        self.assertEqual(sorted([volume1, volume2], key=lambda k: k['name']),
+                         sorted(volumes, key=lambda k: k['name']))
+
+    def test__add_volumes_no_volumes(self):
+        volume1 = {'name': 'fake_sshkey',
+                   'configMap': {'name': 'fake_sshkey'}}
+        k8s_obj = kubernetes.KubernetesObject('name', ssh_key='fake_sshkey')
+        k8s_obj._add_volumes()
+        volumes = k8s_obj.template['spec']['template']['spec']['volumes']
+        self.assertEqual([volume1], volumes)
+
+    def test__create_ssh_key_volume(self):
+        expected = {'name': 'fake_sshkey',
+                    'configMap': {'name': 'fake_sshkey'}}
+        k8s_obj = kubernetes.KubernetesObject('name', ssh_key='fake_sshkey')
+        self.assertEqual(expected, k8s_obj._create_ssh_key_volume())
+
+    def test__create_volume(self):
+        for vol_type in kubernetes_utils.get_volume_types():
+            volume = {'name': 'vol_name',
+                      vol_type: 'data'}
+            self.assertEqual(
+                volume, kubernetes.KubernetesObject._create_volume(volume))
+
+    def test__create_volume_invalid_type(self):
+        volume = {'name': 'vol_name',
+                  'invalid_type': 'data'}
+        with self.assertRaises(exceptions.KubernetesTemplateInvalidVolumeType):
+            kubernetes.KubernetesObject._create_volume(volume)