Add multiple VM support

Add support to run Doctor test with more than one VM by setting
VM_COUNT to wanted amount of VMs. Normally half of those VMs
gets scheduled to host that is used for Doctor testing.

JIRA: DOCTOR-77

Change-Id: I9fac96c6f53c290fccc35a2fd933b9fb106b96f2
Signed-off-by: Tomi Juvonen <tomi.juvonen@nokia.com>

1 files changed, 43 insertions, 12 deletions

diff --git a/tests/inspector.py b/tests/inspector.py
index c1f95697..7195969a 100644
--- a/tests/inspector.py
+++ b/tests/inspector.py
@@ -14,6 +14,7 @@ from flask import request
 import json
 import logger as doctor_log
 import os
+import threading
 import time
 
 import novaclient.client as novaclient
@@ -23,19 +24,41 @@ import nova_force_down
 LOG = doctor_log.Logger('doctor_inspector').getLogger()
 
 
+class ThreadedResetState(threading.Thread):
+
+    def __init__(self, nova, state, server):
+        threading.Thread.__init__(self)
+        self.nova = nova
+        self.state = state
+        self.server = server
+
+    def run(self):
+        self.nova.servers.reset_state(self.server, self.state)
+        LOG.info('doctor mark vm(%s) error at %s' % (self.server, time.time()))
+
+
 class DoctorInspectorSample(object):
 
-    nova_api_version = '2.11'
+    NOVA_API_VERSION = '2.11'
+    NUMBER_OF_CLIENTS = 50
+    # TODO(tojuvone): This could be enhanced in future with dynamic
+    # reuse of self.novaclients when all threads in use and
+    # self.NUMBER_OF_CLIENTS based on amount of cores or overriden by input
+    # argument
 
     def __init__(self):
         self.servers = collections.defaultdict(list)
-        self.nova = novaclient.Client(self.nova_api_version,
-                                      os.environ['OS_USERNAME'],
-                                      os.environ['OS_PASSWORD'],
-                                      os.environ['OS_TENANT_NAME'],
-                                      os.environ['OS_AUTH_URL'],
-                                      connection_pool=True)
-        # check nova is available
+        self.novaclients = list()
+        # Pool of novaclients for redundant usage
+        for i in range(self.NUMBER_OF_CLIENTS):
+            self.novaclients.append(novaclient.Client(self.NOVA_API_VERSION,
+                                    os.environ['OS_USERNAME'],
+                                    os.environ['OS_PASSWORD'],
+                                    os.environ['OS_TENANT_NAME'],
+                                    os.environ['OS_AUTH_URL'],
+                                    connection_pool=True))
+        # Normally we use this client for non redundant API calls
+        self.nova=self.novaclients[0]
         self.nova.servers.list(detailed=False)
         self.init_servers_list()
 
@@ -52,10 +75,18 @@ class DoctorInspectorSample(object):
                 LOG.error('can not get hostname from server=%s' % server)
 
     def disable_compute_host(self, hostname):
-        for server in self.servers[hostname]:
-            self.nova.servers.reset_state(server, 'error')
-            LOG.info('doctor mark vm(%s) error at %s' % (server, time.time()))
-
+        threads = []
+        if len(self.servers[hostname]) > self.NUMBER_OF_CLIENTS:
+            # TODO(tojuvone): This could be enhanced in future with dynamic
+            # reuse of self.novaclients when all threads in use
+            LOG.error('%d servers in %s. Can handle only %d'%(
+                      self.servers[hostname], hostname, self.NUMBER_OF_CLIENTS))
+        for nova, server in zip(self.novaclients, self.servers[hostname]):
+            t = ThreadedResetState(nova, "error", server)
+            t.start()
+            threads.append(t)
+        for t in threads:
+            t.join()
         # NOTE: We use our own client here instead of this novaclient for a
         #       workaround.  Once keystone provides v2.1 nova api endpoint
         #       in the service catalog which is configured by OpenStack