Create Stack Fixes

Removes check stack as that can timeout when a large
number of agents are requested

Waits for stack to be created before returning

JIRA: STORPERF-70

Change-Id: I1cbe45b1f60f0ecab0691f0822ed5643778d218b
Signed-off-by: Mark Beierl <mark.beierl@dell.com>

3 files changed, 32 insertions, 24 deletions

diff --git a/rest_server.py b/rest_server.py
index 390cb8c..2b5bbfb 100644
--- a/rest_server.py
+++ b/rest_server.py
@@ -7,9 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from storperf.db.job_db import JobDB
-from storperf.plot.barchart import Barchart
-from storperf.storperf_master import StorPerfMaster
 import io
 import json
 import logging
@@ -20,6 +17,10 @@ from flask import abort, Flask, request, jsonify, send_from_directory
 from flask_restful import Resource, Api, fields
 from flask_restful_swagger import swagger
 
+from storperf.db.job_db import JobDB
+from storperf.plot.barchart import Barchart
+from storperf.storperf_master import StorPerfMaster
+
 
 app = Flask(__name__, static_url_path="")
 api = swagger.docs(Api(app), apiVersion='1.0')
@@ -191,7 +192,6 @@ class Configure(Resource):
             if ('volume_size' in request.json):
                 storperf.volume_size = request.json['volume_size']
 
-            storperf.validate_stack()
             storperf.create_stack()
 
             return jsonify({'agent_count': storperf.agent_count,
diff --git a/storperf/fio/fio_invoker.py b/storperf/fio/fio_invoker.py
index 4f39eb7..59dbdaf 100644
--- a/storperf/fio/fio_invoker.py
+++ b/storperf/fio/fio_invoker.py
@@ -77,6 +77,8 @@ class FIOInvoker(object):
         else:
             cmd = "ssh"
             additional_args = ['-o', 'StrictHostKeyChecking=no',
+                               '-o', 'UserKnownHostsFile=/dev/null',
+                               '-o', 'LogLevel=error',
                                '-i', 'storperf/resources/ssh/storperf_rsa',
                                'storperf@' + self.remote_host,
                                "sudo", "./fio"]
@@ -102,6 +104,8 @@ class FIOInvoker(object):
     def terminate(self):
         self.logger.debug("Terminating fio on " + self.remote_host)
         cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
+               '-o', 'UserKnownHostsFile=/dev/null',
+               '-o', 'LogLevel=error',
                '-i', 'storperf/resources/ssh/storperf_rsa',
                'storperf@' + self.remote_host,
                'sudo', 'killall', '-9', 'fio']
diff --git a/storperf/storperf_master.py b/storperf/storperf_master.py
index c4e6e77..aa1b9b6 100644
--- a/storperf/storperf_master.py
+++ b/storperf/storperf_master.py
@@ -8,19 +8,19 @@
 ##############################################################################
 
 from datetime import datetime
-from storperf.db.graphite_db import GraphiteDB
-from threading import Thread
-from time import sleep
 import logging
 import os
 import subprocess
+from threading import Thread
+from time import sleep
 
+import cinderclient.v2 as cinderclient
 from db.configuration_db import ConfigurationDB
 from db.job_db import JobDB
-from test_executor import TestExecutor
-import cinderclient.v2 as cinderclient
 import heatclient.client as heatclient
 import keystoneclient.v2_0 as ksclient
+from storperf.db.graphite_db import GraphiteDB
+from test_executor import TestExecutor
 
 
 class ParameterError(Exception):
@@ -257,21 +257,19 @@ class StorPerfMaster(object):
 
         self.stack_id = stack['stack']['id']
 
-    def validate_stack(self):
-        self._attach_to_openstack()
-        volume_quota = self.volume_quota
-        if (volume_quota > 0 and self.agent_count > volume_quota):
-            message = "ERROR: Volume quota too low: " + \
-                str(self.agent_count) + " > " + str(self.volume_quota)
-            self.logger.error(message)
-            raise ParameterError(message)
-
-        self._heat_client.stacks.preview(
-            stack_name="StorPerfAgentGroup",
-            template=self._agent_group_hot,
-            files=self._hot_files,
-            parameters=self._make_parameters())
-        return True
+        while True:
+            stack = self._heat_client.stacks.get(self.stack_id)
+            status = getattr(stack, 'stack_status')
+            self.logger.debug("Stack status=%s" % (status,))
+            if (status == u'CREATE_COMPLETE'):
+                return True
+            if (status == u'DELETE_COMPLETE'):
+                self.stack_id = None
+                return True
+            if (status == u'CREATE_FAILED'):
+                sleep(5)
+                self._heat_client.stacks.delete(stack_id=self.stack_id)
+            sleep(2)
 
     def delete_stack(self):
         if (self.stack_id is None):
@@ -338,6 +336,8 @@ class StorPerfMaster(object):
         logger.info("Initializing slave at " + slave)
 
         args = ['scp', '-o', 'StrictHostKeyChecking=no',
+                '-o', 'UserKnownHostsFile=/dev/null',
+                '-o', 'LogLevel=error',
                 '-i', 'storperf/resources/ssh/storperf_rsa',
                 '/lib/x86_64-linux-gnu/libaio.so.1',
                 'storperf@' + slave + ":"]
@@ -355,6 +355,8 @@ class StorPerfMaster(object):
             logger.error(stderr.decode('utf-8').strip())
 
         args = ['scp', '-o', 'StrictHostKeyChecking=no',
+                '-o', 'UserKnownHostsFile=/dev/null',
+                '-o', 'LogLevel=error',
                 '-i', 'storperf/resources/ssh/storperf_rsa',
                 '/usr/local/bin/fio',
                 'storperf@' + slave + ":"]
@@ -372,6 +374,8 @@ class StorPerfMaster(object):
             logger.error(stderr.decode('utf-8').strip())
 
         args = ['ssh', '-o', 'StrictHostKeyChecking=no',
+                '-o', 'UserKnownHostsFile=/dev/null',
+                '-o', 'LogLevel=error',
                 '-i', 'storperf/resources/ssh/storperf_rsa',
                 'storperf@' + slave,
                 'sudo cp -v libaio.so.1 /lib/x86_64-linux-gnu/libaio.so.1'