summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTaseer Ahmed <taseer94@gmail.com>2017-11-14 16:57:31 +0500
committermbeierl <mark.beierl@dell.com>2018-01-11 15:38:36 -0500
commitf0e1ee0ccc25acf74f472146f4219350bfec0248 (patch)
treee7e28e04ab01c384f3cc545eb631b2a9e5af0061
parent9100f1afcc426b387dee3673151786680bfe0718 (diff)
WIP - Use SNAPS to launch StorPerf Stack
Change-Id: I8857e6e0dd2bf9cd7c38cf18a7c239d7eb993d05 Signed-off-by: Taseer Ahmed <taseer94@gmail.com>
-rwxr-xr-xci/verify-build.sh9
-rwxr-xr-xci/verify.sh21
-rw-r--r--docker/storperf-master/requirements.pip2
-rw-r--r--docker/storperf-master/rest_server.py4
-rw-r--r--docker/storperf-master/storperf/storperf_master.py209
-rw-r--r--docker/storperf-master/storperf/test_executor.py4
-rw-r--r--docker/storperf-master/tests/storperf_master_test.py18
7 files changed, 87 insertions, 180 deletions
diff --git a/ci/verify-build.sh b/ci/verify-build.sh
index 0477e17..c98fea4 100755
--- a/ci/verify-build.sh
+++ b/ci/verify-build.sh
@@ -19,7 +19,12 @@ export CARBON_DIR=${ci}/job/carbon/
${ci}/remove_docker_container.sh
mkdir -p ${CARBON_DIR}
-touch ${ENV_FILE}
+cat << EOF > ${ENV_FILE}
+OS_USERNAME=username
+OS_PASSWORD=password
+OS_AUTH_URL=http://localhost
+OS_PROJECT_NAME=project
+EOF
if [ -z $ARCH ]
then
@@ -60,7 +65,7 @@ set +e
check_for_life storperf-httpfrontend "/"
FAILURES=$((FAILURES + $?))
-check_for_life storperf-master "/api/v1.0/configurations"
+check_for_life storperf-master "/api/v1.0/jobs"
FAILURES=$((FAILURES + $?))
check_for_life storperf-reporting "/reporting/"
diff --git a/ci/verify.sh b/ci/verify.sh
index cd5bbbb..996a3af 100755
--- a/ci/verify.sh
+++ b/ci/verify.sh
@@ -22,28 +22,11 @@ pip install --upgrade setuptools==33.1.1
pip install autoflake==0.6.6
pip install autopep8==1.2.2
pip install coverage==4.1
-pip install flask==0.10
-pip install flask_cors==3.0.2
-pip install flask-restful==0.3.5
-pip install flask-restful-swagger==0.19
-pip install flask-swagger==0.2.12
-pip install funcsigs==0.4
+pip install cryptography==1.7.2
pip install flake8==2.5.4
-pip install html2text==2016.1.8
-pip install keystoneauth1==2.12.1
pip install mock==1.3.0
pip install nose==1.3.7
-pip install paramiko==2.0.2
-pip install python-cinderclient==1.6.0
-pip install python-glanceclient==1.1.0
-pip install python-heatclient==0.8.0
-pip install python-keystoneclient==1.6.0
-pip install python-neutronclient==2.6.0
-pip install python-novaclient==2.28.1
-pip install pyyaml==3.10
-pip install requests==2.13.0
-pip install scp==0.10.2
-pip install six==1.10.0
+pip install -r docker/storperf-master/requirements.pip
final_rc=0
diff --git a/docker/storperf-master/requirements.pip b/docker/storperf-master/requirements.pip
index 5eb31cc..a591e84 100644
--- a/docker/storperf-master/requirements.pip
+++ b/docker/storperf-master/requirements.pip
@@ -8,4 +8,4 @@ html2text==2016.1.8
paramiko==2.0.2
requests==2.13.0
scp==0.10.2
-snaps==6.0.0.dev309
+git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps
diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 9280721..67d2d05 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -199,8 +199,9 @@ class Configure(Resource):
)
def delete(self):
try:
- storperf.delete_stack()
+ return jsonify({'stack_id': storperf.delete_stack()})
except Exception as e:
+ self.logger.exception(e)
abort(400, str(e))
@@ -355,6 +356,7 @@ for any single test iteration.
return jsonify({'job_id': job_id})
except Exception as e:
+ self.logger.exception(e)
abort(400, str(e))
@swagger.operation(
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index ad6e16c..45d5d89 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -14,16 +14,16 @@ import socket
from threading import Thread
from time import sleep
-from cinderclient import client as cinderclient
-from keystoneauth1 import loading
-from keystoneauth1 import session
import paramiko
from scp import SCPClient
+from snaps.config.stack import StackConfig
+from snaps.openstack.create_stack import OpenStackHeatStack
+from snaps.openstack.os_credentials import OSCreds
-import heatclient.client as heatclient
from storperf.db.configuration_db import ConfigurationDB
from storperf.db.job_db import JobDB
from storperf.test_executor import TestExecutor
+from snaps.openstack.utils import heat_utils
class ParameterError(Exception):
@@ -38,20 +38,17 @@ class StorPerfMaster(object):
self.configuration_db = ConfigurationDB()
self.job_db = JobDB()
- template_file = open("storperf/resources/hot/agent-group.yaml")
- self._agent_group_hot = template_file.read()
- template_file = open("storperf/resources/hot/storperf-agent.yaml")
- self._agent_resource_hot = template_file.read()
- self._hot_files = {
- 'storperf-agent.yaml': self._agent_resource_hot
- }
- self.logger.debug(
- "Loaded agent-group template as: " + self._agent_group_hot)
- self.logger.debug(
- "Loaded agent-resource template as: " + self._agent_resource_hot)
-
- self._cinder_client = None
- self._heat_client = None
+ self.stack_settings = StackConfig(
+ name='StorPerfAgent',
+ template_path='storperf/resources/hot/agent-group.yaml')
+
+ self.os_creds = OSCreds(username=os.environ.get('OS_USERNAME'),
+ password=os.environ.get('OS_PASSWORD'),
+ auth_url=os.environ.get('OS_AUTH_URL'),
+ project_name=os.environ.get('OS_PROJECT_NAME'))
+
+ self.heat_stack = OpenStackHeatStack(self.os_creds,
+ self.stack_settings)
self._test_executor = TestExecutor()
self._last_openstack_auth = datetime.now()
@@ -157,16 +154,11 @@ class StorPerfMaster(object):
@property
def stack_id(self):
- return self.configuration_db.get_configuration_value(
- 'stack',
- 'stack_id')
-
- @stack_id.setter
- def stack_id(self, value):
- self.configuration_db.set_configuration_value(
- 'stack',
- 'stack_id',
- value)
+ self.heat_stack.initialize()
+ if self.heat_stack.get_stack() is not None:
+ return self.heat_stack.get_stack().id
+ else:
+ return None
@property
def availability_zone(self):
@@ -183,10 +175,8 @@ class StorPerfMaster(object):
@property
def volume_quota(self):
- self._attach_to_openstack()
- quotas = self._cinder_client.quotas.get(
- os.environ.get('OS_TENANT_ID'))
- return int(quotas.volumes)
+ # (TODO) Use SNAPS equivalent for Volume Quotas
+ pass
@property
def filename(self):
@@ -230,17 +220,8 @@ class StorPerfMaster(object):
@property
def is_stack_created(self):
- if (self.stack_id is not None):
- self._attach_to_openstack()
-
- stack = self._heat_client.stacks.get(self.stack_id)
- status = getattr(stack, 'stack_status')
-
- self.logger.info("Status=" + status)
- if (status == u'CREATE_COMPLETE'):
- return True
-
- return False
+ return (self.stack_id is not None and
+ self.heat_stack.get_status() == u'CREATE_COMPLETE')
@property
def workloads(self):
@@ -305,74 +286,44 @@ class StorPerfMaster(object):
return logs
def create_stack(self):
- if (self.stack_id is not None):
- raise ParameterError("ERROR: Stack has already been created")
-
- self._attach_to_openstack()
- volume_quota = self.volume_quota
- if (volume_quota > 0 and self.agent_count > volume_quota):
- message = "ERROR: Volume quota too low: " + \
- str(self.agent_count) + " > " + str(self.volume_quota)
- raise ParameterError(message)
-
- self.logger.debug("Creating stack")
- stack = self._heat_client.stacks.create(
- stack_name="StorPerfAgentGroup",
- template=self._agent_group_hot,
- files=self._hot_files,
- parameters=self._make_parameters())
-
- self.stack_id = stack['stack']['id']
-
- while True:
- stack = self._heat_client.stacks.get(self.stack_id)
- status = getattr(stack, 'stack_status')
- self.logger.debug("Stack status=%s" % (status,))
- if (status == u'CREATE_COMPLETE'):
- return True
- if (status == u'DELETE_COMPLETE'):
- self.stack_id = None
- return True
- if (status == u'CREATE_FAILED'):
- self.status_reason = getattr(stack, 'stack_status_reason')
- sleep(5)
- self._heat_client.stacks.delete(stack_id=self.stack_id)
- sleep(2)
+ self.stack_settings.resource_files = [
+ 'storperf/resources/hot/storperf-agent.yaml']
+ self.stack_settings.env_values = self._make_parameters()
+ try:
+ self.heat_stack.create()
+ except Exception:
+ heat_cli = heat_utils.heat_client(self.os_creds)
+ res = heat_utils.get_resources(heat_cli,
+ self.heat_stack.get_stack().id)
+ self.logger.error("Stack creation failed")
+ for resource in res:
+ status = resource.status
+ self.logger.error("%s: %s" % (resource.name, status))
+ if status == u'CREATE_FAILED':
+ self.delete_stack()
+ raise Exception(resource.status_reason)
def delete_stack(self):
- if (self.stack_id is None):
- raise ParameterError("ERROR: Stack does not exist")
+ if self._test_executor is not None:
+ self._test_executor.terminate()
- self._attach_to_openstack()
- while True:
- stack = self._heat_client.stacks.get(self.stack_id)
- status = getattr(stack, 'stack_status')
- self.logger.debug("Stack status=%s" % (status,))
- if (status == u'CREATE_COMPLETE'):
- self._heat_client.stacks.delete(stack_id=self.stack_id)
- if (status == u'DELETE_COMPLETE'):
- self.stack_id = None
- return True
- if (status == u'DELETE_FAILED'):
- sleep(5)
- self._heat_client.stacks.delete(stack_id=self.stack_id)
- sleep(2)
+ stack_id = None
+ if (self.stack_id is not None):
+ stack_id = self.stack_id
+ self.heat_stack.clean()
+ return stack_id
def execute_workloads(self, metadata={}):
if (self.stack_id is None):
raise ParameterError("ERROR: Stack does not exist")
- job_list = self.job_db.fetch_jobs()
- for job in job_list:
- report = self.fetch_job_status(job)
- if report['Status'] == 'Running':
- raise "ERROR: Job {} is already running".format(job)
-
- self._attach_to_openstack()
+ if (not self._test_executor.terminated and
+ self._test_executor.job_id is not None):
+ raise Exception("ERROR: Job {} is already running".format(
+ self._test_executor.job_id))
- stack = self._heat_client.stacks.get(self.stack_id)
- outputs = getattr(stack, 'outputs')
- slaves = outputs[0]['output_value']
+ outputs = self.heat_stack.get_outputs()
+ slaves = outputs[0].value
setup_threads = []
@@ -511,57 +462,3 @@ class StorPerfMaster(object):
heat_parameters['agent_flavor'] = self.agent_flavor
heat_parameters['availability_zone'] = self.availability_zone
return heat_parameters
-
- def _attach_to_openstack(self):
-
- time_since_last_auth = datetime.now() - self._last_openstack_auth
-
- if (self._heat_client is None or
- time_since_last_auth.total_seconds() > 600):
- self._last_openstack_auth = datetime.now()
-
- creds = {
- "username": os.environ.get('OS_USERNAME'),
- "password": os.environ.get('OS_PASSWORD'),
- "auth_url": os.environ.get('OS_AUTH_URL'),
- "project_domain_id":
- os.environ.get('OS_PROJECT_DOMAIN_ID'),
- "project_domain_name":
- os.environ.get('OS_PROJECT_DOMAIN_NAME'),
- "project_id": os.environ.get('OS_PROJECT_ID'),
- "project_name": os.environ.get('OS_PROJECT_NAME'),
- "tenant_name": os.environ.get('OS_TENANT_NAME'),
- "tenant_id": os.environ.get("OS_TENANT_ID"),
- "user_domain_id": os.environ.get('OS_USER_DOMAIN_ID'),
- "user_domain_name": os.environ.get('OS_USER_DOMAIN_NAME')
- }
-
- self.logger.debug("Creds: %s" % creds)
-
- loader = loading.get_plugin_loader('password')
- auth = loader.load_from_options(**creds)
-
- https_cacert = os.getenv('OS_CACERT', '')
- https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true'
-
- self.logger.info("cacert=%s" % https_cacert)
-
- sess = session.Session(auth=auth,
- verify=(https_cacert or not https_insecure))
-
- self.logger.debug("Looking up orchestration endpoint")
- heat_endpoint = sess.get_endpoint(auth=auth,
- service_type="orchestration",
- endpoint_type='publicURL')
-
- self.logger.debug("Orchestration endpoint is %s" % heat_endpoint)
-
- self._heat_client = heatclient.Client(
- "1",
- endpoint=heat_endpoint,
- session=sess)
-
- self.logger.debug("Creating cinder client")
- self._cinder_client = cinderclient.Client("2", session=sess,
- cacert=https_cacert)
- self.logger.debug("OpenStack authentication complete")
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 96c2ee4..9ed6386 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -104,6 +104,10 @@ class TestExecutor(object):
def terminated(self):
return self._terminated
+ @property
+ def job_id(self):
+ return self.job_db.job_id
+
@block_sizes.setter
def block_sizes(self, block_sizes):
self.logger.debug("Set block_sizes to: " + str(block_sizes))
diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py
index e824a5f..f328982 100644
--- a/docker/storperf-master/tests/storperf_master_test.py
+++ b/docker/storperf-master/tests/storperf_master_test.py
@@ -10,10 +10,21 @@
import os
import unittest
+import mock
+
from storperf.db.configuration_db import ConfigurationDB
from storperf.storperf_master import StorPerfMaster
+class MockStack(object):
+
+ def __init__(self):
+ pass
+
+ def get_stack(self):
+ return None
+
+
class StorPerfMasterTest(unittest.TestCase):
def setUp(self):
@@ -22,7 +33,12 @@ class StorPerfMasterTest(unittest.TestCase):
os.remove(ConfigurationDB.db_name)
except OSError:
pass
- self.storperf = StorPerfMaster()
+ with mock.patch("storperf.storperf_master.OSCreds"), \
+ mock.patch(
+ "storperf.storperf_master.OpenStackHeatStack") as oshs:
+ oshs.return_value.get_stack.return_value = None
+
+ self.storperf = StorPerfMaster()
def tearDown(self):
try: