diff options
| -rwxr-xr-x | ci/verify-build.sh | 9 | ||||
| -rwxr-xr-x | ci/verify.sh | 21 | ||||
| -rw-r--r-- | docker/storperf-master/requirements.pip | 2 | ||||
| -rw-r--r-- | docker/storperf-master/rest_server.py | 4 | ||||
| -rw-r--r-- | docker/storperf-master/storperf/storperf_master.py | 209 | ||||
| -rw-r--r-- | docker/storperf-master/storperf/test_executor.py | 4 | ||||
| -rw-r--r-- | docker/storperf-master/tests/storperf_master_test.py | 18 |
7 files changed, 87 insertions, 180 deletions
diff --git a/ci/verify-build.sh b/ci/verify-build.sh index 0477e17..c98fea4 100755 --- a/ci/verify-build.sh +++ b/ci/verify-build.sh @@ -19,7 +19,12 @@ export CARBON_DIR=${ci}/job/carbon/ ${ci}/remove_docker_container.sh mkdir -p ${CARBON_DIR} -touch ${ENV_FILE} +cat << EOF > ${ENV_FILE} +OS_USERNAME=username +OS_PASSWORD=password +OS_AUTH_URL=http://localhost +OS_PROJECT_NAME=project +EOF if [ -z $ARCH ] then @@ -60,7 +65,7 @@ set +e check_for_life storperf-httpfrontend "/" FAILURES=$((FAILURES + $?)) -check_for_life storperf-master "/api/v1.0/configurations" +check_for_life storperf-master "/api/v1.0/jobs" FAILURES=$((FAILURES + $?)) check_for_life storperf-reporting "/reporting/" diff --git a/ci/verify.sh b/ci/verify.sh index cd5bbbb..996a3af 100755 --- a/ci/verify.sh +++ b/ci/verify.sh @@ -22,28 +22,11 @@ pip install --upgrade setuptools==33.1.1 pip install autoflake==0.6.6 pip install autopep8==1.2.2 pip install coverage==4.1 -pip install flask==0.10 -pip install flask_cors==3.0.2 -pip install flask-restful==0.3.5 -pip install flask-restful-swagger==0.19 -pip install flask-swagger==0.2.12 -pip install funcsigs==0.4 +pip install cryptography==1.7.2 pip install flake8==2.5.4 -pip install html2text==2016.1.8 -pip install keystoneauth1==2.12.1 pip install mock==1.3.0 pip install nose==1.3.7 -pip install paramiko==2.0.2 -pip install python-cinderclient==1.6.0 -pip install python-glanceclient==1.1.0 -pip install python-heatclient==0.8.0 -pip install python-keystoneclient==1.6.0 -pip install python-neutronclient==2.6.0 -pip install python-novaclient==2.28.1 -pip install pyyaml==3.10 -pip install requests==2.13.0 -pip install scp==0.10.2 -pip install six==1.10.0 +pip install -r docker/storperf-master/requirements.pip final_rc=0 diff --git a/docker/storperf-master/requirements.pip b/docker/storperf-master/requirements.pip index 5eb31cc..a591e84 100644 --- a/docker/storperf-master/requirements.pip +++ b/docker/storperf-master/requirements.pip @@ -8,4 +8,4 @@ html2text==2016.1.8 paramiko==2.0.2 requests==2.13.0 scp==0.10.2 -snaps==6.0.0.dev309 +git+https://gerrit.opnfv.org/gerrit/snaps#egg=snaps diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 9280721..67d2d05 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -199,8 +199,9 @@ class Configure(Resource): ) def delete(self): try: - storperf.delete_stack() + return jsonify({'stack_id': storperf.delete_stack()}) except Exception as e: + self.logger.exception(e) abort(400, str(e)) @@ -355,6 +356,7 @@ for any single test iteration. return jsonify({'job_id': job_id}) except Exception as e: + self.logger.exception(e) abort(400, str(e)) @swagger.operation( diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index ad6e16c..45d5d89 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -14,16 +14,16 @@ import socket from threading import Thread from time import sleep -from cinderclient import client as cinderclient -from keystoneauth1 import loading -from keystoneauth1 import session import paramiko from scp import SCPClient +from snaps.config.stack import StackConfig +from snaps.openstack.create_stack import OpenStackHeatStack +from snaps.openstack.os_credentials import OSCreds -import heatclient.client as heatclient from storperf.db.configuration_db import ConfigurationDB from storperf.db.job_db import JobDB from storperf.test_executor import TestExecutor +from snaps.openstack.utils import heat_utils class ParameterError(Exception): @@ -38,20 +38,17 @@ class StorPerfMaster(object): self.configuration_db = ConfigurationDB() self.job_db = JobDB() - template_file = open("storperf/resources/hot/agent-group.yaml") - self._agent_group_hot = template_file.read() - template_file = open("storperf/resources/hot/storperf-agent.yaml") - self._agent_resource_hot = template_file.read() - self._hot_files = { - 'storperf-agent.yaml': self._agent_resource_hot - } - self.logger.debug( - "Loaded agent-group template as: " + self._agent_group_hot) - self.logger.debug( - "Loaded agent-resource template as: " + self._agent_resource_hot) - - self._cinder_client = None - self._heat_client = None + self.stack_settings = StackConfig( + name='StorPerfAgent', + template_path='storperf/resources/hot/agent-group.yaml') + + self.os_creds = OSCreds(username=os.environ.get('OS_USERNAME'), + password=os.environ.get('OS_PASSWORD'), + auth_url=os.environ.get('OS_AUTH_URL'), + project_name=os.environ.get('OS_PROJECT_NAME')) + + self.heat_stack = OpenStackHeatStack(self.os_creds, + self.stack_settings) self._test_executor = TestExecutor() self._last_openstack_auth = datetime.now() @@ -157,16 +154,11 @@ class StorPerfMaster(object): @property def stack_id(self): - return self.configuration_db.get_configuration_value( - 'stack', - 'stack_id') - - @stack_id.setter - def stack_id(self, value): - self.configuration_db.set_configuration_value( - 'stack', - 'stack_id', - value) + self.heat_stack.initialize() + if self.heat_stack.get_stack() is not None: + return self.heat_stack.get_stack().id + else: + return None @property def availability_zone(self): @@ -183,10 +175,8 @@ class StorPerfMaster(object): @property def volume_quota(self): - self._attach_to_openstack() - quotas = self._cinder_client.quotas.get( - os.environ.get('OS_TENANT_ID')) - return int(quotas.volumes) + # (TODO) Use SNAPS equivalent for Volume Quotas + pass @property def filename(self): @@ -230,17 +220,8 @@ class StorPerfMaster(object): @property def is_stack_created(self): - if (self.stack_id is not None): - self._attach_to_openstack() - - stack = self._heat_client.stacks.get(self.stack_id) - status = getattr(stack, 'stack_status') - - self.logger.info("Status=" + status) - if (status == u'CREATE_COMPLETE'): - return True - - return False + return (self.stack_id is not None and + self.heat_stack.get_status() == u'CREATE_COMPLETE') @property def workloads(self): @@ -305,74 +286,44 @@ class StorPerfMaster(object): return logs def create_stack(self): - if (self.stack_id is not None): - raise ParameterError("ERROR: Stack has already been created") - - self._attach_to_openstack() - volume_quota = self.volume_quota - if (volume_quota > 0 and self.agent_count > volume_quota): - message = "ERROR: Volume quota too low: " + \ - str(self.agent_count) + " > " + str(self.volume_quota) - raise ParameterError(message) - - self.logger.debug("Creating stack") - stack = self._heat_client.stacks.create( - stack_name="StorPerfAgentGroup", - template=self._agent_group_hot, - files=self._hot_files, - parameters=self._make_parameters()) - - self.stack_id = stack['stack']['id'] - - while True: - stack = self._heat_client.stacks.get(self.stack_id) - status = getattr(stack, 'stack_status') - self.logger.debug("Stack status=%s" % (status,)) - if (status == u'CREATE_COMPLETE'): - return True - if (status == u'DELETE_COMPLETE'): - self.stack_id = None - return True - if (status == u'CREATE_FAILED'): - self.status_reason = getattr(stack, 'stack_status_reason') - sleep(5) - self._heat_client.stacks.delete(stack_id=self.stack_id) - sleep(2) + self.stack_settings.resource_files = [ + 'storperf/resources/hot/storperf-agent.yaml'] + self.stack_settings.env_values = self._make_parameters() + try: + self.heat_stack.create() + except Exception: + heat_cli = heat_utils.heat_client(self.os_creds) + res = heat_utils.get_resources(heat_cli, + self.heat_stack.get_stack().id) + self.logger.error("Stack creation failed") + for resource in res: + status = resource.status + self.logger.error("%s: %s" % (resource.name, status)) + if status == u'CREATE_FAILED': + self.delete_stack() + raise Exception(resource.status_reason) def delete_stack(self): - if (self.stack_id is None): - raise ParameterError("ERROR: Stack does not exist") + if self._test_executor is not None: + self._test_executor.terminate() - self._attach_to_openstack() - while True: - stack = self._heat_client.stacks.get(self.stack_id) - status = getattr(stack, 'stack_status') - self.logger.debug("Stack status=%s" % (status,)) - if (status == u'CREATE_COMPLETE'): - self._heat_client.stacks.delete(stack_id=self.stack_id) - if (status == u'DELETE_COMPLETE'): - self.stack_id = None - return True - if (status == u'DELETE_FAILED'): - sleep(5) - self._heat_client.stacks.delete(stack_id=self.stack_id) - sleep(2) + stack_id = None + if (self.stack_id is not None): + stack_id = self.stack_id + self.heat_stack.clean() + return stack_id def execute_workloads(self, metadata={}): if (self.stack_id is None): raise ParameterError("ERROR: Stack does not exist") - job_list = self.job_db.fetch_jobs() - for job in job_list: - report = self.fetch_job_status(job) - if report['Status'] == 'Running': - raise "ERROR: Job {} is already running".format(job) - - self._attach_to_openstack() + if (not self._test_executor.terminated and + self._test_executor.job_id is not None): + raise Exception("ERROR: Job {} is already running".format( + self._test_executor.job_id)) - stack = self._heat_client.stacks.get(self.stack_id) - outputs = getattr(stack, 'outputs') - slaves = outputs[0]['output_value'] + outputs = self.heat_stack.get_outputs() + slaves = outputs[0].value setup_threads = [] @@ -511,57 +462,3 @@ class StorPerfMaster(object): heat_parameters['agent_flavor'] = self.agent_flavor heat_parameters['availability_zone'] = self.availability_zone return heat_parameters - - def _attach_to_openstack(self): - - time_since_last_auth = datetime.now() - self._last_openstack_auth - - if (self._heat_client is None or - time_since_last_auth.total_seconds() > 600): - self._last_openstack_auth = datetime.now() - - creds = { - "username": os.environ.get('OS_USERNAME'), - "password": os.environ.get('OS_PASSWORD'), - "auth_url": os.environ.get('OS_AUTH_URL'), - "project_domain_id": - os.environ.get('OS_PROJECT_DOMAIN_ID'), - "project_domain_name": - os.environ.get('OS_PROJECT_DOMAIN_NAME'), - "project_id": os.environ.get('OS_PROJECT_ID'), - "project_name": os.environ.get('OS_PROJECT_NAME'), - "tenant_name": os.environ.get('OS_TENANT_NAME'), - "tenant_id": os.environ.get("OS_TENANT_ID"), - "user_domain_id": os.environ.get('OS_USER_DOMAIN_ID'), - "user_domain_name": os.environ.get('OS_USER_DOMAIN_NAME') - } - - self.logger.debug("Creds: %s" % creds) - - loader = loading.get_plugin_loader('password') - auth = loader.load_from_options(**creds) - - https_cacert = os.getenv('OS_CACERT', '') - https_insecure = os.getenv('OS_INSECURE', '').lower() == 'true' - - self.logger.info("cacert=%s" % https_cacert) - - sess = session.Session(auth=auth, - verify=(https_cacert or not https_insecure)) - - self.logger.debug("Looking up orchestration endpoint") - heat_endpoint = sess.get_endpoint(auth=auth, - service_type="orchestration", - endpoint_type='publicURL') - - self.logger.debug("Orchestration endpoint is %s" % heat_endpoint) - - self._heat_client = heatclient.Client( - "1", - endpoint=heat_endpoint, - session=sess) - - self.logger.debug("Creating cinder client") - self._cinder_client = cinderclient.Client("2", session=sess, - cacert=https_cacert) - self.logger.debug("OpenStack authentication complete") diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index 96c2ee4..9ed6386 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -104,6 +104,10 @@ class TestExecutor(object): def terminated(self): return self._terminated + @property + def job_id(self): + return self.job_db.job_id + @block_sizes.setter def block_sizes(self, block_sizes): self.logger.debug("Set block_sizes to: " + str(block_sizes)) diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py index e824a5f..f328982 100644 --- a/docker/storperf-master/tests/storperf_master_test.py +++ b/docker/storperf-master/tests/storperf_master_test.py @@ -10,10 +10,21 @@ import os import unittest +import mock + from storperf.db.configuration_db import ConfigurationDB from storperf.storperf_master import StorPerfMaster +class MockStack(object): + + def __init__(self): + pass + + def get_stack(self): + return None + + class StorPerfMasterTest(unittest.TestCase): def setUp(self): @@ -22,7 +33,12 @@ class StorPerfMasterTest(unittest.TestCase): os.remove(ConfigurationDB.db_name) except OSError: pass - self.storperf = StorPerfMaster() + with mock.patch("storperf.storperf_master.OSCreds"), \ + mock.patch( + "storperf.storperf_master.OpenStackHeatStack") as oshs: + oshs.return_value.get_stack.return_value = None + + self.storperf = StorPerfMaster() def tearDown(self): try: |