diff options
Diffstat (limited to 'func')
| -rw-r--r-- | func/args_handler.py | 18 | ||||
| -rw-r--r-- | func/cli.py | 28 | ||||
| -rw-r--r-- | func/env_setup.py | 6 | ||||
| -rw-r--r-- | func/spawn_vm.py | 207 |
4 files changed, 114 insertions, 145 deletions
diff --git a/func/args_handler.py b/func/args_handler.py index 59712800..624f90c4 100644 --- a/func/args_handler.py +++ b/func/args_handler.py @@ -14,8 +14,8 @@ from func.spawn_vm import SpawnVM from func.driver import Driver -def get_files_in_test_list(suite_name, case_type='all'): - benchmark_list = json.load(file('test_list/{0}'.format(suite_name))) +def get_files_in_suite(suite_name, case_type='all'): + benchmark_list = json.load(file('benchmarks/suite/{0}'.format(suite_name))) return reduce(add, benchmark_list.values()) \ if case_type == 'all' else benchmark_list[case_type] @@ -30,8 +30,8 @@ def get_benchmark_path(lab, suit, benchmark): return './test_cases/{0}/{1}/{2}'.format(lab, suit, benchmark) -def check_suite_in_test_list(suite_name): - return True if os.path.isfile('test_list/' + suite_name) else False +def check_suite(suite_name): + return True if os.path.isfile('benchmarks/suite/' + suite_name) else False def check_lab_name(lab_name): @@ -59,10 +59,12 @@ def prepare_ansible_env(benchmark_test_case): def run_benchmark(installer_type, pwd, benchmark, benchmark_details, proxy_info, env_setup, benchmark_test_case): driver = Driver() - return driver.drive_bench(installer_type, pwd, benchmark, - env_setup.roles_dict.items(), - _get_f_name(benchmark_test_case), - benchmark_details, env_setup.ip_pw_dict.items(), proxy_info) + result = driver.drive_bench(installer_type, pwd, benchmark, + env_setup.roles_dict.items(), + _get_f_name(benchmark_test_case), + benchmark_details, env_setup.ip_pw_dict.items(), proxy_info) + env_setup.cleanup_authorized_keys() + return result def prepare_and_run_benchmark(installer_type, pwd, benchmark_test_case): diff --git a/func/cli.py b/func/cli.py index d914a2de..c5f5d2b7 100644 --- a/func/cli.py +++ b/func/cli.py @@ -11,6 +11,9 @@ import sys import os import args_handler import argparse +from utils import logger_utils + +logger = logger_utils.QtipLogger('cli').get class Cli: @@ -26,40 +29,41 @@ class Cli: ' The user should list default after -l . all the fields in' ' the files are necessary and should be filled') parser.add_argument('-f', '--file', required=True, help='File in ' - 'test_list with the list of tests. there are three files' + 'benchmarks/suite/ with the list of tests. there are three files' '\n compute ' '\n storage ' '\n network ' 'They contain all the tests that will be run. They are listed by suite.' 'Please ensure there are no empty lines') parser.add_argument('-b', '--benchmark', help='Name of the benchmark.' - 'Can be found in test_lists/file_name') + 'Can be found in benchmarks/suite/file_name') return parser.parse_args(args) def __init__(self, args=sys.argv[1:]): args = self._parse_args(args) - if not args_handler.check_suite_in_test_list(args.file): - print('\n\n ERROR: Test File Does not exist in test_list/ please enter correct file \n\n') + if not args_handler.check_suite(args.file): + logger.error("ERROR: This suite file doesn't exist under benchmarks/suite/.\ + Please enter correct file." % str(args.file)) sys.exit(1) if not args_handler.check_lab_name(args.lab): - print('\n\n You have specified a lab that is not present in test_cases/ please enter \ - correct file. If unsure how to proceed, use -l default.\n\n') + logger.error("You have specified a lab that is not present under test_cases/.\ + Please enter correct file. If unsure how to proceed, use -l default.") sys.exit(1) suite = args.file - benchmarks = args_handler.get_files_in_test_list(suite) + benchmarks = args_handler.get_files_in_suite(suite) test_cases = args_handler.get_files_in_test_case(args.lab, suite) benchmarks_list = filter(lambda x: x in test_cases, benchmarks) if args.benchmark: if not args_handler.check_benchmark_name(args.lab, args.file, args.benchmark): - print('\n\n You have specified an incorrect benchmark. Please' - 'enter the correct one.\n\n') + logger.error("You have specified an incorrect benchmark.\ + Please enter the correct one.") sys.exit(1) else: - print("Starting with " + args.benchmark) + logger.info("Starting with " + args.benchmark) args_handler.prepare_and_run_benchmark( os.environ['INSTALLER_TYPE'], os.environ['PWD'], args_handler.get_benchmark_path(args.lab.lower(), args.file, args.benchmark)) @@ -68,5 +72,5 @@ class Cli: os.environ['INSTALLER_TYPE'], os.environ['PWD'], args_handler.get_benchmark_path(args.lab.lower(), suite, x)), benchmarks_list) - print('{0} is not a Template in the Directory Enter a Valid file name.' - 'or use qtip.py -h for list'.format(filter(lambda x: x not in test_cases, benchmarks))) + logger.info("{0} is not a Template in the Directory Enter a Valid file name.\ + or use qtip.py -h for list".format(filter(lambda x: x not in test_cases, benchmarks))) diff --git a/func/env_setup.py b/func/env_setup.py index 9e21a5b6..6027f904 100644 --- a/func/env_setup.py +++ b/func/env_setup.py @@ -208,3 +208,9 @@ class Env_setup: def call_ssh_test(self): self.ssh_test(self.ip_pw_list) + + def cleanup_authorized_keys(self): + for ip, pw in self.ip_pw_list: + cmd = './scripts/cleanup_creds.sh %s' % ip + logger.info("cleanup authorized_keys: %s " % cmd) + os.system(cmd) diff --git a/func/spawn_vm.py b/func/spawn_vm.py index 3a16e02d..0a24d7a4 100644 --- a/func/spawn_vm.py +++ b/func/spawn_vm.py @@ -1,169 +1,138 @@ ##############################################################################
-# Copyright (c) 2015 Dell Inc and others.
+# Copyright (c) 2016 Dell Inc, ZTE and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-
import os
import sys
-from collections import defaultdict
-from func.env_setup import Env_setup
import yaml
import heatclient.client
import keystoneclient
-from novaclient import client
import time
+from func.env_setup import Env_setup
from func.create_zones import AvailabilityZone
+from utils import logger_utils
+
+logger = logger_utils.QtipLogger('spawn_vm').get
class SpawnVM(Env_setup):
- vm_role_ip_dict = defaultdict(list)
- installer = ''
def __init__(self, vm_info):
- print 'SpawnVM Class initiated'
- print 'vm_info: %s' % vm_info
+ logger.info('vm_info: %s' % vm_info)
vm_role_ip_dict = vm_info.copy()
- print 'Generating Heat Template\n'
self._keystone_client = None
self._heat_client = None
self._glance_client = None
self._nova_client = None
- self. _get_nova_client()
self.azone = AvailabilityZone()
# TODO: it should clean up aggregates and stack after test case finished.
self.azone.clean_all_aggregates()
self.azone.create_aggs(vm_info['availability_zone'])
- installer = self.get_installer_type()
- self.Heat_template1 = self.heat_template_vm(vm_info, installer)
- self.create_stack(vm_role_ip_dict, self.Heat_template1)
+ self.heat_template = self.generate_heat_template(vm_info)
+ self.create_stack(vm_role_ip_dict)
@staticmethod
- def get_installer_type():
- print 'Getting Installer Name'
- return os.environ['INSTALLER_TYPE']
-
- @staticmethod
- def get_public_network(installer_detected):
+ def get_public_network():
"""
TODO: GET THE NAMES OF THE PUBLIC NETWORKS for OTHER PROJECTS
"""
- print 'Getting Public Network'
- if installer_detected.lower() == 'fuel':
+ installer = os.environ['INSTALLER_TYPE']
+
+ if installer.lower() == 'fuel':
return 'admin_floating_net'
- if installer_detected.lower() == 'apex':
+ if installer.lower() == 'apex':
return 'external'
- if installer_detected.lower() == 'compass':
+ if installer.lower() == 'compass':
return 'ext-net'
- if installer_detected.lower() == 'joid':
+ if installer.lower() == 'joid':
return 'ext-net'
- def heat_template_vm(self, vm_params, installer):
- Heat_Dic = {}
+ def generate_heat_template(self, vm_params):
+ logger.info('Generating Heat Template')
+ heat_dict = {}
try:
with open('./config/SampleHeat.yaml', 'r+') as H_temp:
- Heat_Dic = yaml.safe_load(H_temp)
+ heat_dict = yaml.safe_load(H_temp)
except yaml.YAMLError as exc:
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark
- print 'Error in qtip/config/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1, mark.column + 1)
- print 'EXITING PROGRAM. Correct File and restart'
+ logger.error(
+ 'Error in qtip/config/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1,
+ mark.column + 1))
+ logger.error('EXITING PROGRAM. Correct File and restart')
sys.exit(1)
+
fopen = open('./config/QtipKey.pub', 'r')
fopenstr = fopen.read()
fopenstr = fopenstr.rstrip()
scriptcmd = '#!/bin/bash \n echo {0} >> foo.txt \n echo {1} >> /root/.ssh/authorized_keys'.format(
fopenstr, fopenstr)
- netName = self.get_public_network(installer)
- print netName
- Heat_Dic['heat_template_version'] = '2014-10-16'
- Heat_Dic['resources']['KeyPairSavePrivate'] = {
- 'type': 'OS::Nova::KeyPair',
- 'properties': {
- 'save_private_key': 'true',
- 'name': 'my_key'
- }
- }
- Heat_Dic['parameters']['public_network'] = {
+ netName = self.get_public_network()
+ heat_dict['heat_template_version'] = '2015-04-30'
+
+ heat_dict['parameters']['public_network'] = {
'type': 'string',
'default': netName
}
+
for x in range(1, len(vm_params['availability_zone']) + 1):
avail_zone = vm_params['availability_zone'][x - 1]
- img = vm_params['OS_image'][x - 1]
- flavor = vm_params['flavor'][x - 1]
- Heat_Dic['parameters']['availability_zone_' + str(x)] = \
+ heat_dict['parameters']['availability_zone_' + str(x)] = \
{'description': 'Availability Zone of the instance',
'default': avail_zone,
'type': 'string'}
- Heat_Dic['resources']['public_port_' + str(x)] = \
+ heat_dict['resources']['public_port_' + str(x)] = \
{'type': 'OS::Neutron::Port',
- 'properties': {'network': {'get_resource': 'private_network'},
- 'security_groups': [{'get_resource': 'demo1_security_Group'}],
- 'fixed_ips': [{'subnet_id':
- {'get_resource': 'private_subnet'}}]}}
+ 'properties': {'network': {'get_resource': 'network'},
+ 'security_groups': [{'get_resource': 'security_group'}],
+ 'fixed_ips': [{'subnet_id': {'get_resource': 'subnet'}}]}}
- Heat_Dic['resources']['floating_ip_' + str(x)] = {
+ heat_dict['resources']['floating_ip_' + str(x)] = {
'type': 'OS::Neutron::FloatingIP',
- 'properties': {
- 'floating_network': {'get_param': 'public_network'}}}
+ 'properties': {'floating_network': {'get_param': 'external_net_name'}}}
- Heat_Dic['resources']['floating_ip_assoc_' + str(x)] = {
+ heat_dict['resources']['floating_ip_assoc_' + str(x)] = {
'type': 'OS::Neutron::FloatingIPAssociation',
'properties': {
'floatingip_id': {'get_resource': 'floating_ip_' + str(x)},
'port_id': {'get_resource': 'public_port_' + str(x)}}}
- Heat_Dic['resources']['my_instance_' + str(x)] = \
+ heat_dict['resources']['my_instance_' + str(x)] = \
{'type': 'OS::Nova::Server',
- 'properties': {'image': img,
+ 'properties': {'image': {'get_param': 'image'},
'networks':
[{'port': {'get_resource': 'public_port_' + str(x)}}],
- 'flavor': flavor,
+ 'flavor': {'get_resource': 'flavor'},
'availability_zone': avail_zone,
+ 'security_groups': [{'get_resource': 'security_group'}],
'name': 'instance' + str(x),
- 'key_name': {'get_resource': 'KeyPairSavePrivate'},
'user_data_format': 'RAW',
'user_data': scriptcmd}}
- Heat_Dic['resources']['demo1_security_Group'] = {
- 'type': 'OS::Neutron::SecurityGroup',
- 'properties': {
- 'name': 'demo1_security_Group',
- 'rules': [{
- 'protocol': 'tcp',
- 'port_range_min': 22,
- 'port_range_max': 5201},
- {'protocol': 'udp',
- 'port_range_min': 22,
- 'port_range_max': 5201},
- {'protocol': 'icmp'}]}}
-
- Heat_Dic['outputs']['instance_PIP_' + str(x)] = {
+ heat_dict['outputs']['instance_PIP_' + str(x)] = {
'description': 'IP address of the instance',
'value': {'get_attr': ['my_instance_' + str(x), 'first_address']}}
- Heat_Dic['outputs']['instance_ip_' + str(x)] = {
+
+ heat_dict['outputs']['instance_ip_' + str(x)] = {
'description': 'IP address of the instance',
'value': {'get_attr': ['floating_ip_' + str(x), 'floating_ip_address']}}
- Heat_Dic['outputs']['availability_instance_' + str(x)] = {
+ heat_dict['outputs']['availability_instance_' + str(x)] = {
'description': 'Availability Zone of the Instance',
'value': {'get_param': 'availability_zone_' + str(x)}}
- Heat_Dic['outputs']['KeyPair_PublicKey'] = {
- 'description': 'Private Key',
- 'value': {'get_attr': ['KeyPairSavePrivate', 'private_key']}
- }
- del Heat_Dic['outputs']['description']
- print Heat_Dic
- return Heat_Dic
+ del heat_dict['outputs']['description']
+ logger.info(heat_dict)
+
+ return heat_dict
def _get_keystone_client(self):
"""returns a keystone client instance"""
@@ -176,12 +145,6 @@ class SpawnVM(Env_setup): tenant_name=os.environ.get('OS_TENANT_NAME'))
return self._keystone_client
- def _get_nova_client(self):
- if self._nova_client is None:
- keystone = self._get_keystone_client()
- self._nova_client = client.Client('2', token=keystone.auth_token)
- return self._nova_client
-
def _get_heat_client(self):
"""returns a heat client instance"""
if self._heat_client is None:
@@ -192,45 +155,29 @@ class SpawnVM(Env_setup): '1', endpoint=heat_endpoint, token=keystone.auth_token)
return self._heat_client
- def create_stack(self, vm_role_ip_dict, heat_template):
-
- global sshkey
+ def create_stack(self, vm_role_ip_dict):
stackname = 'QTIP'
heat = self._get_heat_client()
- for checks in range(3):
- print "Try to delete heats %s" % checks
- for prev_stacks in heat.stacks.list():
- if prev_stacks.stack_name == 'QTIP':
- print 'QTIP Stacks exists.\nDeleting Existing Stack'
- heat.stacks.delete('QTIP')
- time.sleep(10)
- print '\nStack Creating Started\n'
+ self.delete_stack(stackname)
- try:
- heat.stacks.create(stack_name=stackname, template=heat_template)
- except Exception:
- print 'Create Failed :( '
-
- cluster_detail = heat.stacks.get(stackname)
- while cluster_detail.status != 'COMPLETE':
- if cluster_detail.status == 'IN_PROGRESS':
- print 'Stack Creation in Progress'
- cluster_detail = heat.stacks.get(stackname)
- time.sleep(10)
- print 'Stack Created'
- print 'Getting Public IP(s)'
- zone = []
- s = 0
- for vm in range(len(vm_role_ip_dict['OS_image'])):
+ logger.info('Start to create stack %s' % stackname)
+ heat.stacks.create(stack_name=stackname, template=self.heat_template)
+
+ stack_status = "IN_PROGRESS"
+ while stack_status != 'COMPLETE':
+ if stack_status == 'IN_PROGRESS':
+ logger.debug('Create in Progress')
+ if stack_status == 'CREATE_FAILED':
+ raise RuntimeError("Stack %s created failed!" % stackname)
+ stack_status = heat.stacks.get(stackname).status
+ time.sleep(15)
+ logger.info('Stack %s Created Complete!' % stackname)
- for I in cluster_detail.outputs:
- availabilityKey = 'availability_instance_' + str(vm + 1)
+ stack_outputs = heat.stacks.get(stackname).outputs
- if I['output_key'] == availabilityKey:
- zone.insert(s, str(I['output_value']))
- s = s + 1
- for i in cluster_detail.outputs:
+ for vm in range(len(vm_role_ip_dict['OS_image'])):
+ for i in stack_outputs:
instanceKey = "instance_ip_" + str(vm + 1)
privateIPkey = 'instance_PIP_' + str(vm + 1)
if i['output_key'] == instanceKey:
@@ -240,10 +187,20 @@ class SpawnVM(Env_setup): if i['output_key'] == privateIPkey:
Env_setup.ip_pw_dict[vm_role_ip_dict['role'][vm]] = str(i['output_value'])
- if i['output_key'] == 'KeyPair_PublicKey':
- sshkey = str(i['output_value'])
- with open('./config/my_key.pem', 'w') as fopen:
- fopen.write(sshkey)
- fopen.close()
- print Env_setup.ip_pw_list
+ logger.info('Getting Public IP(s): %s' % Env_setup.ip_pw_list)
+
+ def delete_stack(self, stack_name):
+ heat = self._get_heat_client()
+
+ stacks = heat.stacks.list()
+ exists = map(lambda x: x.stack_name, stacks)
+ if stack_name in exists:
+ logger.info("Delete stack %s" % stack_name)
+ heat.stacks.delete(stack_name)
+ while stack_name in exists:
+ time.sleep(10)
+ stacks = heat.stacks.list()
+ exists = map(lambda x: x.stack_name, stacks)
+ logger.debug("exists_stacks: %s" % exists)
+ logger.info("%s doesn't exist" % stack_name)
|