diff options
Diffstat (limited to 'func')
| -rw-r--r-- | func/cli.py | 121 | ||||
| -rw-r--r-- | func/create_zones.py | 257 | ||||
| -rw-r--r-- | func/driver.py | 38 | ||||
| -rw-r--r-- | func/env_setup.py | 132 | ||||
| -rw-r--r-- | func/fetchimg.py | 65 | ||||
| -rw-r--r-- | func/spawn_vm.py | 561 | ||||
| -rw-r--r-- | func/validate_yaml.py | 28 |
7 files changed, 588 insertions, 614 deletions
diff --git a/func/cli.py b/func/cli.py index 129ab96c..5e8f02cf 100644 --- a/func/cli.py +++ b/func/cli.py @@ -15,94 +15,95 @@ from func.spawn_vm import SpawnVM import argparse -class cli(): - - def _getfile(self, filepath): - - with open('test_list/'+filepath,'r') as finput: - _benchmarks=finput.readlines() - for items in range( len(_benchmarks)): - _benchmarks[items]=_benchmarks[items].rstrip() +class cli: + + @staticmethod + def _getfile(file_path): + with open('test_list/' + file_path, 'r') as fin_put: + _benchmarks = fin_put.readlines() + for items in range(len(_benchmarks)): + _benchmarks[items] = _benchmarks[items].rstrip() return _benchmarks - def _getsuite(self, filepath): + @staticmethod + def _getsuite(file_path): - return filepath + return file_path - def _checkTestList(self, filename): + @staticmethod + def _check_test_list(filename): - if os.path.isfile('test_list/'+filename): + if os.path.isfile('test_list/' + filename): return True else: return False - def _checkLabName(self, labname): + @staticmethod + def _check_lab_name(lab_name): - if os.path.isdir('test_cases/'+labname): + if os.path.isdir('test_cases/' + lab_name): return True else: return False - def _get_fname(self,file_name): + @staticmethod + def _get_f_name(file_name): return file_name[0: file_name.find('.')] - def __init__(self): - - suite=[] + @staticmethod + def _parse_args(args): parser = argparse.ArgumentParser() - parser.add_argument('-l ', '--lab', help='Name of Lab on which being tested, These can' \ - 'be found in the test_cases/ directory. Please ' \ - 'ensure that you have edited the respective files '\ - 'before using them. For testing other than through Jenkins'\ - ' The user should list default after -l . all the fields in'\ - ' the files are necessary and should be filled') - parser.add_argument('-f', '--file', help = 'File in test_list with the list of tests. there are three files' \ - '\n compute '\ - '\n storage '\ - '\n network '\ - 'They contain all the tests that will be run. They are listed by suite.' \ - 'Please ensure there are no empty lines') - args = parser.parse_args() - - if not self._checkTestList(args.file): + parser.add_argument('-l ', '--lab', help='Name of Lab on which being tested, These can' + 'be found in the test_cases/ directory. Please ' + 'ensure that you have edited the respective files ' + 'before using them. For testing other than through Jenkins' + ' The user should list default after -l . all the fields in' + ' the files are necessary and should be filled') + parser.add_argument('-f', '--file', help='File in test_list with the list of tests. there are three files' + '\n compute ' + '\n storage ' + '\n network ' + 'They contain all the tests that will be run. They are listed by suite.' + 'Please ensure there are no empty lines') + return parser.parse_args(args) + + def __init__(self, args=sys.argv[1:]): + + suite = [] + args = self._parse_args(args) + + if not self._check_test_list(args.file): print '\n\n ERROR: Test File Does not exist in test_list/ please enter correct file \n\n' sys.exit(0) - if not self._checkLabName(args.lab): - print '\n\n You have specified a lab that is not present in test_cases/ please enter correct'\ - ' file. If unsure how to proceed, use -l default.\n\n' + if not self._check_lab_name(args.lab): + print '\n\n You have specified a lab that is not present in test_cases/ please enter correct \ + file. If unsure how to proceed, use -l default.\n\n' sys.exit(0) benchmarks = self._getfile(args.file) suite.append(args.file) - suite=self._getsuite(suite) - for items in range (len(benchmarks)): - if (suite and benchmarks): - - roles='' - vm_info='' - benchmark_details='' - pip='' - obj='' + suite = self._getsuite(suite) + for items in range(len(benchmarks)): + if suite and benchmarks: obj = Env_setup() - if os.path.isfile('./test_cases/'+args.lab.lower()+'/'+suite[0]+'/' +benchmarks[items]): - [benchmark, roles, vm_info, benchmark_details, pip, proxy_info] = obj.parse('./test_cases/' - +args.lab.lower()+'/'+suite[0]+'/'+benchmarks[items]) + if os.path.isfile('./test_cases/' + args.lab.lower() + '/' + suite[0] + '/' + benchmarks[items]): + [benchmark, vm_info, benchmark_details, proxy_info] = \ + obj.parse('./test_cases/' + args.lab.lower() + '/' + suite[0] + '/' + benchmarks[items]) if len(vm_info) != 0: - vmObj ='' - vmObj = SpawnVM(vm_info) - if obj.callpingtest(): - obj.callsshtest() - obj.updateAnsible() - dvr = Driver() - dvr.drive_bench(benchmark, - obj.roles_dict.items(), - self._get_fname(benchmarks[items]), - benchmark_details, - obj.ip_pw_dict.items(), - proxy_info) + SpawnVM(vm_info) + obj.call_ping_test() + obj.call_ssh_test() + obj.update_ansible() + dvr = Driver() + dvr.drive_bench(benchmark, + obj.roles_dict.items(), + self._get_f_name(benchmarks[items]), + benchmark_details, + obj.ip_pw_dict.items(), + proxy_info) else: print (benchmarks[items], ' is not a Template in the Directory - \ Enter a Valid file name. or use qtip.py -h for list') diff --git a/func/create_zones.py b/func/create_zones.py index 44ba7568..e715dfd4 100644 --- a/func/create_zones.py +++ b/func/create_zones.py @@ -1,127 +1,130 @@ -############################################################################## -# Copyright (c) 2015 Dell Inc and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - - -from keystoneclient.auth.identity import v2 -from keystoneclient import session -from novaclient import client -import os -import re -from collections import defaultdict - - -class create_zones: - - def __init__(self): - print 'Creating Zones' - self._keystone_client = None - self._nova_client = None - - def _get_keystone_client(self): - '''returns a keystone client instance''' - - if self._keystone_client is None: - ''' - self._keystone_client = keystoneclient.v2_0.client.Client( - auth_url=os.environ.get('OS_AUTH_URL'), - username=os.environ.get('OS_USERNAME'), - password=os.environ.get('OS_PASSWORD'), - tenant_name=os.environ.get('OS_TENANT_NAME')) - ''' - auth = v2.Password(auth_url=os.environ.get('OS_AUTH_URL'), - username=os.environ.get('OS_USERNAME'), - password=os.environ.get('OS_PASSWORD'), - tenant_name=os.environ.get('OS_TENANT_NAME')) - - sess = session.Session(auth=auth) - - return sess - - def _get_nova_client(self): - if self._nova_client is None: - keystone = self._get_keystone_client() - self._nova_client = client.Client('2', session=keystone) - return self._nova_client - - def check_aggregate(self, nova, agg_name): - list1 = nova.aggregates.list() - - agg_name_exist = False - for x in list1: - - if x.name == agg_name: - agg_name_exist = True - return agg_name_exist - - def get_aggregate_id(self, nova, agg_name): - list1 = nova.aggregates.list() - agg_id = 0 - agg_name_exist = False - for x in list1: - if x.name == agg_name: - agg_id = x.id - return agg_id - - def check_host_added_to_aggregate(self, nova, agg_id, hostname): - host_added = False - list1 = nova.aggregates.get_details(agg_id) - - nme = str(list1.hosts) - if(hostname in nme): - host_added = True - return host_added - - def del_agg(self, nova, id, host): - - nova.aggregates.remove_host(id, host) - nova.aggregates.delete(id) - - def get_compute_num(self, computeName): - - num = re.findall(r'\d+',computeName) - return (int(num[0])-1) - - def create_agg(self, D): - nova = self._get_nova_client() - hyper_list = nova.hypervisors.list() - hostnA = [] - zone_machine = defaultdict(list) - - x = 0 - for x in range(len(hyper_list)): - - hostnA.append(hyper_list[x].service['host']) - hostnA[x] = str(hostnA[x]) - - hostnA.sort() - for k in D: - - zone_machine[k].append(' ') - - for x in range(len(zone_machine)): - compute_index = self.get_compute_num(D[x]) - if compute_index > len(hyper_list): - print '\n The specified compute node doesnt exist. using compute 1' - compute_index = 1 - if not self.check_aggregate(nova, hostnA[compute_index]): - agg_idA = nova.aggregates.create(hostnA[compute_index], D[x]) - nova.aggregates.add_host(aggregate=agg_idA, host=hostnA[compute_index]) - - else: - - id1 = self.get_aggregate_id(nova, hostnA[compute_index]) - self.del_agg(nova, id1, hostnA[compute_index]) - agg_idA = nova.aggregates.create(hostnA[compute_index], D[x]) - id1 = self.get_aggregate_id(nova, hostnA[compute_index]) - - if not self.check_host_added_to_aggregate( - nova, id1, hostnA[compute_index]): - - nova.aggregates.add_host(aggregate=id1, host=hostnA[compute_index]) +##############################################################################
+# Copyright (c) 2015 Dell Inc and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from keystoneclient.auth.identity import v2
+from keystoneclient import session
+from novaclient import client
+import os
+import re
+from collections import defaultdict
+
+
+class create_zones:
+
+ def __init__(self):
+ print 'Creating Zones'
+ self._keystone_client = None
+ self._nova_client = None
+
+ def _get_keystone_client(self):
+ """returns a keystone client instance"""
+
+ if self._keystone_client is None:
+ '''
+ self._keystone_client = keystoneclient.v2_0.client.Client(
+ auth_url=os.environ.get('OS_AUTH_URL'),
+ username=os.environ.get('OS_USERNAME'),
+ password=os.environ.get('OS_PASSWORD'),
+ tenant_name=os.environ.get('OS_TENANT_NAME'))
+ '''
+ auth = v2.Password(auth_url=os.environ.get('OS_AUTH_URL'),
+ username=os.environ.get('OS_USERNAME'),
+ password=os.environ.get('OS_PASSWORD'),
+ tenant_name=os.environ.get('OS_TENANT_NAME'))
+
+ sess = session.Session(auth=auth)
+ else:
+ return self._keystone_client
+
+ return sess
+
+ def _get_nova_client(self):
+ if self._nova_client is None:
+ keystone = self._get_keystone_client()
+ self._nova_client = client.Client('2', session=keystone)
+ return self._nova_client
+
+ @staticmethod
+ def check_aggregate(nova, agg_name):
+ list1 = nova.aggregates.list()
+ agg_name_exist = False
+ for x in list1:
+ if x.name == agg_name:
+ agg_name_exist = True
+ return agg_name_exist
+
+ @staticmethod
+ def get_aggregate_id(nova, agg_name):
+ list1 = nova.aggregates.list()
+ for x in list1:
+ if x.name == agg_name:
+ agg_id = x.id
+ return agg_id
+
+ @staticmethod
+ def check_host_added_to_aggregate(nova, agg_id, hostname):
+ host_added = False
+ list1 = nova.aggregates.get_details(agg_id)
+
+ nme = str(list1.hosts)
+ if hostname in nme:
+ host_added = True
+ return host_added
+
+ @staticmethod
+ def del_agg(nova, id, host):
+
+ nova.aggregates.remove_host(id, host)
+ nova.aggregates.delete(id)
+
+ @staticmethod
+ def get_compute_num(compute_name):
+
+ num = re.findall(r'\d+', compute_name)
+ return int(num[0]) - 1
+
+ def test(self):
+ nova = self._get_nova_client()
+ hyper_list = nova.hypervisors.list()
+ return hyper_list
+
+ def create_agg(self, d):
+ nova = self._get_nova_client()
+ hyper_list = nova.hypervisors.list()
+ host_a = []
+ zone_machine = defaultdict(list)
+
+ for x in range(len(hyper_list)):
+
+ host_a.append(hyper_list[x].service['host'])
+ host_a[x] = str(host_a[x])
+
+ host_a.sort()
+ for k in d:
+
+ zone_machine[k].append(' ')
+
+ for x in range(len(zone_machine)):
+ compute_index = self.get_compute_num(d[x])
+ if compute_index > len(hyper_list):
+ print '\n The specified compute node doesnt exist. using compute 1'
+ compute_index = 1
+ if not self.check_aggregate(nova, host_a[compute_index]):
+ agg_id_a = nova.aggregates.create(host_a[compute_index], d[x])
+ nova.aggregates.add_host(aggregate=agg_id_a, host=host_a[compute_index])
+
+ else:
+ id1 = self.get_aggregate_id(nova, host_a[compute_index])
+ self.del_agg(nova, id1, host_a[compute_index])
+ nova.aggregates.create(host_a[compute_index], d[x])
+ id1 = self.get_aggregate_id(nova, host_a[compute_index])
+
+ if not self.check_host_added_to_aggregate(
+ nova, id1, host_a[compute_index]):
+
+ nova.aggregates.add_host(aggregate=id1, host=host_a[compute_index])
diff --git a/func/driver.py b/func/driver.py index 48c09c5d..33dbe320 100644 --- a/func/driver.py +++ b/func/driver.py @@ -6,13 +6,11 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - - - import os import json from collections import defaultdict + class Driver: def __init__(self): @@ -21,8 +19,8 @@ class Driver: print os.environ['PWD'] self.dic_json = defaultdict() - def drive_bench(self, benchmark, roles, benchmark_fname, benchmark_detail = None, pip_dict = None, proxy_info = None): - roles= sorted(roles) + def drive_bench(self, benchmark, roles, benchmark_fname, benchmark_detail=None, pip_dict=None, proxy_info=None): + roles = sorted(roles) pip_dict = sorted(pip_dict) result_dir = 'results' benchmark_name = benchmark + '.yaml' @@ -33,30 +31,30 @@ class Driver: self.dic_json['workingdir'] = str(os.environ['PWD']) self.dic_json['fname'] = str(benchmark_fname) self.dic_json['username'] = str('root') - + for key in proxy_info.keys(): self.dic_json[key] = proxy_info[key] - + if os.environ['INSTALLER_TYPE'] == str('joid'): - self.dic_json['username']=str('ubuntu') + self.dic_json['username'] = str('ubuntu') if os.environ['INSTALLER_TYPE'] == str('apex'): - self.dic_json['username']=str('heat-admin') - for k,v in benchmark_detail: - self.dic_json[k]=v + self.dic_json['username'] = str('heat-admin') + for k, v in benchmark_detail: + self.dic_json[k] = v for k, v in roles: - self.dic_json['role']=k - index=1 - if benchmark_detail != None: + self.dic_json['role'] = k + index = 1 + if benchmark_detail is not None: for values in v: - if k == '1-server': + if k == '1-server': print values, 'saving IP' - self.dic_json['ip'+str(index)]= str(values) + self.dic_json['ip' + str(index)] = str(values) if pip_dict[0][1][0]: - self.dic_json['privateip'+str(index)] = pip_dict[0][1] + self.dic_json['privateip' + str(index)] = pip_dict[0][1] if not pip_dict[0][1][0]: - self.dic_json['privateip'+str(index)] = 'NONE' - index= index+1 + self.dic_json['privateip' + str(index)] = 'NONE' + index = index + 1 dic_json = json.dumps(dict(self.dic_json.items())) print dic_json run_play = 'ansible-playbook ./benchmarks/playbooks/{0} --private-key=./data/QtipKey -i ./data/hosts --extra-vars \'{1}\''.format(benchmark_name, dic_json) - status = os.system(run_play) + os.system(run_play) diff --git a/func/env_setup.py b/func/env_setup.py index c1e2a003..9c0dadb3 100644 --- a/func/env_setup.py +++ b/func/env_setup.py @@ -13,19 +13,22 @@ from collections import defaultdict import yaml import time import paramiko -class Env_setup(): +import socket + + +class Env_setup: roles_ip_list = [] # ROLE and its corresponding IP address list ip_pw_list = [] # IP and password, this will be used to ssh roles_dict = defaultdict(list) ip_pw_dict = defaultdict(list) ip_pip_list = [] vm_parameters = defaultdict(list) - benchmark_details= defaultdict() + benchmark_details = defaultdict() benchmark = '' def __init__(self): print '\nParsing class initiated\n' - self.roles_ip_list[:]=[] + self.roles_ip_list[:] = [] self.ip_pw_list[:] = [] self.roles_dict.clear() self.ip_pw_dict.clear() @@ -35,41 +38,44 @@ class Env_setup(): self.benchmark_details.clear() self.benchmark = '' - def writeTofile(self, role): - fname2 = open('./data/hosts', 'w') + @staticmethod + def write_to_file(role): + f_name_2 = open('./data/hosts', 'w') print role.items() for k in role: - fname2.write('[' + k + ']\n') + f_name_2.write('[' + k + ']\n') num = len(role[k]) for x in range(num): - fname2.write(role[k][x] + '\n') - fname2.close + f_name_2.write(role[k][x] + '\n') + f_name_2.close() - def sshtest(self, lister): - print 'list: ',lister + @staticmethod + def ssh_test(lister): + print 'list: ', lister for k, v in lister: - ipvar = k - pwvar = v + ip_var = k print '\nBeginning SSH Test!\n' if v != '': - print ('\nSSH->>>>> {0} {1}\n'.format(k,v)) + print ('\nSSH->>>>> {0} {1}\n'.format(k, v)) time.sleep(2) ssh_c = 'ssh-keyscan {0} >> ~/.ssh/known_hosts'.format(k) os.system(ssh_c) - ssh_cmd = './data/qtip_creds.sh {0}'.format(ipvar) + ssh_cmd = './data/qtip_creds.sh {0}'.format(ip_var) print ssh_cmd - res = os.system(ssh_cmd) + os.system(ssh_cmd) for infinity in range(100): - try : + try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(k , key_filename= './data/QtipKey') + ssh.connect(k, key_filename='./data/QtipKey') stdin, stdout, stderr = ssh.exec_command('ls') print('SSH successful') + for line in stdout: + print '... ' + line.strip('\n') break - except: - print 'Retrying aSSH' + except socket.error: + print 'Retrying aSSH %s' % infinity time.sleep(1) if v == '': print ('SSH->>>>>', k) @@ -79,99 +85,91 @@ class Env_setup(): os.system(ssh_c) for infinity in range(10): - try : + try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(k, key_filename= './data/QtipKey') + ssh.connect(k, key_filename='./data/QtipKey') stdin, stdout, stderr = ssh.exec_command('ls') + print('SSH successful') + for line in stdout: + print '... ' + line.strip('\n') break - except: - print 'Retrying SSH' + except socket.error: + print 'Retrying SSH %s' % infinity + + @staticmethod + def ping_test(lister): - def pingtest(self, lister): - pingFlag = 0 - result = True for k, v in lister.iteritems(): time.sleep(10) for val in v: ipvar = val ping_cmd = 'ping -D -c1 {0}'.format(ipvar) - while (os.system(ping_cmd) != 0) &(pingFlag <=20): + while os.system(ping_cmd) != 0: print '\nWaiting for machine\n' time.sleep(10) - pingFlag = pingFlag+1 - if pingFlag <= 2: - print ('\n\n %s is UP \n\n ' % ipvar) - else: - result = False - return result - + print ('\n\n %s is UP \n\n ' % ipvar) - def GetHostMachineinfo(self, Hosttag): + def get_host_machine_info(self, host_tag): - num = len(Hosttag) + num = len(host_tag) offset = len(self.roles_ip_list) for x in range(num): hostlabel = 'machine_' + str(x + 1) self.roles_ip_list.insert( - offset, (Hosttag[hostlabel]['role'], Hosttag[hostlabel]['ip'])) + offset, (host_tag[hostlabel]['role'], host_tag[hostlabel]['ip'])) self.ip_pw_list.insert( - offset, (Hosttag[hostlabel]['ip'], Hosttag[hostlabel]['pw'])) + offset, (host_tag[hostlabel]['ip'], host_tag[hostlabel]['pw'])) - def GetVirtualMachineinfo(self, Virtualtag): + def get_virtual_machine_info(self, virtual_tag): - num = len(Virtualtag) + num = len(virtual_tag) for x in range(num): - hostlabel = 'virtualmachine_' + str(x + 1) - for k, v in Virtualtag[hostlabel].iteritems(): + host_label = 'virtualmachine_' + str(x + 1) + for k, v in virtual_tag[host_label].iteritems(): self.vm_parameters[k].append(v) - def GetBenchmarkDetails(self, detail_dic): + def get_bench_mark_details(self, detail_dic): print detail_dic - for k,v in detail_dic.items(): - self.benchmark_details[k]= v + for k, v in detail_dic.items(): + self.benchmark_details[k] = v - def parse(self, configfilepath): + def parse(self, config_file_path): try: - fname = open(configfilepath, 'r+') - doc = yaml.load(fname) -# valid_file = validate_yaml.Validate_Yaml(doc) - fname.close() - for scenario in doc: + f_name = open(config_file_path, 'r+') + doc = yaml.load(f_name) + f_name.close() + if doc['Scenario']['benchmark']: self.benchmark = doc['Scenario']['benchmark'] if doc['Context']['Virtual_Machines']: - self.GetVirtualMachineinfo(doc['Context']['Virtual_Machines']) + self.get_virtual_machine_info(doc['Context']['Virtual_Machines']) if doc['Context']['Host_Machines']: - self.GetHostMachineinfo(doc['Context']['Host_Machines']) - if doc.get('Scenario',{}).get('benchmark_details',{}): - self.GetBenchmarkDetails(doc.get('Scenario',{}).get('benchmark_details',{})) - if 'Proxy_Environment' in doc['Context'].keys(): + self.get_host_machine_info(doc['Context']['Host_Machines']) + if doc.get('Scenario', {}).get('benchmark_details', {}): + self.get_bench_mark_details(doc.get('Scenario', {}).get('benchmark_details', {})) + if 'Proxy_Environment' in doc['Context'].keys(): self.proxy_info['http_proxy'] = doc['Context']['Proxy_Environment']['http_proxy'] self.proxy_info['https_proxy'] = doc['Context']['Proxy_Environment']['https_proxy'] - self.proxy_info['no_proxy'] = doc['Context']['Proxy_Environment']['no_proxy'] + self.proxy_info['no_proxy'] = doc['Context']['Proxy_Environment']['no_proxy'] for k, v in self.roles_ip_list: self.roles_dict[k].append(v) for k, v in self.ip_pw_list: self.ip_pw_dict[k].append(v) return ( self.benchmark, - self.roles_dict.items(), self.vm_parameters, self.benchmark_details.items(), - self.ip_pw_dict.items(), self.proxy_info) - except KeyboardInterrupt: - fname.close() print 'ConfigFile Closed: exiting!' sys.exit(0) - def updateAnsible(self): - self.writeTofile(self.roles_dict) + def update_ansible(self): + self.write_to_file(self.roles_dict) - def callpingtest(self): - self.pingtest(self.roles_dict) + def call_ping_test(self): + self.ping_test(self.roles_dict) - def callsshtest(self): - self.sshtest(self.ip_pw_list) + def call_ssh_test(self): + self.ssh_test(self.ip_pw_list) diff --git a/func/fetchimg.py b/func/fetchimg.py index 1c621a3b..1ed3def6 100644 --- a/func/fetchimg.py +++ b/func/fetchimg.py @@ -1,30 +1,35 @@ -############################################################################## -# Copyright (c) 2015 Dell Inc and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - - -import os -import time - - -class FetchImg: - - def __init__(self): - print 'Fetching Image!' - print 'Fetching QTIP_VM Image' - - def download(self): - time.sleep(2) - os.system( - 'mkdir -p Temp_Img && wget http://artifacts.opnfv.org/qtip/QTIP_CentOS.qcow2 -P Temp_Img') - - filepath = './Temp_Img/QTIP_CentOS.qcow2' - while not os.path.isfile(filepath): - time.sleep(10) - print 'Download Completed!' +##############################################################################
+# Copyright (c) 2015 Dell Inc and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import os
+import time
+
+IMGSTORE = "/home/opnfv/imgstore"
+
+
+class FetchImg:
+
+ def __init__(self):
+ print 'Fetching Image!'
+
+ @staticmethod
+ def download():
+ time.sleep(2)
+ os.system('mkdir -p Temp_Img')
+ filepath = './Temp_Img/QTIP_CentOS.qcow2'
+ imgstorepath = IMGSTORE + "/QTIP_CentOS.qcow2"
+ if os.path.isfile(imgstorepath):
+ os.system("ln -s %s %s" % (imgstorepath, filepath))
+ print "QTIP_CentOS.qcow2 exists locally. Skipping the download and using the file from IMG store"
+ else:
+ print 'Fetching QTIP_CentOS.qcow2'
+ os.system('wget http://artifacts.opnfv.org/qtip/QTIP_CentOS.qcow2 -P Temp_Img')
+
+ while not os.path.isfile(filepath):
+ time.sleep(10)
+ print 'Download Completed!'
diff --git a/func/spawn_vm.py b/func/spawn_vm.py index 94aa424a..ec63230c 100644 --- a/func/spawn_vm.py +++ b/func/spawn_vm.py @@ -1,282 +1,279 @@ -############################################################################## -# Copyright (c) 2015 Dell Inc and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import os -import sys -from collections import defaultdict -from func.env_setup import Env_setup -from func.fetchimg import FetchImg -import yaml -import heatclient.client -import keystoneclient -import glanceclient -from novaclient import client -import time -import json -from func.create_zones import create_zones - -class SpawnVM(Env_setup): - vm_role_ip_dict = defaultdict(list) - installer = '' - - def __init__(self, vm_info): - print 'SpawnVM Class initiated' - vm_role_ip_dict = vm_info.copy() - print 'Generating Heat Template\n' - self._keystone_client = None - self._heat_client = None - self._glance_client = None - self._nova_client = None - nova =self. _get_nova_client() - azoneobj = create_zones() - azoneobj.create_agg(vm_info['availability_zone']) - installer= self.get_installer_type() - self.Heat_template1 = self.HeatTemplate_vm(vm_info,installer) - self.create_stack(vm_role_ip_dict, self.Heat_template1) - - def get_installer_type(self): - print 'Getting Installer Name' - return os.environ['INSTALLER_TYPE'] - - def get_public_network(self,installer_detected): - - ''' - TODO: GET THE NAMES OF THE PUBLIC NETWORKS for OTHER PROJECTS - ''' - print 'Getting Public Network' - if installer_detected.lower() == 'fuel': - return 'admin_floating_net' - if installer_detected.lower() == 'apex': - return 'external' - if installer_detected.lower() == 'compass': - return 'ext-net' - if installer_detected.lower() == 'joid': - return 'ext-net' - - def HeatTemplate_vm(self, vm_params, installer): - try: - Heat_Dic='' - with open('./heat/SampleHeat.yaml', 'r+') as H_temp: - Heat_Dic = yaml.load(H_temp) - except yaml.YAMLError as exc: - if hasattr(exc, 'problem_mark'): - mark = exc.problem_mark - print 'Error in qtip/heat/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1, mark.column + 1) - print 'EXITING PROGRAM. Correct File and restart' - sys.exit(0) - fopen = open('./data/QtipKey.pub', 'r') - fopenstr = fopen.read() - fopenstr = fopenstr.rstrip() - scriptcmd = '#!/bin/bash \n echo {0} >> foo.txt \n echo {1} >> /root/.ssh/authorized_keys'.format( - fopenstr, fopenstr) - - netName = self.get_public_network(installer) - print netName - Heat_Dic['heat_template_version'] = '2014-10-16' - Heat_Dic['resources']['KeyPairSavePrivate'] = { - 'type': 'OS::Nova::KeyPair', - 'properties': { - 'save_private_key': 'true', - 'name': 'my_key' - } - } - Heat_Dic['parameters']['public_network'] = { - 'type': 'string', - #'default': vm_params['public_network'][0] - 'default': netName - } - for x in range(1, len(vm_params['availability_zone']) + 1): - avail_zone = vm_params['availability_zone'][x - 1] - img = vm_params['OS_image'][x - 1] - flavor = vm_params['flavor'][x - 1] - - Heat_Dic['parameters']['availability_zone_' +str(x)] = { - 'description': 'Availability Zone of the instance', - 'default': avail_zone, - 'type': 'string' - - } - - Heat_Dic['resources']['public_port_' +str(x)] = { - 'type': 'OS::Neutron::Port', - 'properties': { - 'network': {'get_resource': 'private_network'}, - 'security_groups': [{ 'get_resource': 'demo1_security_Group'}], - 'fixed_ips': [ - {'subnet_id': {'get_resource': 'private_subnet'}}]}} - - Heat_Dic['resources']['floating_ip_' + str(x)] = { - 'type': 'OS::Neutron::FloatingIP', - 'properties': { - 'floating_network': {'get_param': 'public_network'}}} - - Heat_Dic['resources']['floating_ip_assoc_' + str(x)] = { - 'type': 'OS::Neutron::FloatingIPAssociation', - 'properties': { - 'floatingip_id': {'get_resource': 'floating_ip_' + str(x)}, - 'port_id': {'get_resource': 'public_port_' + str(x)}}} - - Heat_Dic['resources']['my_instance_' + str(x)] = { - 'type': 'OS::Nova::Server', - 'properties': { - 'image': img, - 'networks':[ - {'port': {'get_resource': 'public_port_' + str(x)}}], - 'flavor': flavor, - 'availability_zone': avail_zone, - 'name': 'instance' + str(x), - 'key_name': {'get_resource': 'KeyPairSavePrivate'}, - 'user_data_format': 'RAW', - 'user_data': scriptcmd}} - - Heat_Dic['resources']['demo1_security_Group'] = { - 'type': 'OS::Neutron::SecurityGroup', - 'properties': { - 'name': 'demo1_security_Group', - 'rules': [{ - 'protocol': 'tcp', - 'port_range_min': 22, - 'port_range_max': 5201}, - {'protocol': 'udp', - 'port_range_min': 22, - 'port_range_max': 5201}, - {'protocol': 'icmp'}]}} - - Heat_Dic['outputs']['instance_PIP_' +str(x)] = { - 'description': 'IP address of the instance', - 'value': {'get_attr': ['my_instance_' + str(x), 'first_address']}} - Heat_Dic['outputs']['instance_ip_' +str(x)] = { - 'description': 'IP address of the instance', - 'value': {'get_attr': ['floating_ip_' + str(x), 'floating_ip_address']}} - - Heat_Dic['outputs']['availability_instance_' + str(x)] = { - 'description': 'Availability Zone of the Instance', - 'value': { 'get_param': 'availability_zone_'+str(x)}} - - - Heat_Dic['outputs']['KeyPair_PublicKey'] = { - 'description': 'Private Key', - 'value': {'get_attr': ['KeyPairSavePrivate', 'private_key']} - } - del Heat_Dic['outputs']['description'] - return Heat_Dic - - def _get_keystone_client(self): - '''returns a keystone client instance''' - - if self._keystone_client is None: - self._keystone_client = keystoneclient.v2_0.client.Client( - auth_url=os.environ.get('OS_AUTH_URL'), - username=os.environ.get('OS_USERNAME'), - password=os.environ.get('OS_PASSWORD'), - tenant_name=os.environ.get('OS_TENANT_NAME')) - return self._keystone_client - - def _get_nova_client(self): - if self._nova_client is None: - keystone = self._get_keystone_client() - self._nova_client = client.Client('2', token=keystone.auth_token) - return self._nova_client - - def _get_heat_client(self): - '''returns a heat client instance''' - if self._heat_client is None: - keystone = self._get_keystone_client() - heat_endpoint = keystone.service_catalog.url_for( - service_type='orchestration') - self._heat_client = heatclient.client.Client( - '1', endpoint=heat_endpoint, token=keystone.auth_token) - return self._heat_client - - def _get_glance_client(self): - if self._glance_client is None: - keystone = self._get_keystone_client() - glance_endpoint = keystone.service_catalog.url_for( - service_type='image') - self._glance_client = glanceclient.Client( - '2', glance_endpoint, token=keystone.auth_token) - return self._glance_client - - def create_stack(self, vm_role_ip_dict, Heat_template): - - stackname = 'QTIP' - heat = self._get_heat_client() - glance = self._get_glance_client() - - available_images = [] - for image_list in glance.images.list(): - - available_images.append(image_list.name) - - if 'QTIP_CentOS' in available_images: - print 'Image Present' - - elif 'QTIP_CentOS' not in available_images: - fetchImage = FetchImg() - fetchImage.download() - print 'Uploading Image to Glance. Please wait' - qtip_image = glance.images.create( - name='QTIP_CentOS', - visibility='public', - disk_format='qcow2', - container_format='bare') - qtip_image = glance.images.upload( - qtip_image.id, open('./Temp_Img/QTIP_CentOS.qcow2')) - json_temp = json.dumps(Heat_template) - - for checks in range(3): - for prev_stacks in heat.stacks.list(): - - if prev_stacks.stack_name == 'QTIP': - print 'QTIP Stacks exists.\nDeleting Existing Stack' - heat.stacks.delete('QTIP') - time.sleep(10) - - print '\nStack Creating Started\n' - - try: - heat.stacks.create(stack_name=stackname, template=Heat_template) - except: - print 'Create Failed :( ' - - cluster_detail = heat.stacks.get(stackname) - while(cluster_detail.status != 'COMPLETE'): - if cluster_detail.status == 'IN_PROGRESS': - print 'Stack Creation in Progress' - cluster_detail = heat.stacks.get(stackname) - time.sleep(10) - print 'Stack Created' - print 'Getting Public IP(s)' - zone = [] - s=0 - for vm in range(len(vm_role_ip_dict['OS_image'])): - - for I in cluster_detail.outputs: - availabilityKey = 'availability_instance_'+str(vm+1) - - if I['output_key'] == availabilityKey: - zone.insert(s,str(I['output_value'])) - s=s+1 - for i in cluster_detail.outputs: - instanceKey = "instance_ip_" + str(vm + 1) - privateIPkey = 'instance_PIP_' + str(vm +1) - if i['output_key'] == instanceKey: - Env_setup.roles_dict[vm_role_ip_dict['role'][vm]].append( - str(i['output_value'])) - Env_setup.ip_pw_list.append((str(i['output_value']),'')) - - if i['output_key'] == privateIPkey: - Env_setup.ip_pw_dict[vm_role_ip_dict['role'][vm]]=str(i['output_value']) - if i['output_key'] == 'KeyPair_PublicKey': - sshkey = str(i['output_value']) - - with open('./data/my_key.pem', 'w') as fopen: - fopen.write(sshkey) - fopen.close() - print Env_setup.ip_pw_list +##############################################################################
+# Copyright (c) 2015 Dell Inc and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+import sys
+from collections import defaultdict
+from func.env_setup import Env_setup
+from func.fetchimg import FetchImg
+import yaml
+import heatclient.client
+import keystoneclient
+import glanceclient
+from novaclient import client
+import time
+from func.create_zones import create_zones
+
+
+class SpawnVM(Env_setup):
+ vm_role_ip_dict = defaultdict(list)
+ installer = ''
+
+ def __init__(self, vm_info):
+ Env_setup.__init__(self)
+ print 'SpawnVM Class initiated'
+ vm_role_ip_dict = vm_info.copy()
+ print 'Generating Heat Template\n'
+ self._keystone_client = None
+ self._heat_client = None
+ self._glance_client = None
+ self._nova_client = None
+ self. _get_nova_client()
+ azoneobj = create_zones()
+ azoneobj.create_agg(vm_info['availability_zone'])
+ installer = self.get_installer_type()
+ self.Heat_template1 = self.heat_template_vm(vm_info, installer)
+ self.create_stack(vm_role_ip_dict, self.Heat_template1)
+
+ @staticmethod
+ def get_installer_type():
+ print 'Getting Installer Name'
+ return os.environ['INSTALLER_TYPE']
+
+ @staticmethod
+ def get_public_network(installer_detected):
+
+ """
+ TODO: GET THE NAMES OF THE PUBLIC NETWORKS for OTHER PROJECTS
+ """
+ print 'Getting Public Network'
+ if installer_detected.lower() == 'fuel':
+ return 'admin_floating_net'
+ if installer_detected.lower() == 'apex':
+ return 'external'
+ if installer_detected.lower() == 'compass':
+ return 'ext-net'
+ if installer_detected.lower() == 'joid':
+ return 'ext-net'
+
+ def heat_template_vm(self, vm_params, installer):
+ Heat_Dic = {}
+ try:
+ with open('./heat/SampleHeat.yaml', 'r+') as H_temp:
+ Heat_Dic = yaml.load(H_temp)
+ except yaml.YAMLError as exc:
+ if hasattr(exc, 'problem_mark'):
+ mark = exc.problem_mark
+ print 'Error in qtip/heat/SampleHeat.yaml at: (%s,%s)' % (mark.line + 1, mark.column + 1)
+ print 'EXITING PROGRAM. Correct File and restart'
+ sys.exit(0)
+ fopen = open('./data/QtipKey.pub', 'r')
+ fopenstr = fopen.read()
+ fopenstr = fopenstr.rstrip()
+ scriptcmd = '#!/bin/bash \n echo {0} >> foo.txt \n echo {1} >> /root/.ssh/authorized_keys'.format(
+ fopenstr, fopenstr)
+
+ netName = self.get_public_network(installer)
+ print netName
+ Heat_Dic['heat_template_version'] = '2014-10-16'
+ Heat_Dic['resources']['KeyPairSavePrivate'] = {
+ 'type': 'OS::Nova::KeyPair',
+ 'properties': {
+ 'save_private_key': 'true',
+ 'name': 'my_key'
+ }
+ }
+ Heat_Dic['parameters']['public_network'] = {
+ 'type': 'string',
+ 'default': netName
+ }
+ for x in range(1, len(vm_params['availability_zone']) + 1):
+ avail_zone = vm_params['availability_zone'][x - 1]
+ img = vm_params['OS_image'][x - 1]
+ flavor = vm_params['flavor'][x - 1]
+
+ Heat_Dic['parameters']['availability_zone_' + str(x)] = \
+ {'description': 'Availability Zone of the instance',
+ 'default': avail_zone,
+ 'type': 'string'}
+
+ Heat_Dic['resources']['public_port_' + str(x)] = \
+ {'type': 'OS::Neutron::Port',
+ 'properties': {'network': {'get_resource': 'private_network'},
+ 'security_groups': [{'get_resource': 'demo1_security_Group'}],
+ 'fixed_ips': [{'subnet_id':
+ {'get_resource': 'private_subnet'}}]}}
+
+ Heat_Dic['resources']['floating_ip_' + str(x)] = {
+ 'type': 'OS::Neutron::FloatingIP',
+ 'properties': {
+ 'floating_network': {'get_param': 'public_network'}}}
+
+ Heat_Dic['resources']['floating_ip_assoc_' + str(x)] = {
+ 'type': 'OS::Neutron::FloatingIPAssociation',
+ 'properties': {
+ 'floatingip_id': {'get_resource': 'floating_ip_' + str(x)},
+ 'port_id': {'get_resource': 'public_port_' + str(x)}}}
+
+ Heat_Dic['resources']['my_instance_' + str(x)] = \
+ {'type': 'OS::Nova::Server',
+ 'properties': {'image': img,
+ 'networks':
+ [{'port': {'get_resource': 'public_port_' + str(x)}}],
+ 'flavor': flavor,
+ 'availability_zone': avail_zone,
+ 'name': 'instance' + str(x),
+ 'key_name': {'get_resource': 'KeyPairSavePrivate'},
+ 'user_data_format': 'RAW',
+ 'user_data': scriptcmd}}
+
+ Heat_Dic['resources']['demo1_security_Group'] = {
+ 'type': 'OS::Neutron::SecurityGroup',
+ 'properties': {
+ 'name': 'demo1_security_Group',
+ 'rules': [{
+ 'protocol': 'tcp',
+ 'port_range_min': 22,
+ 'port_range_max': 5201},
+ {'protocol': 'udp',
+ 'port_range_min': 22,
+ 'port_range_max': 5201},
+ {'protocol': 'icmp'}]}}
+
+ Heat_Dic['outputs']['instance_PIP_' + str(x)] = {
+ 'description': 'IP address of the instance',
+ 'value': {'get_attr': ['my_instance_' + str(x), 'first_address']}}
+ Heat_Dic['outputs']['instance_ip_' + str(x)] = {
+ 'description': 'IP address of the instance',
+ 'value': {'get_attr': ['floating_ip_' + str(x), 'floating_ip_address']}}
+
+ Heat_Dic['outputs']['availability_instance_' + str(x)] = {
+ 'description': 'Availability Zone of the Instance',
+ 'value': {'get_param': 'availability_zone_' + str(x)}}
+
+ Heat_Dic['outputs']['KeyPair_PublicKey'] = {
+ 'description': 'Private Key',
+ 'value': {'get_attr': ['KeyPairSavePrivate', 'private_key']}
+ }
+ del Heat_Dic['outputs']['description']
+ print Heat_Dic
+ return Heat_Dic
+
+ def _get_keystone_client(self):
+ """returns a keystone client instance"""
+
+ if self._keystone_client is None:
+ self._keystone_client = keystoneclient.v2_0.client.Client(
+ auth_url=os.environ.get('OS_AUTH_URL'),
+ username=os.environ.get('OS_USERNAME'),
+ password=os.environ.get('OS_PASSWORD'),
+ tenant_name=os.environ.get('OS_TENANT_NAME'))
+ return self._keystone_client
+
+ def _get_nova_client(self):
+ if self._nova_client is None:
+ keystone = self._get_keystone_client()
+ self._nova_client = client.Client('2', token=keystone.auth_token)
+ return self._nova_client
+
+ def _get_heat_client(self):
+ """returns a heat client instance"""
+ if self._heat_client is None:
+ keystone = self._get_keystone_client()
+ heat_endpoint = keystone.service_catalog.url_for(
+ service_type='orchestration')
+ self._heat_client = heatclient.client.Client(
+ '1', endpoint=heat_endpoint, token=keystone.auth_token)
+ return self._heat_client
+
+ def _get_glance_client(self):
+ if self._glance_client is None:
+ keystone = self._get_keystone_client()
+ glance_endpoint = keystone.service_catalog.url_for(
+ service_type='image')
+ self._glance_client = glanceclient.Client(
+ '2', glance_endpoint, token=keystone.auth_token)
+ return self._glance_client
+
+ def create_stack(self, vm_role_ip_dict, heat_template):
+
+ global sshkey
+ stackname = 'QTIP'
+ heat = self._get_heat_client()
+ glance = self._get_glance_client()
+
+ available_images = []
+ for image_list in glance.images.list():
+
+ available_images.append(image_list.name)
+
+ if 'QTIP_CentOS' in available_images:
+ print 'Image Present'
+
+ elif 'QTIP_CentOS' not in available_images:
+ fetchImage = FetchImg()
+ fetchImage.download()
+ print 'Uploading Image to Glance. Please wait'
+ qtip_image = glance.images.create(
+ name='QTIP_CentOS',
+ visibility='public',
+ disk_format='qcow2',
+ container_format='bare')
+ glance.images.upload(
+ qtip_image.id, open('./Temp_Img/QTIP_CentOS.qcow2'))
+ for checks in range(3):
+ print "Try to delete heats %s" % checks
+ for prev_stacks in heat.stacks.list():
+ if prev_stacks.stack_name == 'QTIP':
+ print 'QTIP Stacks exists.\nDeleting Existing Stack'
+ heat.stacks.delete('QTIP')
+ time.sleep(10)
+
+ print '\nStack Creating Started\n'
+
+ try:
+ heat.stacks.create(stack_name=stackname, template=heat_template)
+ except Exception:
+ print 'Create Failed :( '
+
+ cluster_detail = heat.stacks.get(stackname)
+ while cluster_detail.status != 'COMPLETE':
+ if cluster_detail.status == 'IN_PROGRESS':
+ print 'Stack Creation in Progress'
+ cluster_detail = heat.stacks.get(stackname)
+ time.sleep(10)
+ print 'Stack Created'
+ print 'Getting Public IP(s)'
+ zone = []
+ s = 0
+ for vm in range(len(vm_role_ip_dict['OS_image'])):
+
+ for I in cluster_detail.outputs:
+ availabilityKey = 'availability_instance_' + str(vm + 1)
+
+ if I['output_key'] == availabilityKey:
+ zone.insert(s, str(I['output_value']))
+ s = s + 1
+ for i in cluster_detail.outputs:
+ instanceKey = "instance_ip_" + str(vm + 1)
+ privateIPkey = 'instance_PIP_' + str(vm + 1)
+ if i['output_key'] == instanceKey:
+ Env_setup.roles_dict[vm_role_ip_dict['role'][vm]] \
+ .append(str(i['output_value']))
+ Env_setup.ip_pw_list.append((str(i['output_value']), ''))
+
+ if i['output_key'] == privateIPkey:
+ Env_setup.ip_pw_dict[vm_role_ip_dict['role'][vm]] = str(i['output_value'])
+ if i['output_key'] == 'KeyPair_PublicKey':
+ sshkey = str(i['output_value'])
+
+ with open('./data/my_key.pem', 'w') as fopen:
+ fopen.write(sshkey)
+ fopen.close()
+ print Env_setup.ip_pw_list
diff --git a/func/validate_yaml.py b/func/validate_yaml.py deleted file mode 100644 index fcf32a21..00000000 --- a/func/validate_yaml.py +++ /dev/null @@ -1,28 +0,0 @@ -##############################################################################
-# Copyright (c) 2015 Dell Inc and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-class Validate_Yaml(object):
-
- def __init__(self, doc):
-
- print('Validating YAML CONFIG FILE')
-
- if not doc['Scenario']:
- print('\nScenario Field missing\na')
- if not doc['Scenario']['benchmark']:
- print('\nBenchmark field missing')
- if not doc['Scenario']['pointless']:
- print('')
- if not doc['Context']:
- print('\nEntire Context is missing')
- if not doc['Context']['Host_Machine']:
- print('\nNo Host Machine')
- if not doc['Context']['Host_Machine']['machine_1']:
- print('\nNo Host Machine')
|