diff options
Diffstat (limited to 'qtip/util/env.py')
| -rw-r--r-- | qtip/util/env.py | 288 |
1 files changed, 182 insertions, 106 deletions
diff --git a/qtip/util/env.py b/qtip/util/env.py index 4e7a31c7..9299f8c0 100644 --- a/qtip/util/env.py +++ b/qtip/util/env.py @@ -1,137 +1,213 @@ ############################################################################## -# Copyright (c) 2016 Dell Inc, ZTE and others. +# Copyright (c) 2017 ZTE and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +from collections import defaultdict import os -import paramiko +from os import path +import re import socket +import sys import time -from os import path -from os.path import expanduser - -SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') -CONFIG_DIR = path.join(path.dirname(__file__), path.pardir, path.pardir, - 'config') -PRIVATE_KEY = CONFIG_DIR + '/QtipKey' -PUBLIC_KEY = CONFIG_DIR + '/QtipKey.pub' -IPS_FILE = expanduser('~') + "/qtip/ips.log" -HOST_FILE = CONFIG_DIR + "/host" - - -def fetch_compute_ips_via_installer(): - clean_file(IPS_FILE) - - installer_type = str(os.environ['INSTALLER_TYPE'].lower()) - installer_ip = str(os.environ['INSTALLER_IP']) - if installer_type not in ["fuel"]: - raise RuntimeError("%s is not supported" % installer_type) - if not installer_ip: - raise RuntimeError("undefine environment variable INSTALLER_IP") - - cmd = "bash %s/fetch_compute_ips.sh -i %s -a %s" % \ - (SCRIPT_DIR, installer_type, installer_ip) - os.system(cmd) - if path.isfile(IPS_FILE): - return True - else: - return False +import paramiko -def parse_ips(): - ip_list = [] - with open(IPS_FILE, "r") as outfile: - data = outfile.read() - if data: - ip_list.extend(data.rstrip('\n').split('\n')) - return ip_list +from qtip.util.logger import QtipLogger +logger = QtipLogger('env').get -def ssh_test(ip): - os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) - time.sleep(2) +SCRIPT_DIR = path.join(path.dirname(__file__), path.pardir, 'scripts') +KEYNAME = 'QtipKey' +PRIVATE_KEY = '{0}/qtip/{1}'.format(os.environ['HOME'], KEYNAME) +PUBLIC_KEY = PRIVATE_KEY + '.pub' +HOST_FILE = '{0}/qtip/hosts'.format(os.environ['HOME']) - ssh_cmd = '%s/qtip_creds.sh %s' % (SCRIPT_DIR, ip) - os.system(ssh_cmd) - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(ip, key_filename='{0}/QtipKey'.format(CONFIG_DIR)) +def all_files_exist(*files): + if len(files) == 0: + return False + flag = True + for f_item in files: + flag &= path.isfile(f_item) + logger.info("Is {0} existed: {1}".format(f_item, flag)) + return flag - for attempts in range(100): - try: - stdin, stdout, stderr = ssh.exec_command('uname') - if not stderr.readlines(): - print("{0}: SSH test successful") - return True - except socket.error: - if attempts == 99: - return False - time.sleep(2) +def clean_file(*files): + if len(files) == 0: + logger.info('Nothing to clean') + return False -def ping_test(ip, attempts=30): - ping_cmd = 'ping -D -c1 {0}'.format(ip) - for i in range(attempts): - if os.system(ping_cmd): - print('\nWaiting for machine\n') - time.sleep(10) - else: - print('\n\n %s is UP \n\n ' % ip) + def clean(f): + try: + if all_files_exist(f): + os.remove(f) + logger.info("Removed: {0}".format(f)) + else: + logger.info("Not exists: {0}".format(f)) return True - if i == 29: + except OSError as error: + logger.error("Not able to Remove: {0}".format(f), error) return False + results = map(clean, files) + return len(results) == len(files) and False not in results -def check_nodes_connectivity(): - ip_list = parse_ips() - for ip in ip_list: - if not ping_test(ip): - raise RuntimeError("{0}: Ping test failed".format(ip)) - if not ssh_test(ip): - raise RuntimeError("{0}: SSH test failed".format(ip)) +class AnsibleEnvSetup(object): + def __init__(self): + self.keypair = defaultdict(str) + self.hostfile = None + self.host_ip_list = [] -def generate_host_file(): - ip_list = parse_ips() - with open(HOST_FILE, 'w') as host_file: - for index, item in enumerate(ip_list): - host_file.write("[host_{0}]\n".format(index)) - host_file.write(item + '\n') - - -def generate_keypair(): - """Generating ssh keypair""" - if not clean_keypair(): - raise RuntimeError("Cann't remove old keypair") - - cmd = "ssh-keygen -t rsa -N "" -f {0} -q".format(PRIVATE_KEY) - os.system(cmd) - - if path.isfile(PRIVATE_KEY) and path.isfile(PUBLIC_KEY): - return True - else: - return False + def setup(self, config={}): + try: + if 'hostfile' in config: + self.check_hostfile(config['hostfile']) + else: + self.generate_default_hostfile() + self.fetch_host_ip_from_hostfile() + if 'keypair' in config: + self.check_keypair(config['keypair']) + else: + self.generate_default_keypair() + self.pass_keypair_to_remote() + self.check_hosts_ssh_connectivity() + except Exception as error: + logger.info(error) + sys.exit(1) + + def check_keypair(self, keypair): + self.keypair = defaultdict(str) + if all_files_exist(keypair, '{0}.pub'.format(keypair)): + self.keypair['private'] = keypair + self.keypair['public'] = '{0}.pub'.format(keypair) + else: + raise RuntimeError("The keypairs you in the configuration file" + " is invalid or not existed.") + + def generate_default_keypair(self): + if not all_files_exist(PRIVATE_KEY, PUBLIC_KEY): + logger.info("Generate default keypair {0} under " + "{1}".format(KEYNAME, os.environ['HOME'])) + cmd = '''ssh-keygen -t rsa -N "" -f {0} -q -b 2048 + -C qtip@insecure'''.format(PRIVATE_KEY) + os.system(cmd) + self.keypair['private'] = PRIVATE_KEY + self.keypair['public'] = PUBLIC_KEY + + def pass_keypair_to_remote(self): + results = map(lambda ip: self._pass_keypair(ip, self.keypair['private']), + self.host_ip_list) + + if not (len(results) == len(self.host_ip_list) and False not in results): + raise RuntimeError("Failed on passing keypair to remote.") + + @staticmethod + def _pass_keypair(ip, private_key): + try: + os.system('ssh-keyscan %s >> /root/.ssh/known_hosts' % ip) + time.sleep(2) + ssh_cmd = '%s/qtip_creds.sh %s %s' % (SCRIPT_DIR, ip, private_key) + os.system(ssh_cmd) + logger.info('Pass keypair to remote hosts {0} successfully'.format(ip)) + return True + except Exception as error: + logger.error(error) + return False + def check_hostfile(self, hostfile): + if all_files_exist(hostfile): + self.hostfile = hostfile + else: + raise RuntimeError( + "The hostfile {0} is invalid or not existed.".format(hostfile)) -def clean_file(file_path): - try: - if path.isfile(file_path): - os.remove(file_path) - print("Removed: " + file_path) + def generate_default_hostfile(self): + try: + # check whether the file is already existed + self.check_hostfile(HOST_FILE) + except Exception: + logger.info("Generate default hostfile {0} under " + "{1}".format(HOST_FILE, os.environ['HOME'])) + self._generate_hostfile_via_installer() + + def _generate_hostfile_via_installer(self): + self.hostfile = None + + installer_type = str(os.environ['INSTALLER_TYPE'].lower()) + installer_ip = str(os.environ['INSTALLER_IP']) + + if installer_type not in ["fuel"]: + raise ValueError("{0} is not supported".format(installer_type)) + if not installer_ip: + raise ValueError( + "The value of environment variable INSTALLER_IP is empty.") + + cmd = "bash %s/generate_host_file.sh -t %s -i %s -d %s" % \ + (SCRIPT_DIR, installer_type, installer_ip, HOST_FILE) + os.system(cmd) + + self.hostfile = HOST_FILE + + def fetch_host_ip_from_hostfile(self): + self.host_ip_list = [] + logger.info('Fetch host ips from hostfile...') + with open(self.hostfile, 'r') as f: + self.host_ip_list = re.findall('\d+.\d+.\d+.\d+', f.read()) + if self.host_ip_list: + logger.info("The remote compute nodes: {0}".format(self.host_ip_list)) else: - print("Not exists: " + file_path) - except OSError, error: - print("Not able to Remove: " + file_path, error) + raise ValueError("The hostfile doesn't include host ip addresses.") + + def check_hosts_ssh_connectivity(self): + results = map(lambda ip: self._ssh_is_ok(ip, self.keypair['private']), + self.host_ip_list) + if not (len(results) == len(self.host_ip_list) and False not in results): + raise RuntimeError("Failed on checking hosts ssh connectivity.") + + @staticmethod + def _ssh_is_ok(ip, private_key, attempts=100): + logger.info('Check hosts {0} ssh connectivity...'.format(ip)) + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(ip, key_filename=private_key) + + for attempt in range(attempts): + try: + stdin, stdout, stderr = ssh.exec_command('uname') + if not stderr.readlines(): + logger.info("{0}: SSH test successful.".format(ip)) + return True + except socket.error: + logger.debug("%s times ssh test......failed." % str(attempt + 1)) + if attempt == (attempts - 1): + return False + time.sleep(2) return False - return True + def cleanup(self): + CI_DEBUG = os.getenv('CI_DEBUG') -def clean_keypair(): - flag = True - flag &= clean_file(PRIVATE_KEY) - flag &= clean_file(PUBLIC_KEY) - return flag + if CI_DEBUG is not None and CI_DEBUG.lower() == 'true': + logger.info("DEBUG Mode: please do cleanup by manual.") + else: + with open(self.keypair['public'], 'r') as f: + key = f.read().strip('\n').replace('/', '\/') + if key: + for ip in self.host_ip_list: + logger.info("Cleanup authorized_keys from {0}...".format(ip)) + cmd = '''bash {0}/cleanup_creds.sh {1} {2} "{3}"'''.format( + SCRIPT_DIR, ip, self.keypair['private'], key) + os.system(cmd) + else: + logger.error("Nothing in public key file.") + + logger.info("Cleanup hostfile and keypair.") + clean_file(self.hostfile, + self.keypair['private'], + self.keypair['public']) |