diff options
Diffstat (limited to 'app/install/configure/setup_apex_environment.py')
| -rw-r--r-- | app/install/configure/setup_apex_environment.py | 568 |
1 files changed, 568 insertions, 0 deletions
diff --git a/app/install/configure/setup_apex_environment.py b/app/install/configure/setup_apex_environment.py new file mode 100644 index 0000000..7dc49c5 --- /dev/null +++ b/app/install/configure/setup_apex_environment.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from abc import ABC +from logging.handlers import WatchedFileHandler +import argparse +import json +import logging +import re +import shlex +import subprocess +import sys + + +def run_command(cmd, raise_on_error=False) -> str: + try: + output = subprocess.check_output([cmd], shell=True) + return output.decode('utf-8') + except subprocess.CalledProcessError as e: + error_msg = 'Error running command: {}, output: {}'\ + .format(cmd, e.output.decode('utf-8')) + if raise_on_error: + raise RuntimeError(error_msg) + return msg + + +class Logger(ABC): + DEBUG = 'DEBUG' + INFO = 'INFO' + WARNING = 'WARNING' + ERROR = 'ERROR' + CRITICAL = 'CRITICAL' + + PROJECT_NAME = 'Calipso' + + levels = [DEBUG, INFO, WARNING, ERROR, CRITICAL] + log_format = '%(asctime)s %(levelname)s: %(message)s' + formatter = logging.Formatter(log_format) + default_level = INFO + + def __init__(self, logger_name: str = PROJECT_NAME, + level: str = default_level): + super().__init__() + self.check_level(level) + self.log = logging.getLogger(logger_name) + logging.basicConfig(format=self.log_format, + level=level) + self.log.propagate = False + self.set_loglevel(level) + self.env = None + self.level = level + + def set_env(self, env): + self.env = env + + @staticmethod + def check_level(level): + if level.upper() not in Logger.levels: + raise ValueError('Invalid log level: {}. Supported levels: ({})' + .format(level, ", ".join(Logger.levels))) + + @staticmethod + def get_numeric_level(loglevel): + Logger.check_level(loglevel) + numeric_level = getattr(logging, loglevel.upper(), Logger.default_level) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: {}'.format(loglevel)) + return numeric_level + + def set_loglevel(self, loglevel): + # assuming loglevel is bound to the string value obtained from the + # command line argument. Convert to upper case to allow the user to + # specify --log=DEBUG or --log=debug + numeric_level = self.get_numeric_level(loglevel) + + for handler in self.log.handlers: + handler.setLevel(numeric_level) + self.log.setLevel(numeric_level) + self.level = loglevel + + def _log(self, level, message, *args, exc_info=False, **kwargs): + self.log.log(level, message, *args, exc_info=exc_info, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log(logging.DEBUG, message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log(logging.INFO, message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log(logging.WARNING, message, *args, **kwargs) + + def warn(self, message, *args, **kwargs): + self.warning(message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log(logging.ERROR, message, *args, **kwargs) + + def exception(self, message, *args, **kwargs): + self._log(logging.ERROR, message, exc_info=True, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log(logging.CRITICAL, message, *args, **kwargs) + + def add_handler(self, handler): + handler_defined = handler.__class__ in map(lambda h: h.__class__, + self.log.handlers) + + if not handler_defined: + handler.setLevel(self.level) + handler.setFormatter(self.formatter) + self.log.addHandler(handler) + + +class FileLogger(Logger): + + def __init__(self, log_file: str, level: str = Logger.default_level): + super().__init__(logger_name="{}-File".format(self.PROJECT_NAME), + level=level) + self.add_handler(WatchedFileHandler(log_file)) + + +class ApexEnvironmentFetcher: + + DEFAULTS = { + 'logfile': '/home/calipso/log/apex_environment_fetch.log', + 'mongo_config': '/local_dir/calipso_mongo_access.conf', + 'config_dir': '/home/calipso/Calipso/app/install/db', + 'env': 'Apex-Euphrates', + 'loglevel': 'INFO', + 'git_repo': 'https://git.opnfv.org/calipso', + 'root': False + } + + USER_NAME = 'calipso' + USER_PWD = 'calipso_default' + REPO_LOCAL_NAME = 'Calipso' + INSTALLER = 'python3 app/install/calipso-installer.py --command start-all' + CONFIG_FILE_NAME = 'apex-configuration.conf' + ENV_CONFIG_FILE_NAME = 'apex_environment_config.json' + OVERCLOUDRC_FILE = 'overcloudrc.v3' + SSH_DIR = '/home/calipso/.ssh' + SSH_OPTIONS = '-q -o StrictHostKeyChecking=no' + UNDERCLOUD_KEY_FILE = 'uc-id_rsa' + UNDERCLOUD_PUBLIC_KEY_FILE = '{}/uc-id_rsa.pub'.format(SSH_DIR) + OVERCLOUD_USER = 'heat-admin' + OVERCLOUD_KEY_FILE = 'oc-id_rsa' + MOUNT_SSH_DIR = '/local_dir/.ssh' + OVERCLOUD_KEYSTONE_CONF = 'oc-keystone.conf' + OVERCLOUD_ML2_CONF = 'overcloud_ml2_conf.ini' + OVERCLOUD_RABBITMQ_CONF = 'overcloud_rabbitmq_conf.ini' + + def __init__(self): + self.args = self.get_args() + self.log = None + self.config_file = '{}/{}'.format(self.args.config_dir, + self.CONFIG_FILE_NAME) + self.env_config_file = '{}/{}'.format(self.args.config_dir, + self.ENV_CONFIG_FILE_NAME) + self.undercloud_user = 'root' + self.undercloud_host = '192.0.2.1' + self.undercloud_key = '{}/{}'.format(self.SSH_DIR, + self.UNDERCLOUD_KEY_FILE) + self.overcloud_config_file = '{}/{}'\ + .format(self.args.config_dir, self.OVERCLOUDRC_FILE) + self.overcloud_key = '{}/{}'.format(self.SSH_DIR, + self.OVERCLOUD_KEY_FILE) + self.overcloud_key_container = '{}/{}'.format(self.MOUNT_SSH_DIR, + self.OVERCLOUD_KEY_FILE) + self.undercloud_ip = None + self.overcloud_ip = None + self.conf_lines = {} + self.env_config = None + + def get_args(self): + # try to read scan plan from command line parameters + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--mongo_config', nargs='?', type=str, + default=self.DEFAULTS['mongo_config'], + help='name of config file ' + + 'with MongoDB server access details\n' + '(Default: {})' + .format(self.DEFAULTS['mongo_config'])) + parser.add_argument('-d', '--config_dir', nargs='?', type=str, + default=self.DEFAULTS['config_dir'], + help='path to directory with config data\n' + '(Default: {})' + .format(self.DEFAULTS['config_dir'])) + parser.add_argument('-a', '--apex', nargs='?', type=str, + help='name of environment to Apex host') + parser.add_argument('-e', '--env', nargs='?', type=str, + default=self.DEFAULTS['env'], + help='name of environment to create' + '(Default: {})' + .format(self.DEFAULTS['env'])) + parser.add_argument('-l', '--loglevel', nargs='?', type=str, + default=self.DEFAULTS['loglevel'], + help='logging level \n(default: "{}")' + .format(self.DEFAULTS['loglevel'])) + parser.add_argument('-f', '--logfile', nargs='?', type=str, + default=self.DEFAULTS['logfile'], + help='log file \n(default: "{}")' + .format(self.DEFAULTS['logfile'])) + parser.add_argument('-g', '--git', nargs='?', type=str, + help='URL to clone Git repository\n(default: {})' + .format(self.DEFAULTS['git_repo']), + default=self.DEFAULTS['git_repo']) + parser.add_argument('--root', dest='root', action='store_true') + parser.add_argument('--no-root', dest='root', action='store_false') + parser.set_defaults(root=False) + return parser.parse_args() + + @staticmethod + def run_cmd(cmd: str ='', use_sudo=True, as_user=None): + sudo_prefix = '' if not use_sudo \ + else 'sudo {} '.format(as_user if as_user else '') + command = '{}{}'.format(sudo_prefix, cmd) + output = run_command(cmd=command, raise_on_error=True) + return output + + def get_undercloud_ip(self): + output = self.run_cmd('ifconfig br-admin') + lines = output.splitlines() + if not lines or len(lines) < 2: + self.log.error('Unable to feth inet address, output: {}' + .format(output)) + return + inet_parts = lines[1].split() + inet_address = inet_parts[1] + return inet_address + + def get_overcloud_ip(self): + with open('{}'.format(self.overcloud_config_file)) as rc_file: + lines = rc_file.readlines() + no_proxy_line = [l for l in lines if 'no_proxy=' in l] + no_proxy_line = no_proxy_line[0] + value = no_proxy_line[no_proxy_line.index('=')+2:] + parts = value.strip().split(',') + inet_address = parts[-1] + return inet_address + + def set_ssh_dir(self): + self.run_cmd('mkdir -p {}'.format(self.SSH_DIR)) + # will be used to access undercloud VM + self.run_cmd('cp /root/.ssh/id_rsa {}'.format(self.undercloud_key)) + self.run_cmd('cp /root/.ssh/id_rsa.pub {}' + .format(self.UNDERCLOUD_PUBLIC_KEY_FILE)) + self.run_cmd('chown calipso.calipso {}/uc-id_rsa*'.format(self.SSH_DIR)) + self.copy_undercloud_file('/home/stack/.ssh/id_rsa', + local_dir=self.SSH_DIR, + local_name=self.OVERCLOUD_KEY_FILE) + self.copy_undercloud_file('/home/stack/.ssh/id_rsa.pub', + local_dir=self.SSH_DIR, + local_name='oc-id_rsa.pub') + self.run_cmd('chown calipso.calipso {}/oc-id_rsa*'.format(self.SSH_DIR)) + + def copy_undercloud_file(self, file_path, local_dir=None, local_name=None): + cmd = 'scp {} -i {} {}@{}:{} {}/{}' \ + .format(self.SSH_OPTIONS, + self.undercloud_key, + self.undercloud_user, self.undercloud_host, + file_path, + local_dir if local_dir else self.args.config_dir, + local_name if local_name else '') + self.run_cmd(cmd) + + def copy_undercloud_conf_file(self, file_name, local_name=None): + self.copy_undercloud_file('/home/stack/{}'.format(file_name), + local_name) + + def get_undercloud_setup(self): + self.copy_undercloud_conf_file('undercloud.conf') + self.copy_undercloud_conf_file('opnfv-environment.yaml') + self.copy_undercloud_conf_file('overcloudrc') + self.copy_undercloud_conf_file('stackrc') + self.copy_undercloud_conf_file('overcloudrc.v3') + self.copy_undercloud_conf_file('deploy_command') + self.copy_undercloud_conf_file('apex-undercloud-install.log') + self.copy_undercloud_conf_file('undercloud-passwords.conf') + self.copy_undercloud_file('/etc/keystone/keystone.conf', + local_name='uc-keystone.conf') + self.run_cmd('mkdir -p {}/deploy_logs'.format(self.args.config_dir)) + self.copy_undercloud_file('/home/stack/deploy_logs/*', + local_name='deploy_logs/') + + def fetch_conf_file(self, file_name, target_file, lines_property=None): + conf = \ + self.run_cmd('ssh -i {} {} {}@{} ' + 'sudo grep -v "^#" {}' + .format(self.overcloud_key, + self.SSH_OPTIONS, + self.OVERCLOUD_USER, + self.overcloud_ip, + file_name)) + conf_file_path = '{}/{}'.format(self.args.config_dir, target_file) + if lines_property: + self.conf_lines[lines_property] = conf.splitlines() + with open(conf_file_path, 'w') as conf_file: + conf_file.write(conf) + + def fetch_keystone_conf(self): + self.fetch_conf_file('/etc/keystone/keystone.conf', + self.OVERCLOUD_KEYSTONE_CONF, + lines_property='keystone_conf') + + def fetch_ml2_conf(self): + self.fetch_conf_file('/etc/neutron/plugins/ml2/ml2_conf.ini', + self.OVERCLOUD_ML2_CONF, + lines_property='ml2_conf') + + def fetch_rabbitmq_conf(self): + self.fetch_conf_file('/etc/rabbitmq/rabbitmq.config', + self.OVERCLOUD_RABBITMQ_CONF, + lines_property='rabbitmq_conf') + + def copy_local_file_to_overcloud(self, local_file, remote_file_path, + local_dir=None): + source_dir = local_dir if local_dir else self.args.config_dir + local_file_path = '{}/{}'.format(source_dir, local_file) + cmd = 'scp {} -i {} {} {}@{}:{}' \ + .format(self.SSH_OPTIONS, + self.overcloud_key, + local_file_path, + self.OVERCLOUD_USER, self.overcloud_ip, + remote_file_path) + self.run_cmd(cmd) + + def get_overcloud_keys(self): + remote_ssh_dir = '/home/{}/.ssh'.format(self.OVERCLOUD_USER) + remote_private_key = '{}/id_rsa'.format(remote_ssh_dir) + self.copy_local_file_to_overcloud(self.OVERCLOUD_KEY_FILE, + remote_private_key, + local_dir=self.SSH_DIR) + public_key = '{}.pub'.format(self.OVERCLOUD_KEY_FILE) + remote_public_key = '{}/id_rsa.pub'.format(remote_ssh_dir) + self.copy_local_file_to_overcloud(public_key, remote_public_key, + local_dir=self.SSH_DIR) + + def get_overcloud_setup(self): + self.get_overcloud_keys() + self.fetch_keystone_conf() + self.fetch_ml2_conf() + self.fetch_rabbitmq_conf() + + def get_value_from_file(self, file_attr, attr, regex=None, separator='='): + line_prefix = 'export ' if separator == '=' else '' + prefix = '{}{}{}'.format(line_prefix, attr, separator) + lines = self.conf_lines.get(file_attr, {}) + matches = [l for l in lines if l.startswith(prefix)] + if not matches: + self.log.error('failed to find attribute {}'.format(attr)) + return '' + line = matches[0].strip() + value = line[line.index(separator)+len(separator):] + if not regex: + return value + matches = re.search(regex, value) + if not matches: + return '' + match = matches.group(1) + return match + + def get_value_from_rc_file(self, lines, attr, regex=None): + return self.get_value_from_file(lines, attr, regex=regex) + + def get_api_config(self): + with open('{}'.format(self.overcloud_config_file)) as rc_file: + self.conf_lines['overcloudrc'] = rc_file.readlines() + api_config = { + 'name': 'OpenStack', + 'host': self.overcloud_ip, + 'port': self.get_value_from_rc_file('overcloudrc', + 'OS_AUTH_URL', + regex=':(\d+)/'), + 'user': self.get_value_from_rc_file('overcloudrc', 'OS_USERNAME'), + 'pwd': self.get_value_from_rc_file('overcloudrc', 'OS_PASSWORD'), + 'admin_token': self.get_value_from_file('keystone_conf', + 'admin_token', + separator=' = ') + } + return api_config + + def run_command_on_overcloud(self, cmd): + output = \ + self.run_cmd('ssh -i {} {} {}@{} {}' + .format(self.overcloud_key, + self.SSH_OPTIONS, + self.OVERCLOUD_USER, + self.overcloud_ip, + shlex.quote(cmd))) + return output + + def create_mysql_user(self, host, pwd): + mysql_file_name = '/tmp/create_user.sql' + # create calipso MySQL user with access from jump host to all tables + echo_cmd = "echo \"GRANT ALL PRIVILEGES ON *.* " \ + "TO 'calipso'@'{}' " \ + "IDENTIFIED BY '{}'; " \ + "FLUSH PRIVILEGES;\" > {}"\ + .format(host, pwd, mysql_file_name) + self.run_command_on_overcloud(echo_cmd) + run_mysql_cmd = 'sudo mysql < {}'.format(mysql_file_name) + self.run_command_on_overcloud(run_mysql_cmd) + remove_file_cmd = 'rm {}'.format(mysql_file_name) + self.run_command_on_overcloud(remove_file_cmd) + return pwd + + def get_mysql_config(self): + pwd = self.run_cmd('openssl rand -base64 18').strip() + self.create_mysql_user(self.undercloud_ip, pwd) + pwd = self.create_mysql_user(self.overcloud_ip, pwd) + mysql_config = { + 'name': 'mysql', + 'host': self.overcloud_ip, + 'port': '3306', + 'user': 'calipso', + 'pwd': pwd + } + return mysql_config + + def get_cli_config(self): + return { + 'name': 'CLI', + 'host': self.overcloud_ip, + 'user': self.OVERCLOUD_USER, + 'key': self.overcloud_key_container + } + + def get_amqp_config(self): + user = self.get_value_from_file('rabbitmq_conf', + ' {default_user', + separator=',', + regex='"(.+)"') + pwd = self.get_value_from_file('rabbitmq_conf', + ' {default_pass', + separator=',', + regex='"(.+)"') + port = self.get_value_from_file('rabbitmq_conf', + ' {tcp_listeners', + separator=',', + regex=', (\d+)') + port = int(port) + return { + 'name': 'AMQP', + 'host': self.overcloud_ip, + 'port': port, + 'user': user, + 'pwd': pwd + } + + def get_monitoring_config(self): + return { + 'name': 'Monitoring', + 'config_folder': '/local_dir/sensu_config', + 'env_type': 'production', + 'rabbitmq_port': '5671', + 'rabbitmq_user': 'sensu', + 'server_ip': self.undercloud_ip, + 'server_name': 'sensu_server', + 'type': 'Sensu', + 'provision': 'None', + 'ssh_port': '20022', + 'ssh_user': 'root', + 'ssh_password': 'osdna', + 'api_port': 4567, + 'rabbitmq_pass': 'osdna' + } + + def prepare_env_configuration_array(self): + config_array = [ + self.get_api_config(), + self.get_mysql_config(), + self.get_cli_config(), + self.get_amqp_config(), + self.get_monitoring_config() + ] + self.env_config['configuration'] = config_array + + UI_USER = 'wNLeBJxNDyw8G7Ssg' + + def add_env_ui_conf(self): + self.env_config.update({ + 'user': self.UI_USER, + 'auth': { + 'view-env': [self.UI_USER], + 'edit-env': [self.UI_USER] + } + }) + + def get_mechanism_driver(self): + driver = self.get_value_from_file('ml2_conf', 'mechanism_drivers', + separator=' =') + return 'OVS' if driver == 'openvswitch' else driver + + def set_env_level_attributes(self): + self.env_config.update({ + 'distribution': 'Apex', + 'distribution_version': 'Euphrates', + 'type_drivers': self.get_value_from_file('ml2_conf', + 'tenant_network_types', + separator=' = '), + 'mechanism_drivers': [self.get_mechanism_driver()], + "operational": "running", + "scanned": False, + "type": "environment", + "app_path": "/home/scan/calipso_prod/app", + "listen": True, + "enable_monitoring": True, + "aci_enabled": False, + "last_scanned": "", + "monitoring_setup_done": False + }) + + def prepare_env_config(self): + self.prepare_env_configuration_array() + self.set_env_level_attributes() + self.add_env_ui_conf() + config_dump = json.dumps(self.env_config, sort_keys=True, indent=4, + separators=(',', ': ')) + with open(self.env_config_file, 'w') as config_file: + config_file.write(config_dump) + + def setup_environment_config(self, config_file): + self.run_cmd('mkdir -p {}'.format(self.args.config_dir)) + self.env_config = {'name': self.args.env} + self.undercloud_ip = self.get_undercloud_ip() + config_file.write('jumphost_admin_ip {}\n'.format(self.undercloud_ip)) + self.set_ssh_dir() + self.get_undercloud_setup() + self.overcloud_ip = self.get_overcloud_ip() + config_file.write('overcloud_admin_ip {}\n'.format(self.overcloud_ip)) + self.get_overcloud_setup() + # now get correct IP of overcloud from RabbitMQ setup + self.overcloud_ip = self.get_value_from_file('rabbitmq_conf', + ' {tcp_listeners', + regex='"(.*)"', + separator=',') + self.prepare_env_config() + + def get(self): + try: + print('Fetching Apex environment settings') + self.log = FileLogger(self.args.logfile) + self.run_cmd('mkdir -p {}'.format(self.args.config_dir)) + with open(self.config_file, 'w') as config_file: + self.setup_environment_config(config_file) + print('Finished fetching Apex environment settings') + return True, 'Environment setup finished successfully' + except RuntimeError as e: + return False, str(e) + + +if __name__ == '__main__': + fetcher = ApexEnvironmentFetcher() + ret, msg = fetcher.get() + if not ret: + if fetcher.log: + fetcher.log.error(msg) + else: + print(msg) + sys.exit(0 if ret else 1) |