From d32f75145676bacefde0d08a14680a5984623451 Mon Sep 17 00:00:00 2001 From: Koren Lev Date: Fri, 29 Sep 2017 01:38:18 +0300 Subject: release 1.0 calipso for opnfv apex Change-Id: I3e63cd27c5f4d3756e67a07c749863a68e84dde2 Signed-off-by: Koren Lev --- app/install/calipso-installer.py | 55 ++- app/install/configure/setup_apex_environment.py | 568 ++++++++++++++++++++++++ app/install/db/apex_environment_config.json | 3 + app/install/db/api_tokens.json | 7 + app/install/db/connection_tests.json | 156 +++++++ app/install/db/constants.json | 33 ++ app/install/db/environments_config.json | 10 +- app/install/db/supported_environments.json | 428 +++++++----------- app/install/db/user_settings.json | 4 + app/install/ldap.conf.example | 2 +- 10 files changed, 981 insertions(+), 285 deletions(-) create mode 100644 app/install/configure/setup_apex_environment.py create mode 100644 app/install/db/apex_environment_config.json create mode 100644 app/install/db/api_tokens.json create mode 100644 app/install/db/connection_tests.json create mode 100644 app/install/db/user_settings.json (limited to 'app/install') diff --git a/app/install/calipso-installer.py b/app/install/calipso-installer.py index 523a838..c2b8579 100644 --- a/app/install/calipso-installer.py +++ b/app/install/calipso-installer.py @@ -176,6 +176,10 @@ def start_mongo(dbport, copy): copy_file("scheduled_scans") copy_file("statistics") copy_file("supported_environments") + copy_file("connection_tests") + copy_file("api_tokens") + copy_file("user_settings") + copy_file("apex_environment_config") # note : 'messages', 'roles', 'users' and some of the 'constants' # are filled by calipso-ui at runtime @@ -216,14 +220,14 @@ def start_ldap(): volumes=calipso_volume) -def start_api(): +def start_api(apiport): name = "calipso-api" if container_started(name): return print("\nstarting container {}...\n".format(name)) image_name = "korenlev/calipso:api" download_image(image_name) - api_ports = {'8000/tcp': 8000, '22/tcp': 40022} + api_ports = {'8000/tcp': apiport, '22/tcp': 40022} DockerClient.containers.run(image_name, detach=True, name=name, @@ -252,15 +256,15 @@ def start_scan(): volumes=calipso_volume) -def start_sensu(): +def start_sensu(uchiwaport, sensuport, rabbitport, rabbitmport): name = "calipso-sensu" if container_started(name): return print("\nstarting container {}...\n".format(name)) image_name = "korenlev/calipso:sensu" download_image(image_name) - sensu_ports = {'22/tcp': 20022, '3000/tcp': 3000, '4567/tcp': 4567, - '5671/tcp': 5671, '15672/tcp': 15672} + sensu_ports = {'22/tcp': 20022, '3000/tcp': uchiwaport, '4567/tcp': sensuport, + '5671/tcp': rabbitport, '15672/tcp': rabbitmport} DockerClient.containers.run(image_name, detach=True, name=name, @@ -326,6 +330,36 @@ parser.add_argument("--dbport", type=int, default="27017", required=False) +parser.add_argument("--apiport", + help="Port for the Calipso API " + "(default=8000)", + type=int, + default="8000", + required=False) +parser.add_argument("--uchiwaport", + help="Port for the Calipso Uchiwa " + "(default=3000)", + type=int, + default="3000", + required=False) +parser.add_argument("--rabbitmport", + help="Port for the Calipso Sensu RabbitMQ Managment " + "(default=15672)", + type=int, + default="15672", + required=False) +parser.add_argument("--sensuport", + help="Port for the Calipso Sensu-api " + "(default=4567)", + type=int, + default="4567", + required=False) +parser.add_argument("--rabbitport", + help="Port for the Calipso Sensu RabbitMQ " + "(default=5671)", + type=int, + default="5671", + required=False) parser.add_argument("--dbuser", help="User for the Calipso MongoDB " "(default=calipso)", @@ -339,14 +373,14 @@ parser.add_argument("--dbpassword", default="calipso_default", required=False) parser.add_argument("--command", - help="'start-all' or 'stop-all' the calipso containers " + help="'start-all' or 'stop-all' the Calipso containers " "(default=None)", type=str, default=None, required=False) parser.add_argument("--copy", help="'c' to copy json files from 'db' folder to mongoDB, 'q' to skip copy of files " - "(default=q)", + "(default=None)", type=str, default=None, required=False) @@ -382,9 +416,10 @@ if action == "start": calipso_mongo_access_text = \ "server {}\n" \ "user {}\n" \ + "port {}\n" \ "pwd {}\n" \ "auth_db calipso" \ - .format(args.hostname, args.dbuser, args.dbpassword) + .format(args.hostname, args.dbuser, args.dbport, args.dbpassword) LDAP_PWD_ATTRIBUTE = "password password" LDAP_USER_PWD_ATTRIBUTE = "userpassword" ldap_text = \ @@ -421,13 +456,13 @@ if action == "start": start_ldap() time.sleep(1) if container == "calipso-api" or container == "all": - start_api() + start_api(args.apiport) time.sleep(1) if container == "calipso-scan" or container == "all": start_scan() time.sleep(1) if container == "calipso-sensu" or container == "all": - start_sensu() + start_sensu(args.uchiwaport, args.sensuport, args.rabbitport, args.rabbitmport) time.sleep(1) if container == "calipso-ui" or container == "all": start_ui(args.hostname, args.dbuser, args.dbpassword, args.webport, diff --git a/app/install/configure/setup_apex_environment.py b/app/install/configure/setup_apex_environment.py new file mode 100644 index 0000000..7dc49c5 --- /dev/null +++ b/app/install/configure/setup_apex_environment.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python3 +############################################################################### +# Copyright (c) 2017 Koren Lev (Cisco Systems), Yaron Yogev (Cisco Systems) # +# and others # +# # +# All rights reserved. This program and the accompanying materials # +# are made available under the terms of the Apache License, Version 2.0 # +# which accompanies this distribution, and is available at # +# http://www.apache.org/licenses/LICENSE-2.0 # +############################################################################### +from abc import ABC +from logging.handlers import WatchedFileHandler +import argparse +import json +import logging +import re +import shlex +import subprocess +import sys + + +def run_command(cmd, raise_on_error=False) -> str: + try: + output = subprocess.check_output([cmd], shell=True) + return output.decode('utf-8') + except subprocess.CalledProcessError as e: + error_msg = 'Error running command: {}, output: {}'\ + .format(cmd, e.output.decode('utf-8')) + if raise_on_error: + raise RuntimeError(error_msg) + return msg + + +class Logger(ABC): + DEBUG = 'DEBUG' + INFO = 'INFO' + WARNING = 'WARNING' + ERROR = 'ERROR' + CRITICAL = 'CRITICAL' + + PROJECT_NAME = 'Calipso' + + levels = [DEBUG, INFO, WARNING, ERROR, CRITICAL] + log_format = '%(asctime)s %(levelname)s: %(message)s' + formatter = logging.Formatter(log_format) + default_level = INFO + + def __init__(self, logger_name: str = PROJECT_NAME, + level: str = default_level): + super().__init__() + self.check_level(level) + self.log = logging.getLogger(logger_name) + logging.basicConfig(format=self.log_format, + level=level) + self.log.propagate = False + self.set_loglevel(level) + self.env = None + self.level = level + + def set_env(self, env): + self.env = env + + @staticmethod + def check_level(level): + if level.upper() not in Logger.levels: + raise ValueError('Invalid log level: {}. Supported levels: ({})' + .format(level, ", ".join(Logger.levels))) + + @staticmethod + def get_numeric_level(loglevel): + Logger.check_level(loglevel) + numeric_level = getattr(logging, loglevel.upper(), Logger.default_level) + if not isinstance(numeric_level, int): + raise ValueError('Invalid log level: {}'.format(loglevel)) + return numeric_level + + def set_loglevel(self, loglevel): + # assuming loglevel is bound to the string value obtained from the + # command line argument. Convert to upper case to allow the user to + # specify --log=DEBUG or --log=debug + numeric_level = self.get_numeric_level(loglevel) + + for handler in self.log.handlers: + handler.setLevel(numeric_level) + self.log.setLevel(numeric_level) + self.level = loglevel + + def _log(self, level, message, *args, exc_info=False, **kwargs): + self.log.log(level, message, *args, exc_info=exc_info, **kwargs) + + def debug(self, message, *args, **kwargs): + self._log(logging.DEBUG, message, *args, **kwargs) + + def info(self, message, *args, **kwargs): + self._log(logging.INFO, message, *args, **kwargs) + + def warning(self, message, *args, **kwargs): + self._log(logging.WARNING, message, *args, **kwargs) + + def warn(self, message, *args, **kwargs): + self.warning(message, *args, **kwargs) + + def error(self, message, *args, **kwargs): + self._log(logging.ERROR, message, *args, **kwargs) + + def exception(self, message, *args, **kwargs): + self._log(logging.ERROR, message, exc_info=True, *args, **kwargs) + + def critical(self, message, *args, **kwargs): + self._log(logging.CRITICAL, message, *args, **kwargs) + + def add_handler(self, handler): + handler_defined = handler.__class__ in map(lambda h: h.__class__, + self.log.handlers) + + if not handler_defined: + handler.setLevel(self.level) + handler.setFormatter(self.formatter) + self.log.addHandler(handler) + + +class FileLogger(Logger): + + def __init__(self, log_file: str, level: str = Logger.default_level): + super().__init__(logger_name="{}-File".format(self.PROJECT_NAME), + level=level) + self.add_handler(WatchedFileHandler(log_file)) + + +class ApexEnvironmentFetcher: + + DEFAULTS = { + 'logfile': '/home/calipso/log/apex_environment_fetch.log', + 'mongo_config': '/local_dir/calipso_mongo_access.conf', + 'config_dir': '/home/calipso/Calipso/app/install/db', + 'env': 'Apex-Euphrates', + 'loglevel': 'INFO', + 'git_repo': 'https://git.opnfv.org/calipso', + 'root': False + } + + USER_NAME = 'calipso' + USER_PWD = 'calipso_default' + REPO_LOCAL_NAME = 'Calipso' + INSTALLER = 'python3 app/install/calipso-installer.py --command start-all' + CONFIG_FILE_NAME = 'apex-configuration.conf' + ENV_CONFIG_FILE_NAME = 'apex_environment_config.json' + OVERCLOUDRC_FILE = 'overcloudrc.v3' + SSH_DIR = '/home/calipso/.ssh' + SSH_OPTIONS = '-q -o StrictHostKeyChecking=no' + UNDERCLOUD_KEY_FILE = 'uc-id_rsa' + UNDERCLOUD_PUBLIC_KEY_FILE = '{}/uc-id_rsa.pub'.format(SSH_DIR) + OVERCLOUD_USER = 'heat-admin' + OVERCLOUD_KEY_FILE = 'oc-id_rsa' + MOUNT_SSH_DIR = '/local_dir/.ssh' + OVERCLOUD_KEYSTONE_CONF = 'oc-keystone.conf' + OVERCLOUD_ML2_CONF = 'overcloud_ml2_conf.ini' + OVERCLOUD_RABBITMQ_CONF = 'overcloud_rabbitmq_conf.ini' + + def __init__(self): + self.args = self.get_args() + self.log = None + self.config_file = '{}/{}'.format(self.args.config_dir, + self.CONFIG_FILE_NAME) + self.env_config_file = '{}/{}'.format(self.args.config_dir, + self.ENV_CONFIG_FILE_NAME) + self.undercloud_user = 'root' + self.undercloud_host = '192.0.2.1' + self.undercloud_key = '{}/{}'.format(self.SSH_DIR, + self.UNDERCLOUD_KEY_FILE) + self.overcloud_config_file = '{}/{}'\ + .format(self.args.config_dir, self.OVERCLOUDRC_FILE) + self.overcloud_key = '{}/{}'.format(self.SSH_DIR, + self.OVERCLOUD_KEY_FILE) + self.overcloud_key_container = '{}/{}'.format(self.MOUNT_SSH_DIR, + self.OVERCLOUD_KEY_FILE) + self.undercloud_ip = None + self.overcloud_ip = None + self.conf_lines = {} + self.env_config = None + + def get_args(self): + # try to read scan plan from command line parameters + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--mongo_config', nargs='?', type=str, + default=self.DEFAULTS['mongo_config'], + help='name of config file ' + + 'with MongoDB server access details\n' + '(Default: {})' + .format(self.DEFAULTS['mongo_config'])) + parser.add_argument('-d', '--config_dir', nargs='?', type=str, + default=self.DEFAULTS['config_dir'], + help='path to directory with config data\n' + '(Default: {})' + .format(self.DEFAULTS['config_dir'])) + parser.add_argument('-a', '--apex', nargs='?', type=str, + help='name of environment to Apex host') + parser.add_argument('-e', '--env', nargs='?', type=str, + default=self.DEFAULTS['env'], + help='name of environment to create' + '(Default: {})' + .format(self.DEFAULTS['env'])) + parser.add_argument('-l', '--loglevel', nargs='?', type=str, + default=self.DEFAULTS['loglevel'], + help='logging level \n(default: "{}")' + .format(self.DEFAULTS['loglevel'])) + parser.add_argument('-f', '--logfile', nargs='?', type=str, + default=self.DEFAULTS['logfile'], + help='log file \n(default: "{}")' + .format(self.DEFAULTS['logfile'])) + parser.add_argument('-g', '--git', nargs='?', type=str, + help='URL to clone Git repository\n(default: {})' + .format(self.DEFAULTS['git_repo']), + default=self.DEFAULTS['git_repo']) + parser.add_argument('--root', dest='root', action='store_true') + parser.add_argument('--no-root', dest='root', action='store_false') + parser.set_defaults(root=False) + return parser.parse_args() + + @staticmethod + def run_cmd(cmd: str ='', use_sudo=True, as_user=None): + sudo_prefix = '' if not use_sudo \ + else 'sudo {} '.format(as_user if as_user else '') + command = '{}{}'.format(sudo_prefix, cmd) + output = run_command(cmd=command, raise_on_error=True) + return output + + def get_undercloud_ip(self): + output = self.run_cmd('ifconfig br-admin') + lines = output.splitlines() + if not lines or len(lines) < 2: + self.log.error('Unable to feth inet address, output: {}' + .format(output)) + return + inet_parts = lines[1].split() + inet_address = inet_parts[1] + return inet_address + + def get_overcloud_ip(self): + with open('{}'.format(self.overcloud_config_file)) as rc_file: + lines = rc_file.readlines() + no_proxy_line = [l for l in lines if 'no_proxy=' in l] + no_proxy_line = no_proxy_line[0] + value = no_proxy_line[no_proxy_line.index('=')+2:] + parts = value.strip().split(',') + inet_address = parts[-1] + return inet_address + + def set_ssh_dir(self): + self.run_cmd('mkdir -p {}'.format(self.SSH_DIR)) + # will be used to access undercloud VM + self.run_cmd('cp /root/.ssh/id_rsa {}'.format(self.undercloud_key)) + self.run_cmd('cp /root/.ssh/id_rsa.pub {}' + .format(self.UNDERCLOUD_PUBLIC_KEY_FILE)) + self.run_cmd('chown calipso.calipso {}/uc-id_rsa*'.format(self.SSH_DIR)) + self.copy_undercloud_file('/home/stack/.ssh/id_rsa', + local_dir=self.SSH_DIR, + local_name=self.OVERCLOUD_KEY_FILE) + self.copy_undercloud_file('/home/stack/.ssh/id_rsa.pub', + local_dir=self.SSH_DIR, + local_name='oc-id_rsa.pub') + self.run_cmd('chown calipso.calipso {}/oc-id_rsa*'.format(self.SSH_DIR)) + + def copy_undercloud_file(self, file_path, local_dir=None, local_name=None): + cmd = 'scp {} -i {} {}@{}:{} {}/{}' \ + .format(self.SSH_OPTIONS, + self.undercloud_key, + self.undercloud_user, self.undercloud_host, + file_path, + local_dir if local_dir else self.args.config_dir, + local_name if local_name else '') + self.run_cmd(cmd) + + def copy_undercloud_conf_file(self, file_name, local_name=None): + self.copy_undercloud_file('/home/stack/{}'.format(file_name), + local_name) + + def get_undercloud_setup(self): + self.copy_undercloud_conf_file('undercloud.conf') + self.copy_undercloud_conf_file('opnfv-environment.yaml') + self.copy_undercloud_conf_file('overcloudrc') + self.copy_undercloud_conf_file('stackrc') + self.copy_undercloud_conf_file('overcloudrc.v3') + self.copy_undercloud_conf_file('deploy_command') + self.copy_undercloud_conf_file('apex-undercloud-install.log') + self.copy_undercloud_conf_file('undercloud-passwords.conf') + self.copy_undercloud_file('/etc/keystone/keystone.conf', + local_name='uc-keystone.conf') + self.run_cmd('mkdir -p {}/deploy_logs'.format(self.args.config_dir)) + self.copy_undercloud_file('/home/stack/deploy_logs/*', + local_name='deploy_logs/') + + def fetch_conf_file(self, file_name, target_file, lines_property=None): + conf = \ + self.run_cmd('ssh -i {} {} {}@{} ' + 'sudo grep -v "^#" {}' + .format(self.overcloud_key, + self.SSH_OPTIONS, + self.OVERCLOUD_USER, + self.overcloud_ip, + file_name)) + conf_file_path = '{}/{}'.format(self.args.config_dir, target_file) + if lines_property: + self.conf_lines[lines_property] = conf.splitlines() + with open(conf_file_path, 'w') as conf_file: + conf_file.write(conf) + + def fetch_keystone_conf(self): + self.fetch_conf_file('/etc/keystone/keystone.conf', + self.OVERCLOUD_KEYSTONE_CONF, + lines_property='keystone_conf') + + def fetch_ml2_conf(self): + self.fetch_conf_file('/etc/neutron/plugins/ml2/ml2_conf.ini', + self.OVERCLOUD_ML2_CONF, + lines_property='ml2_conf') + + def fetch_rabbitmq_conf(self): + self.fetch_conf_file('/etc/rabbitmq/rabbitmq.config', + self.OVERCLOUD_RABBITMQ_CONF, + lines_property='rabbitmq_conf') + + def copy_local_file_to_overcloud(self, local_file, remote_file_path, + local_dir=None): + source_dir = local_dir if local_dir else self.args.config_dir + local_file_path = '{}/{}'.format(source_dir, local_file) + cmd = 'scp {} -i {} {} {}@{}:{}' \ + .format(self.SSH_OPTIONS, + self.overcloud_key, + local_file_path, + self.OVERCLOUD_USER, self.overcloud_ip, + remote_file_path) + self.run_cmd(cmd) + + def get_overcloud_keys(self): + remote_ssh_dir = '/home/{}/.ssh'.format(self.OVERCLOUD_USER) + remote_private_key = '{}/id_rsa'.format(remote_ssh_dir) + self.copy_local_file_to_overcloud(self.OVERCLOUD_KEY_FILE, + remote_private_key, + local_dir=self.SSH_DIR) + public_key = '{}.pub'.format(self.OVERCLOUD_KEY_FILE) + remote_public_key = '{}/id_rsa.pub'.format(remote_ssh_dir) + self.copy_local_file_to_overcloud(public_key, remote_public_key, + local_dir=self.SSH_DIR) + + def get_overcloud_setup(self): + self.get_overcloud_keys() + self.fetch_keystone_conf() + self.fetch_ml2_conf() + self.fetch_rabbitmq_conf() + + def get_value_from_file(self, file_attr, attr, regex=None, separator='='): + line_prefix = 'export ' if separator == '=' else '' + prefix = '{}{}{}'.format(line_prefix, attr, separator) + lines = self.conf_lines.get(file_attr, {}) + matches = [l for l in lines if l.startswith(prefix)] + if not matches: + self.log.error('failed to find attribute {}'.format(attr)) + return '' + line = matches[0].strip() + value = line[line.index(separator)+len(separator):] + if not regex: + return value + matches = re.search(regex, value) + if not matches: + return '' + match = matches.group(1) + return match + + def get_value_from_rc_file(self, lines, attr, regex=None): + return self.get_value_from_file(lines, attr, regex=regex) + + def get_api_config(self): + with open('{}'.format(self.overcloud_config_file)) as rc_file: + self.conf_lines['overcloudrc'] = rc_file.readlines() + api_config = { + 'name': 'OpenStack', + 'host': self.overcloud_ip, + 'port': self.get_value_from_rc_file('overcloudrc', + 'OS_AUTH_URL', + regex=':(\d+)/'), + 'user': self.get_value_from_rc_file('overcloudrc', 'OS_USERNAME'), + 'pwd': self.get_value_from_rc_file('overcloudrc', 'OS_PASSWORD'), + 'admin_token': self.get_value_from_file('keystone_conf', + 'admin_token', + separator=' = ') + } + return api_config + + def run_command_on_overcloud(self, cmd): + output = \ + self.run_cmd('ssh -i {} {} {}@{} {}' + .format(self.overcloud_key, + self.SSH_OPTIONS, + self.OVERCLOUD_USER, + self.overcloud_ip, + shlex.quote(cmd))) + return output + + def create_mysql_user(self, host, pwd): + mysql_file_name = '/tmp/create_user.sql' + # create calipso MySQL user with access from jump host to all tables + echo_cmd = "echo \"GRANT ALL PRIVILEGES ON *.* " \ + "TO 'calipso'@'{}' " \ + "IDENTIFIED BY '{}'; " \ + "FLUSH PRIVILEGES;\" > {}"\ + .format(host, pwd, mysql_file_name) + self.run_command_on_overcloud(echo_cmd) + run_mysql_cmd = 'sudo mysql < {}'.format(mysql_file_name) + self.run_command_on_overcloud(run_mysql_cmd) + remove_file_cmd = 'rm {}'.format(mysql_file_name) + self.run_command_on_overcloud(remove_file_cmd) + return pwd + + def get_mysql_config(self): + pwd = self.run_cmd('openssl rand -base64 18').strip() + self.create_mysql_user(self.undercloud_ip, pwd) + pwd = self.create_mysql_user(self.overcloud_ip, pwd) + mysql_config = { + 'name': 'mysql', + 'host': self.overcloud_ip, + 'port': '3306', + 'user': 'calipso', + 'pwd': pwd + } + return mysql_config + + def get_cli_config(self): + return { + 'name': 'CLI', + 'host': self.overcloud_ip, + 'user': self.OVERCLOUD_USER, + 'key': self.overcloud_key_container + } + + def get_amqp_config(self): + user = self.get_value_from_file('rabbitmq_conf', + ' {default_user', + separator=',', + regex='"(.+)"') + pwd = self.get_value_from_file('rabbitmq_conf', + ' {default_pass', + separator=',', + regex='"(.+)"') + port = self.get_value_from_file('rabbitmq_conf', + ' {tcp_listeners', + separator=',', + regex=', (\d+)') + port = int(port) + return { + 'name': 'AMQP', + 'host': self.overcloud_ip, + 'port': port, + 'user': user, + 'pwd': pwd + } + + def get_monitoring_config(self): + return { + 'name': 'Monitoring', + 'config_folder': '/local_dir/sensu_config', + 'env_type': 'production', + 'rabbitmq_port': '5671', + 'rabbitmq_user': 'sensu', + 'server_ip': self.undercloud_ip, + 'server_name': 'sensu_server', + 'type': 'Sensu', + 'provision': 'None', + 'ssh_port': '20022', + 'ssh_user': 'root', + 'ssh_password': 'osdna', + 'api_port': 4567, + 'rabbitmq_pass': 'osdna' + } + + def prepare_env_configuration_array(self): + config_array = [ + self.get_api_config(), + self.get_mysql_config(), + self.get_cli_config(), + self.get_amqp_config(), + self.get_monitoring_config() + ] + self.env_config['configuration'] = config_array + + UI_USER = 'wNLeBJxNDyw8G7Ssg' + + def add_env_ui_conf(self): + self.env_config.update({ + 'user': self.UI_USER, + 'auth': { + 'view-env': [self.UI_USER], + 'edit-env': [self.UI_USER] + } + }) + + def get_mechanism_driver(self): + driver = self.get_value_from_file('ml2_conf', 'mechanism_drivers', + separator=' =') + return 'OVS' if driver == 'openvswitch' else driver + + def set_env_level_attributes(self): + self.env_config.update({ + 'distribution': 'Apex', + 'distribution_version': 'Euphrates', + 'type_drivers': self.get_value_from_file('ml2_conf', + 'tenant_network_types', + separator=' = '), + 'mechanism_drivers': [self.get_mechanism_driver()], + "operational": "running", + "scanned": False, + "type": "environment", + "app_path": "/home/scan/calipso_prod/app", + "listen": True, + "enable_monitoring": True, + "aci_enabled": False, + "last_scanned": "", + "monitoring_setup_done": False + }) + + def prepare_env_config(self): + self.prepare_env_configuration_array() + self.set_env_level_attributes() + self.add_env_ui_conf() + config_dump = json.dumps(self.env_config, sort_keys=True, indent=4, + separators=(',', ': ')) + with open(self.env_config_file, 'w') as config_file: + config_file.write(config_dump) + + def setup_environment_config(self, config_file): + self.run_cmd('mkdir -p {}'.format(self.args.config_dir)) + self.env_config = {'name': self.args.env} + self.undercloud_ip = self.get_undercloud_ip() + config_file.write('jumphost_admin_ip {}\n'.format(self.undercloud_ip)) + self.set_ssh_dir() + self.get_undercloud_setup() + self.overcloud_ip = self.get_overcloud_ip() + config_file.write('overcloud_admin_ip {}\n'.format(self.overcloud_ip)) + self.get_overcloud_setup() + # now get correct IP of overcloud from RabbitMQ setup + self.overcloud_ip = self.get_value_from_file('rabbitmq_conf', + ' {tcp_listeners', + regex='"(.*)"', + separator=',') + self.prepare_env_config() + + def get(self): + try: + print('Fetching Apex environment settings') + self.log = FileLogger(self.args.logfile) + self.run_cmd('mkdir -p {}'.format(self.args.config_dir)) + with open(self.config_file, 'w') as config_file: + self.setup_environment_config(config_file) + print('Finished fetching Apex environment settings') + return True, 'Environment setup finished successfully' + except RuntimeError as e: + return False, str(e) + + +if __name__ == '__main__': + fetcher = ApexEnvironmentFetcher() + ret, msg = fetcher.get() + if not ret: + if fetcher.log: + fetcher.log.error(msg) + else: + print(msg) + sys.exit(0 if ret else 1) diff --git a/app/install/db/apex_environment_config.json b/app/install/db/apex_environment_config.json new file mode 100644 index 0000000..918cd01 --- /dev/null +++ b/app/install/db/apex_environment_config.json @@ -0,0 +1,3 @@ +{ + "_id" : "apex_environment_config_temp_id" +} \ No newline at end of file diff --git a/app/install/db/api_tokens.json b/app/install/db/api_tokens.json new file mode 100644 index 0000000..94cc63a --- /dev/null +++ b/app/install/db/api_tokens.json @@ -0,0 +1,7 @@ +{ + "token" : "ec56537a-a448-43f9-b36e-3e2bee44f018", + "issued_at" : "2017-04-06T14:32:17.893797Z", + "expires_at" : "2017-11-11T15:32:17.893769Z", + "lifetime" : "86400", + "method" : "credentials" +} diff --git a/app/install/db/connection_tests.json b/app/install/db/connection_tests.json new file mode 100644 index 0000000..9965ea3 --- /dev/null +++ b/app/install/db/connection_tests.json @@ -0,0 +1,156 @@ +[ +{ + "environment" : "My-Environment", + "test_targets" : [ + "AMQP", + "CLI", + "ACI", + "mysql", + "OpenStack", + "Monitoring" + ], + "test_results" : { + "AMQP" : false, + "CLI" : true, + "ACI" : false, + "mysql" : false, + "OpenStack" : false, + "Monitoring" : false + }, + "targets_configuration" : + [ + { + "name" : "OpenStack", + "host" : "1.1.1.1", + "port" : "5000", + "user" : "admin", + "pwd" : "admin", + "admin_token" : "tokentoken" + }, + { + "name" : "ACI", + "host" : "2.2.2.2", + "user" : "admin", + "pwd" : "password" + }, + { + "name" : "mysql", + "host" : "1.1.1.1", + "port" : "3307", + "user" : "root", + "pwd" : "password" + }, + { + "name" : "CLI", + "host" : "1.1.1.7", + "user" : "root", + "key" : "/local_dir/ACI-id_rsa" + }, + { + "name" : "AMQP", + "host" : "1.1.1.1", + "port" : "5673", + "user" : "nova", + "pwd" : "password" + }, + { + "name" : "Monitoring", + "config_folder" : "/local_dir/sensu_config", + "env_type" : "production", + "rabbitmq_port" : "5671", + "rabbitmq_user" : "sensu", + "server_ip" : "the_sensu_server", + "server_name" : "sensu_server", + "type" : "Sensu", + "provision" : "None", + "ssh_port" : "20022", + "ssh_user" : "root", + "ssh_password" : "osdna", + "api_port" : 4567, + "rabbitmq_pass" : "osdna" + } + ], + "submit_timestamp" : "2017-05-17T07:53:09.194+0000", + "response_time" : "78ms", + "response_timestamp" : "2017-03-17T11:00:17.939+0000", + "status" : "response", + "last_response_message" : "cli says yes i am alive" +}, +{ + "environment" : "Mirantis-Liberty", + "test_targets" : [ + "AMQP", + "CLI", + "ACI", + "mysql", + "OpenStack", + "Monitoring" + ], + "targets_configuration" : + [ + { + "name" : "OpenStack", + "host" : "1.1.1.1", + "port" : "5000", + "user" : "admin", + "pwd" : "admin", + "admin_token" : "tokentoken" + }, + { + "name" : "ACI", + "host" : "2.2.2.2", + "user" : "admin", + "pwd" : "password" + }, + { + "name" : "mysql", + "host" : "1.1.1.1", + "port" : "3307", + "user" : "root", + "pwd" : "password" + }, + { + "name" : "CLI", + "host" : "1.1.1.7", + "user" : "root", + "key" : "/local_dir/ACI-id_rsa" + }, + { + "name" : "AMQP", + "host" : "1.1.1.1", + "port" : "5673", + "user" : "nova", + "pwd" : "password" + }, + { + "name" : "Monitoring", + "config_folder" : "/local_dir/sensu_config", + "env_type" : "production", + "rabbitmq_port" : "5671", + "rabbitmq_user" : "sensu", + "server_ip" : "the_sensu_server", + "server_name" : "sensu_server", + "type" : "Sensu", + "provision" : "None", + "ssh_port" : "20022", + "ssh_user" : "root", + "ssh_password" : "osdna", + "api_port" : 4567, + "rabbitmq_pass" : "osdna" + } + ], + "test_results" : { + "AMQP" : false, + "CLI" : false, + "ACI" : false, + "mysql" : true, + "OpenStack" : false, + "Monitoring" : false + }, + "submit_timestamp" : "2017-05-17T07:53:09.194+0000", + "response_time" : "56ms", + "response_timestamp" : "2017-05-17T11:00:17.939+0000", + "status" : "response", + "last_response_message" : "mysql says yes i am alive" +} +] diff --git a/app/install/db/constants.json b/app/install/db/constants.json index 2ad8921..15522b7 100644 --- a/app/install/db/constants.json +++ b/app/install/db/constants.json @@ -528,6 +528,10 @@ { "value" : "10239", "label" : "10239" + }, + { + "value" : "10918", + "label" : "10918" } ] }, @@ -721,5 +725,34 @@ "value" : "switch" } ] +}, +{ + "name" : "configuration_targets", + "data" : [ + { + "label" : "AMQP", + "value" : "AMQP" + }, + { + "label" : "CLI", + "value" : "CLI" + }, + { + "label" : "ACI", + "value" : "ACI" + }, + { + "label" : "mysql", + "value" : "mysql" + }, + { + "label" : "OpenStack", + "value" : "OpenStack" + }, + { + "label" : "Monitoring", + "value" : "Monitoring" + } + ] } ] diff --git a/app/install/db/environments_config.json b/app/install/db/environments_config.json index 93971a2..d7157e7 100644 --- a/app/install/db/environments_config.json +++ b/app/install/db/environments_config.json @@ -7,14 +7,14 @@ "name" : "OpenStack", "admin_token" : "dummy_token", "user" : "adminuser", - "port" : 5000, + "port" : "5000", "pwd" : "dummy_pwd", "host" : "10.0.0.1" }, { "name" : "mysql", "pwd" : "dummy_pwd", - "port" : 3307, + "port" : "3307", "user" : "mysqluser", "host" : "10.0.0.1" }, @@ -27,18 +27,18 @@ { "name" : "AMQP", "pwd" : "dummy_pwd", - "port" : 5673, + "port" : "5673", "user" : "rabbitmquser", "host" : "10.0.0.1" }, { - "rabbitmq_port" : 5671, + "rabbitmq_port" : "5671", "ssh_user" : "root", "server_name" : "sensu_server", "env_type" : "production", "provision" : "None", "name" : "Monitoring", - "ssh_port" : 20022, + "ssh_port" : "20022", "rabbitmq_pass" : "dummy_pwd", "ssh_password" : "dummy_pwd", "rabbitmq_user" : "sensu", diff --git a/app/install/db/supported_environments.json b/app/install/db/supported_environments.json index 9a9ddcb..0d507c3 100644 --- a/app/install/db/supported_environments.json +++ b/app/install/db/supported_environments.json @@ -1,288 +1,178 @@ [ -{ + { "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "10.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mercury", - "distribution_version" : "10239", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : false - } -}, -{ - "environment" : { - "distribution" : "Apex", - "distribution_version" : "Euphrates", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Stratoscale", - "distribution_version" : "2.1.6", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : false, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "6.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "7.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "8.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, + "distribution" : "Apex", + "distribution_version" : ["Euphrates"], + "mechanism_drivers" : "OVS", + "type_drivers" : "vxlan" + }, "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true + "listening" : true, + "scanning" : true, + "monitoring" : true } -}, -{ + }, + { "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "9.1", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, + "distribution" : "Devstack", + "distribution_version" : ["Mitaka"], + "mechanism_drivers" : "VPP", + "type_drivers" : "vlan" + }, "features" : { - "listening" : false, - "scanning" : true, - "monitoring" : true + "listening" : true, + "scanning" : true, + "monitoring" : true } -}, -{ + }, + { "environment" : { - "distribution" : "RDO", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, + "distribution" : "Devstack", + "distribution_version" : ["Mitaka"], + "mechanism_drivers" : "VPP", + "type_drivers" : "vxlan" + }, "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true + "listening" : true, + "scanning" : true, + "monitoring" : true } -}, -{ + }, + { "environment" : { - "distribution" : "RDO", - "distribution_version" : "Liberty", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, + "distribution" : "Mercury", + "distribution_version" : ["10239"], + "mechanism_drivers" : "OVS", + "type_drivers" : "vlan" + }, "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true + "listening" : true, + "scanning" : true, + "monitoring" : false } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "9.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vxlan" - }, + }, + { "features" : { "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "9.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" + "monitoring" : true, + "scanning" : true }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "8.0", "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "6.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "7.0", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Mirantis", - "distribution_version" : "9.1", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "RDO", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "VPP", - "type_drivers" : "vxlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "RDO", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "VPP", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Devstack", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "VPP", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "Devstack", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "VPP", - "type_drivers" : "vxlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "RDO", - "distribution_version" : "Mitaka", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -}, -{ - "environment" : { - "distribution" : "RDO", - "distribution_version" : "Liberty", - "mechanism_drivers" : "OVS", - "type_drivers" : "vlan" - }, - "features" : { - "listening" : true, - "scanning" : true, - "monitoring" : true - } -} + "type_drivers" : "vlan", + "distribution" : "Mercury", + "distribution_version" : ["10918"] + } + }, + { + "environment" : { + "distribution" : "Mirantis", + "distribution_version" : [ + "6.0", + "7.0", + "8.0", + "9.0", + "9.1", + "10.0" + ], + "mechanism_drivers" : "OVS", + "type_drivers" : "vlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "Mirantis", + "distribution_version" : [ + "6.0", + "7.0", + "8.0", + "9.0", + "9.1", + "10.0" + ], + "mechanism_drivers" : "OVS", + "type_drivers" : "vxlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "RDO", + "distribution_version" : [ + "Liberty", + "Mitaka" + ], + "mechanism_drivers" : "OVS", + "type_drivers" : "vlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "RDO", + "distribution_version" : [ + "Liberty", + "Mitaka" + ], + "mechanism_drivers" : "OVS", + "type_drivers" : "vxlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "RDO", + "distribution_version" : ["Mitaka"], + "mechanism_drivers" : "VPP", + "type_drivers" : "vxlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "RDO", + "distribution_version" : ["Mitaka"], + "mechanism_drivers" : "VPP", + "type_drivers" : "vlan" + }, + "features" : { + "listening" : true, + "scanning" : true, + "monitoring" : true + } + }, + { + "environment" : { + "distribution" : "Stratoscale", + "distribution_version" : ["2.1.6"], + "mechanism_drivers" : "OVS", + "type_drivers" : "vlan" + }, + "features" : { + "listening" : false, + "scanning" : true, + "monitoring" : true + } + } ] diff --git a/app/install/db/user_settings.json b/app/install/db/user_settings.json new file mode 100644 index 0000000..ad0e868 --- /dev/null +++ b/app/install/db/user_settings.json @@ -0,0 +1,4 @@ +{ + "messages_view_backward_delta" : 1209600000, + "user_id" : "wNLeBJxNDyw8G7Ssg" +} diff --git a/app/install/ldap.conf.example b/app/install/ldap.conf.example index b1798f7..6a4f926 100644 --- a/app/install/ldap.conf.example +++ b/app/install/ldap.conf.example @@ -1,6 +1,6 @@ user admin password password -url ldap://korlev-calipso-dev.cisco.com:389 +url ldap://your-server.cisco.com:389 user_id_attribute CN user_pass_attribute userpassword user_objectclass inetOrgPerson -- cgit 1.2.3-korg