diff options
Diffstat (limited to 'deploy')
| -rw-r--r-- | deploy/README | 6 | ||||
| -rw-r--r-- | deploy/cloud/deployment.py | 116 | ||||
| -rw-r--r-- | deploy/common.py | 14 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-odl_0.9.0.yaml | 4 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-tacker_0.9.0.yaml | 48 | ||||
| -rw-r--r-- | deploy/deploy-config.py | 534 | ||||
| -rw-r--r-- | deploy/environments/execution_environment.py | 2 | ||||
| -rw-r--r-- | deploy/install_fuel_master.py | 12 | ||||
| -rw-r--r-- | deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml | 9 | ||||
| -rw-r--r-- | deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml | 5 | ||||
| -rw-r--r-- | deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml | 9 | ||||
| -rw-r--r-- | deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml | 5 | ||||
| -rwxr-xr-x | deploy/templater.py | 41 | ||||
| -rw-r--r-- | deploy/templates/hardware_environment/vms/enea_lab/fuel.xml | 88 |
14 files changed, 582 insertions, 311 deletions
diff --git a/deploy/README b/deploy/README index 40f95ef92..ee6bc3156 100644 --- a/deploy/README +++ b/deploy/README @@ -13,9 +13,11 @@ the following dependencies and python modules are required to be installed: - for Ubuntu: -sudo apt-get install -y libvirt-bin qemu-kvm python-pip fuseiso mkisofs genisoimage +sudo apt-get update +sudo apt-get install -y libvirt-bin qemu-kvm python-pip fuseiso mkisofs \ +genisoimage ipmitool sudo apt-get install -y python-dev libz-dev libxml2-dev libxslt-dev libyaml-dev -sudo pip install pyyaml netaddr paramiko lxml scp pycrypto ecdsa +sudo pip install pyyaml netaddr paramiko lxml scp pycrypto ecdsa amt During libvirt install the user is added to the libvirtd group, so you have to logout then login back again diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 4a9fcd9a8..ecccc241f 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py @@ -9,6 +9,7 @@ import time import re +import json from common import ( N, @@ -29,8 +30,16 @@ GREP_LINES_OF_LEADING_CONTEXT = 100 GREP_LINES_OF_TRAILING_CONTEXT = 100 LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"'] -class Deployment(object): +class DeployNotStart(Exception): + """Unable to start deployment""" + + +class NodesGoOffline(Exception): + """Nodes goes offline during deployment""" + + +class Deployment(object): def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict, no_health_check, deploy_timeout): @@ -43,7 +52,6 @@ class Deployment(object): self.pattern = re.compile( '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d') - def collect_error_logs(self): for node_id, roles_blade in self.node_id_roles_dict.iteritems(): log_list = [] @@ -89,7 +97,7 @@ class Deployment(object): log_msg += details if log_msg: - log_list.append(log_msg) + log_list.append(log_msg) if log_list: role = ('controller' if 'controller' in roles_blade[0] @@ -99,47 +107,88 @@ class Deployment(object): for log_msg in log_list: print(log_msg + '\n') - def run_deploy(self): SLEEP_TIME = 60 - LOG_FILE = 'cloud.log' + abort_after = 60 * int(self.deploy_timeout) + start = time.time() log('Starting deployment of environment %s' % self.env_id) - deploy_proc = run_proc('fuel --env %s deploy-changes | strings > %s' - % (self.env_id, LOG_FILE)) - + deploy_id = None ready = False - for i in range(int(self.deploy_timeout)): - env = parse(exec_cmd('fuel env --env %s' % self.env_id)) - log('Environment status: %s' % env[0][E['status']]) - r, _ = exec_cmd('tail -2 %s | head -1' % LOG_FILE, False) - if r: - log(r) - if env[0][E['status']] == 'operational': - ready = True - break - elif (env[0][E['status']] == 'error' - or env[0][E['status']] == 'stopped'): - break - else: + timeout = False + + attempts = 0 + while attempts < 3: + try: + if time.time() > start + abort_after: + timeout = True + break + if not deploy_id: + deploy_id = self._start_deploy_task() + sts, prg, msg = self._deployment_status(deploy_id) + if sts == 'error': + log('Error during deployment: {}'.format(msg)) + break + if sts == 'running': + log('Environmnent deploymnet progress: {}%'.format(prg)) + elif sts == 'ready': + ready = True + break time.sleep(SLEEP_TIME) - - if (env[0][E['status']] <> 'operational' - and env[0][E['status']] <> 'error' - and env[0][E['status']] <> 'stopped'): - err('Deployment timed out, environment %s is not operational, snapshot will not be performed' - % self.env_id, self.collect_logs) - - run_proc_wait_terminated(deploy_proc) - delete(LOG_FILE) - + except (DeployNotStart, NodesGoOffline) as e: + log(e) + attempts += 1 + deploy_id = None + time.sleep(SLEEP_TIME * attempts) + + if timeout: + err('Deployment timed out, environment %s is not operational, ' + 'snapshot will not be performed' + % self.env_id) if ready: - log('Environment %s successfully deployed' % self.env_id) + log('Environment %s successfully deployed' + % self.env_id) else: self.collect_error_logs() err('Deployment failed, environment %s is not operational' % self.env_id, self.collect_logs) + def _start_deploy_task(self): + out, _ = exec_cmd('fuel2 env deploy {}'.format(self.env_id), False) + id = self._deployment_task_id(out) + return id + + def _deployment_task_id(self, response): + response = str(response) + if response.startswith('Deployment task with id'): + for s in response.split(): + if s.isdigit(): + return int(s) + raise DeployNotStart('Unable to start deployment: {}'.format(response)) + + def _deployment_status(self, id): + task = self._task_fields(id) + if task['status'] == 'error': + if task['message'].endswith( + 'offline. Remove them from environment and try again.'): + raise NodesGoOffline(task['message']) + return task['status'], task['progress'], task['message'] + + def _task_fields(self, id): + try: + out, _ = exec_cmd('fuel2 task show {} -f json'.format(id), False) + task_info = json.loads(out) + properties = {} + # for 9.0 this can be list of dicts or dict + # see https://bugs.launchpad.net/fuel/+bug/1625518 + if isinstance(task_info, list): + for d in task_info: + properties.update({d['Field']: d['Value']}) + else: + return task_info + return properties + except ValueError as e: + err('Unable to fetch task info: {}'.format(e)) def collect_logs(self): log('Cleaning out any previous deployment logs') @@ -155,7 +204,6 @@ class Deployment(object): r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False) log(r) - def verify_node_status(self): node_list = parse(exec_cmd('fuel --env %s node' % self.env_id)) failed_nodes = [] @@ -169,7 +217,6 @@ class Deployment(object): summary += '[node %s, status %s]\n' % (node, status) err('Deployment failed: %s' % summary, self.collect_logs) - def health_check(self): log('Now running sanity and smoke health checks') r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id) @@ -177,7 +224,6 @@ class Deployment(object): if 'failure' in r: err('Healthcheck failed!', self.collect_logs) - def deploy(self): self.run_deploy() self.verify_node_status() diff --git a/deploy/common.py b/deploy/common.py index 80832e201..dab9602c5 100644 --- a/deploy/common.py +++ b/deploy/common.py @@ -57,7 +57,7 @@ def exec_cmd(cmd, check=True, attempts=1, delay=5, verbose=False, mask_args=[], # a negative value means forever while attempts != 0: - attempts = attempts - 1 + attempts -= 1 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -74,13 +74,13 @@ def exec_cmd(cmd, check=True, attempts=1, delay=5, verbose=False, mask_args=[], if check: if return_code > 0: stderr = stderr.strip() - print "Failed command: " + str(masked_cmd) - print "Command returned response: " + str(stderr) - print "Command return code: " + str(return_code) + print("Failed command: " + str(masked_cmd)) + print("Command returned response: " + str(stderr)) + print("Command return code: " + str(return_code)) raise Exception(stderr) else: - print "Command: " + str(masked_cmd) - print str(response) + print("Command: " + str(masked_cmd)) + print(str(response)) return response return response, return_code @@ -140,7 +140,7 @@ def warn(message): def check_file_exists(file_path): if not os.path.dirname(file_path): file_path = '%s/%s' % (CWD, file_path) - if not os.path.isfile(file_path): + if not os.access(file_path, os.R_OK): err('ERROR: File %s not found\n' % file_path) diff --git a/deploy/config/plugins/fuel-odl_0.9.0.yaml b/deploy/config/plugins/fuel-odl_0.9.0.yaml index 6caf4834f..9646d021f 100644 --- a/deploy/config/plugins/fuel-odl_0.9.0.yaml +++ b/deploy/config/plugins/fuel-odl_0.9.0.yaml @@ -118,9 +118,9 @@ opendaylight: vpn: - odl-vpnservice-openstack odl_deb: opendaylight - experimental_odl_deb: opendaylight-boron + experimental_odl_deb: opendaylight-experimental use_experimental_odl: - - enable_sfc + - enable_bgpvpn #plugin_id: Assigned during installation plugin_version: 0.9.0 restrictions: diff --git a/deploy/config/plugins/fuel-tacker_0.9.0.yaml b/deploy/config/plugins/fuel-tacker_0.9.0.yaml new file mode 100644 index 000000000..71e028ffd --- /dev/null +++ b/deploy/config/plugins/fuel-tacker_0.9.0.yaml @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +plugin-config-metadata: + title: Tacker fuel plugin configuration template + version: 0.1 + created: 03.10.2016 + comment: None +tacker: + metadata: + #chosen_id: Assigned during installation + class: plugin + default: false + enabled: true + label: Tacker VNF manager + toggleable: true + versions: + - metadata: + group: 'openstack_services' + db_password: + generator: 'password' + user_password: + generator: 'password' + user: 'tacker' + port: 8889 + service: 'tacker-server' + restrictions: + - condition: "settings:opendaylight == null or settings:opendaylight.metadata.enabled == false or settings:opendaylight.enable_sfc.value == false" + strict: false + message: "Please install OpenDaylight Plugin with SFC features enabled" + - condition: "settings:fuel-plugin-ovs == null or settings:fuel-plugin-ovs.metadata.enabled == false" + strict: false + message: "Please install and enable Openvswitch plugin with NSH support." + #plugin_id: Assigned during installation + plugin_version: 0.2.0 + debug: + value: false + label: 'Debug logging' + description: 'Debug logging mode provides more information, but requires more disk space.' + weight: 25 + type: "checkbox" + weight: 70 diff --git a/deploy/deploy-config.py b/deploy/deploy-config.py index 8896080db..d2567d97b 100644 --- a/deploy/deploy-config.py +++ b/deploy/deploy-config.py @@ -69,8 +69,8 @@ def parse_arguments(): required=True) parser.add_argument('-scenario', dest='scenario', action='store', default=False, - help=('Deployment scenario short-name (priority),' - 'or base file name (in the absense of a' + help=('Deployment scenario short-name (priority), ' + 'or base file name (in the absense of a ' 'shortname defenition)'), required=True) @@ -83,7 +83,6 @@ def parse_arguments(): help='Local path for resulting output configuration files', required=True) args = parser.parse_args() - log(args) kwargs = {'dha_uri': args.dha_uri, 'dea_base_uri': args.dea_base_uri, 'dea_pod_override_uri': args.dea_pod_override_uri, @@ -162,127 +161,6 @@ def merge_dicts(dict1, dict2): yield (k, dict1[k]) -setup_yaml() -kwargs = parse_arguments() - -# Generate final dea.yaml by merging following config files/fragments in revers priority order: -# "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override" -# and "deployment-scenario/dea-override" -print('Generating final dea.yaml configuration....') - -# Fetch dea-base, extract and purge meta-data -print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....") -response = urllib2.urlopen(kwargs["dea_base_uri"]) -dea_base_conf = yaml.load(response.read()) -dea_base_title = dea_base_conf['dea-base-config-metadata']['title'] -dea_base_version = dea_base_conf['dea-base-config-metadata']['version'] -dea_base_creation = dea_base_conf['dea-base-config-metadata']['created'] -dea_base_sha = sha_uri(kwargs["dea_base_uri"]) -dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment'] -dea_base_conf.pop('dea-base-config-metadata') -final_dea_conf = dea_base_conf -dea_pod_override_nodes = None - -# Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure -print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....") -response = urllib2.urlopen(kwargs["dea_pod_override_uri"]) -dea_pod_override_conf = yaml.load(response.read()) -if dea_pod_override_conf: - dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title'] - dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version'] - dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created'] - dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"]) - dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment'] - print('Merging dea-base and dea-pod-override configuration ....') - dea_pod_override_conf.pop('dea-pod-override-config-metadata') - # Copy the list of original nodes, which holds info on their transformations - if 'nodes' in dea_pod_override_conf: - dea_pod_override_nodes = list(dea_pod_override_conf['nodes']) - if dea_pod_override_conf: - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf)) - -# Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/ -# dea-override-configith previous dea data structure -print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....") - -response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml") -scenario_short_translation_conf = yaml.load(response.read()) -if kwargs["scenario"] in scenario_short_translation_conf: - scenario_uri = (kwargs["scenario_base_uri"] - + "/" - + scenario_short_translation_conf[kwargs["scenario"]]['configfile']) -else: - scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"] -response = urllib2.urlopen(scenario_uri) -deploy_scenario_conf = yaml.load(response.read()) - -if deploy_scenario_conf: - deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title'] - deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version'] - deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created'] - deploy_scenario_sha = sha_uri(scenario_uri) - deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment'] - deploy_scenario_conf.pop('deployment-scenario-metadata') -else: - print("Deployment scenario file not found or is empty") - print("Cannot continue, exiting ....") - sys.exit(1) - -dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"] -if dea_scenario_override_conf: - print('Merging dea-base-, dea-pod-override- and deployment-scenario ' - 'configuration into final dea.yaml configuration....') - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf)) - -# Fetch plugin-configuration configuration files, extract and purge meta-data, -# merge/append with previous dea data structure, override plugin-configuration with -# deploy-scenario/module-config-override -modules = [] -module_uris = [] -module_titles = [] -module_versions = [] -module_creations = [] -module_shas = [] -module_comments = [] -if deploy_scenario_conf["stack-extensions"]: - for module in deploy_scenario_conf["stack-extensions"]: - print('Loading configuration for module: ' - + module["module"] - + ' and merging it to final dea.yaml configuration....') - response = urllib2.urlopen(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml') - module_conf = yaml.load(response.read()) - modules.append(module["module"]) - module_uris.append(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml') - module_titles.append(str(module_conf['plugin-config-metadata']['title'])) - module_versions.append(str(module_conf['plugin-config-metadata']['version'])) - module_creations.append(str(module_conf['plugin-config-metadata']['created'])) - module_shas.append(sha_uri(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml')) - module_comments.append(str(module_conf['plugin-config-metadata']['comment'])) - module_conf.pop('plugin-config-metadata') - final_dea_conf['settings']['editable'].update(module_conf) - scenario_module_override_conf = module.get('module-config-override') - if scenario_module_override_conf: - dea_scenario_module_override_conf = {} - dea_scenario_module_override_conf['settings'] = {} - dea_scenario_module_override_conf['settings']['editable'] = {} - dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf)) - def get_node_ifaces_and_trans(nodes, nid): for node in nodes: if node['id'] == nid: @@ -293,117 +171,297 @@ def get_node_ifaces_and_trans(nodes, nid): return None -if dea_pod_override_nodes: - for node in final_dea_conf['nodes']: - data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id']) - if data: - print ("Honoring original interfaces and transformations for " - "node %d to %s, %s" % (node['id'], data[0], data[1])) - node['interfaces'] = data[0] - node['transformations'] = data[1] - -# Dump final dea.yaml including configuration management meta-data to argument provided -# directory -if not os.path.exists(kwargs["output_path"]): - os.makedirs(kwargs["output_path"]) -print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....') -with open(kwargs["output_path"] + '/dea.yaml', "w") as f: - f.write("\n".join([("title: DEA.yaml file automatically generated from the" - 'configuration files stated in the "configuration-files"' - "fragment below"), - "version: " + str(calendar.timegm(time.gmtime())), - "created: " + str(time.strftime("%d/%m/%Y")) + " " - + str(time.strftime("%H:%M:%S")), - "comment: none\n"])) - - f.write("\n".join(["configuration-files:", - " dea-base:", - " uri: " + kwargs["dea_base_uri"], - " title: " + str(dea_base_title), - " version: " + str(dea_base_version), - " created: " + str(dea_base_creation), - " sha1: " + str(dea_base_sha), - " comment: " + str(dea_base_comment) + "\n"])) - - f.write("\n".join([" pod-override:", - " uri: " + kwargs["dea_pod_override_uri"], - " title: " + str(dea_pod_title), - " version: " + str(dea_pod_version), - " created: " + str(dea_pod_creation), - " sha1: " + str(dea_pod_sha), - " comment: " + str(dea_pod_comment) + "\n"])) - - f.write("\n".join([" deployment-scenario:", - " uri: " + str(scenario_uri), - " title: " + str(deploy_scenario_title), - " version: " + str(deploy_scenario_version), - " created: " + str(deploy_scenario_creation), - " sha1: " + str(deploy_scenario_sha), - " comment: " + str(deploy_scenario_comment) + "\n"])) - - f.write(" plugin-modules:\n") - for k, _ in enumerate(modules): - f.write("\n".join([" - module: " + modules[k], - " uri: " + module_uris[k], - " title: " + module_titles[k], - " version: " + module_versions[k], - " created: " + module_creations[k], - " sha-1: " + module_shas[k], - " comment: " + module_comments[k] + "\n"])) - - yaml.dump(final_dea_conf, f, default_flow_style=False) - -# Load POD dha and override it with "deployment-scenario/dha-override-config" section -print('Generating final dha.yaml configuration....') -print('Parsing dha-pod yaml configuration....') -response = urllib2.urlopen(kwargs["dha_uri"]) -dha_pod_conf = yaml.load(response.read()) -dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title'] -dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version'] -dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created'] -dha_pod_sha = sha_uri(kwargs["dha_uri"]) -dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment'] -dha_pod_conf.pop('dha-pod-config-metadata') -final_dha_conf = dha_pod_conf - -dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"] -# Only virtual deploy scenarios can override dha.yaml since there -# is no way to programatically override a physical environment: -# wireing, IPMI set-up, etc. -# For Physical environments, dha.yaml overrides will be silently ignored -if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' - or final_dha_conf['adapter'] == 'esxi' - or final_dha_conf['adapter'] == 'vbox'): - print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....') - final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf)) - -# Dump final dha.yaml to argument provided directory -print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....') -with open(kwargs["output_path"] + '/dha.yaml', "w") as f: - f.write("\n".join([("title: DHA.yaml file automatically generated from" - "the configuration files stated in the" - '"configuration-files" fragment below'), - "version: " + str(calendar.timegm(time.gmtime())), - "created: " + str(time.strftime("%d/%m/%Y")) + " " - + str(time.strftime("%H:%M:%S")), - "comment: none\n"])) - - f.write("configuration-files:\n") - - f.write("\n".join([" dha-pod-configuration:", - " uri: " + kwargs["dha_uri"], - " title: " + str(dha_pod_title), - " version: " + str(dha_pod_version), - " created: " + str(dha_pod_creation), - " sha-1: " + str(dha_pod_sha), - " comment: " + str(dha_pod_comment) + "\n"])) - - f.write("\n".join([" deployment-scenario:", - " uri: " + str(scenario_uri), - " title: " + str(deploy_scenario_title), - " version: " + str(deploy_scenario_version), - " created: " + str(deploy_scenario_creation), - " sha-1: " + str(deploy_scenario_sha), - " comment: " + str(deploy_scenario_comment) + "\n"])) - - yaml.dump(final_dha_conf, f, default_flow_style=False) + +class DeployConfig(object): + def __init__(self): + self.kwargs = parse_arguments() + self.dea_conf = dict() + self.dea_metadata = dict() + self.dea_pod_ovr_metadata = dict() + self.dea_pod_ovr_nodes = None + self.scenario_metadata = dict() + self.modules = [] + self.module_uris = [] + self.module_titles = [] + self.module_versions = [] + self.module_createds = [] + self.module_shas = [] + self.module_comments = [] + self.dha_pod_conf = dict() + self.dha_metadata = dict() + + def process_dea_base(self): + # Generate final dea.yaml by merging following config files/fragments in reverse priority order: + # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override" + # and "deployment-scenario/dea-override" + print('Generating final dea.yaml configuration....') + + # Fetch dea-base, extract and purge meta-data + print('Parsing dea-base from: ' + self.kwargs["dea_base_uri"] + "....") + response = urllib2.urlopen(self.kwargs["dea_base_uri"]) + dea_conf = yaml.load(response.read()) + + dea_metadata = dict() + dea_metadata['title'] = dea_conf['dea-base-config-metadata']['title'] + dea_metadata['version'] = dea_conf['dea-base-config-metadata']['version'] + dea_metadata['created'] = dea_conf['dea-base-config-metadata']['created'] + dea_metadata['sha'] = sha_uri(self.kwargs["dea_base_uri"]) + dea_metadata['comment'] = dea_conf['dea-base-config-metadata']['comment'] + self.dea_metadata = dea_metadata + dea_conf.pop('dea-base-config-metadata') + self.dea_conf = dea_conf + + def process_dea_pod_override(self): + # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure + print('Parsing the dea-pod-override from: ' + self.kwargs["dea_pod_override_uri"] + "....") + response = urllib2.urlopen(self.kwargs["dea_pod_override_uri"]) + dea_pod_override_conf = yaml.load(response.read()) + + if dea_pod_override_conf: + metadata = dict() + metadata['title'] = dea_pod_override_conf['dea-pod-override-config-metadata']['title'] + metadata['version'] = dea_pod_override_conf['dea-pod-override-config-metadata']['version'] + metadata['created'] = dea_pod_override_conf['dea-pod-override-config-metadata']['created'] + metadata['sha'] = sha_uri(self.kwargs["dea_pod_override_uri"]) + metadata['comment'] = dea_pod_override_conf['dea-pod-override-config-metadata']['comment'] + self.dea_pod_ovr_metadata = metadata + + print('Merging dea-base and dea-pod-override configuration ....') + dea_pod_override_conf.pop('dea-pod-override-config-metadata') + + # Copy the list of original nodes, which holds info on their transformations + if 'nodes' in dea_pod_override_conf: + self.dea_pod_ovr_nodes = list(dea_pod_override_conf['nodes']) + if dea_pod_override_conf: + self.dea_conf = dict(merge_dicts(self.dea_conf, dea_pod_override_conf)) + + def get_scenario_uri(self): + response = urllib2.urlopen(self.kwargs["scenario_base_uri"] + "/scenario.yaml") + scenario_short_translation_conf = yaml.load(response.read()) + if self.kwargs["scenario"] in scenario_short_translation_conf: + scenario_uri = (self.kwargs["scenario_base_uri"] + + "/" + + scenario_short_translation_conf[self.kwargs["scenario"]]['configfile']) + else: + scenario_uri = self.kwargs["scenario_base_uri"] + "/" + self.kwargs["scenario"] + + return scenario_uri + + def get_scenario_config(self): + self.scenario_metadata['uri'] = self.get_scenario_uri() + response = urllib2.urlopen(self.scenario_metadata['uri']) + return yaml.load(response.read()) + + def process_modules(self): + scenario_conf = self.get_scenario_config() + if scenario_conf["stack-extensions"]: + for module in scenario_conf["stack-extensions"]: + print('Loading configuration for module: ' + + module["module"] + + ' and merging it to final dea.yaml configuration....') + response = urllib2.urlopen(self.kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') + module_conf = yaml.load(response.read()) + self.modules.append(module["module"]) + self.module_uris.append(self.kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') + self.module_titles.append(str(module_conf['plugin-config-metadata']['title'])) + self.module_versions.append(str(module_conf['plugin-config-metadata']['version'])) + self.module_createds.append(str(module_conf['plugin-config-metadata']['created'])) + self.module_shas.append(sha_uri(self.kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml')) + self.module_comments.append(str(module_conf['plugin-config-metadata']['comment'])) + module_conf.pop('plugin-config-metadata') + self.dea_conf['settings']['editable'].update(module_conf) + + scenario_module_override_conf = module.get('module-config-override') + if scenario_module_override_conf: + dea_scenario_module_override_conf = {} + dea_scenario_module_override_conf['settings'] = {} + dea_scenario_module_override_conf['settings']['editable'] = {} + dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf + self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_module_override_conf)) + + def process_scenario_config(self): + # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/ + # dea-override-configith previous dea data structure + print('Parsing deployment-scenario from: ' + self.kwargs["scenario"] + "....") + + scenario_conf = self.get_scenario_config() + + metadata = dict() + if scenario_conf: + metadata['title'] = scenario_conf['deployment-scenario-metadata']['title'] + metadata['version'] = scenario_conf['deployment-scenario-metadata']['version'] + metadata['created'] = scenario_conf['deployment-scenario-metadata']['created'] + metadata['sha'] = sha_uri(self.scenario_metadata['uri']) + metadata['comment'] = scenario_conf['deployment-scenario-metadata']['comment'] + self.scenario_metadata = metadata + scenario_conf.pop('deployment-scenario-metadata') + else: + print("Deployment scenario file not found or is empty") + print("Cannot continue, exiting ....") + sys.exit(1) + + dea_scenario_override_conf = scenario_conf["dea-override-config"] + if dea_scenario_override_conf: + print('Merging dea-base-, dea-pod-override- and deployment-scenario ' + 'configuration into final dea.yaml configuration....') + self.dea_conf = dict(merge_dicts(self.dea_conf, dea_scenario_override_conf)) + + self.process_modules() + + # Fetch plugin-configuration configuration files, extract and purge meta-data, + # merge/append with previous dea data structure, override plugin-configuration with + # deploy-scenario/module-config-override + + if self.dea_pod_ovr_nodes: + for node in self.dea_conf['nodes']: + data = get_node_ifaces_and_trans(self.dea_pod_ovr_nodes, node['id']) + if data: + print("Honoring original interfaces and transformations for " + "node %d to %s, %s" % (node['id'], data[0], data[1])) + node['interfaces'] = data[0] + node['transformations'] = data[1] + + def dump_dea_config(self): + # Dump final dea.yaml including configuration management meta-data to argument provided + # directory + path = self.kwargs["output_path"] + if not os.path.exists(path): + os.makedirs(path) + print('Dumping final dea.yaml to ' + path + '/dea.yaml....') + with open(path + '/dea.yaml', "w") as f: + f.write("\n".join([("title: DEA.yaml file automatically generated from the " + 'configuration files stated in the "configuration-files" ' + "fragment below"), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + time.strftime("%d/%m/%Y %H:%M:%S"), + "comment: none\n"])) + + f.write("\n".join(["configuration-files:", + " dea-base:", + " uri: " + self.kwargs["dea_base_uri"], + " title: " + str(self.dea_metadata['title']), + " version: " + str(self.dea_metadata['version']), + " created: " + str(self.dea_metadata['created']), + " sha1: " + sha_uri(self.kwargs["dea_base_uri"]), + " comment: " + str(self.dea_metadata['comment']) + "\n"])) + + f.write("\n".join([" pod-override:", + " uri: " + self.kwargs["dea_pod_override_uri"], + " title: " + str(self.dea_pod_ovr_metadata['title']), + " version: " + str(self.dea_pod_ovr_metadata['version']), + " created: " + str(self.dea_pod_ovr_metadata['created']), + " sha1: " + self.dea_pod_ovr_metadata['sha'], + " comment: " + str(self.dea_pod_ovr_metadata['comment']) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + self.scenario_metadata['uri'], + " title: " + str(self.scenario_metadata['title']), + " version: " + str(self.scenario_metadata['version']), + " created: " + str(self.scenario_metadata['created']), + " sha1: " + self.scenario_metadata['sha'], + " comment: " + str(self.scenario_metadata['comment']) + "\n"])) + + f.write(" plugin-modules:\n") + for k, _ in enumerate(self.modules): + f.write("\n".join([" - module: " + self.modules[k], + " uri: " + self.module_uris[k], + " title: " + str(self.module_titles[k]), + " version: " + str(self.module_versions[k]), + " created: " + str(self.module_createds[k]), + " sha-1: " + self.module_shas[k], + " comment: " + str(self.module_comments[k]) + "\n"])) + + yaml.dump(self.dea_conf, f, default_flow_style=False) + + def process_dha_pod_config(self): + # Load POD dha and override it with "deployment-scenario/dha-override-config" section + print('Generating final dha.yaml configuration....') + print('Parsing dha-pod yaml configuration....') + response = urllib2.urlopen(self.kwargs["dha_uri"]) + dha_pod_conf = yaml.load(response.read()) + + dha_metadata = dict() + dha_metadata['title'] = dha_pod_conf['dha-pod-config-metadata']['title'] + dha_metadata['version'] = dha_pod_conf['dha-pod-config-metadata']['version'] + dha_metadata['created'] = dha_pod_conf['dha-pod-config-metadata']['created'] + dha_metadata['sha'] = sha_uri(self.kwargs["dha_uri"]) + dha_metadata['comment'] = dha_pod_conf['dha-pod-config-metadata']['comment'] + self.dha_metadata = dha_metadata + dha_pod_conf.pop('dha-pod-config-metadata') + self.dha_pod_conf = dha_pod_conf + + scenario_conf = self.get_scenario_config() + dha_scenario_override_conf = scenario_conf["dha-override-config"] + # Only virtual deploy scenarios can override dha.yaml since there + # is no way to programatically override a physical environment: + # wireing, IPMI set-up, etc. + # For Physical environments, dha.yaml overrides will be silently ignored + if dha_scenario_override_conf and (dha_pod_conf['adapter'] == 'libvirt' + or dha_pod_conf['adapter'] == 'esxi' + or dha_pod_conf['adapter'] == 'vbox'): + print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....') + self.dha_pod_conf = dict(merge_dicts(self.dha_pod_conf, dha_scenario_override_conf)) + + def dump_dha_config(self): + # Dump final dha.yaml to argument provided directory + path = self.kwargs["output_path"] + print('Dumping final dha.yaml to ' + path + '/dha.yaml....') + with open(path + '/dha.yaml', "w") as f: + f.write("\n".join([("title: DHA.yaml file automatically generated from " + "the configuration files stated in the " + '"configuration-files" fragment below'), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + time.strftime("%d/%m/%Y %H:%M:%S"), + "comment: none\n"])) + + f.write("configuration-files:\n") + + f.write("\n".join([" dha-pod-configuration:", + " uri: " + self.kwargs["dha_uri"], + " title: " + str(self.dha_metadata['title']), + " version: " + str(self.dha_metadata['version']), + " created: " + str(self.dha_metadata['created']), + " sha-1: " + self.dha_metadata['sha'], + " comment: " + str(self.dha_metadata['comment']) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + self.scenario_metadata['uri'], + " title: " + str(self.scenario_metadata['title']), + " version: " + str(self.scenario_metadata['version']), + " created: " + str(self.scenario_metadata['created']), + " sha-1: " + self.scenario_metadata['sha'], + " comment: " + str(self.scenario_metadata['comment']) + "\n"])) + + yaml.dump(self.dha_pod_conf, f, default_flow_style=False) + + +def main(): + setup_yaml() + + deploy_config = DeployConfig() + deploy_config.process_dea_base() + deploy_config.process_dea_pod_override() + deploy_config.process_scenario_config() + deploy_config.dump_dea_config() + + deploy_config.process_dha_pod_config() + deploy_config.dump_dha_config() + + +if __name__ == '__main__': + main() diff --git a/deploy/environments/execution_environment.py b/deploy/environments/execution_environment.py index af0e130dd..7a0b4744e 100644 --- a/deploy/environments/execution_environment.py +++ b/deploy/environments/execution_environment.py @@ -46,7 +46,7 @@ class ExecutionEnvironment(object): disk_files.append(source_file) log('Deleting VM %s with disks %s' % (vm_name, disk_files)) exec_cmd('virsh destroy %s' % vm_name, False) - exec_cmd('virsh undefine %s' % vm_name, False) + exec_cmd('virsh undefine --managed-save --remove-all-storage %s' % vm_name, False) for file in disk_files: delete(file) diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py index 808d0b14c..a0e28b033 100644 --- a/deploy/install_fuel_master.py +++ b/deploy/install_fuel_master.py @@ -156,13 +156,12 @@ class InstallFuelMaster(object): def wait_until_fuel_menu_up(self): WAIT_LOOP = 60 SLEEP_TIME = 10 - CMD = 'ps -ef' - SEARCH = 'fuelmenu' + CMD = 'pgrep -f fuelmenu' fuel_menu_pid = None with self.ssh: for i in range(WAIT_LOOP): ret = self.ssh.exec_cmd(CMD) - fuel_menu_pid = self.get_fuel_menu_pid(ret, SEARCH) + fuel_menu_pid = ret.strip() if not fuel_menu_pid: time.sleep(SLEEP_TIME) else: @@ -171,11 +170,6 @@ class InstallFuelMaster(object): raise Exception('Could not find the Fuel Menu Process ID') return fuel_menu_pid - def get_fuel_menu_pid(self, printout, search): - for line in printout.splitlines(): - if line.endswith(search): - return clean(line)[1] - def ssh_exec_cmd(self, cmd, check=True): with self.ssh: ret = self.ssh.exec_cmd(cmd, check=check) @@ -198,7 +192,7 @@ class InstallFuelMaster(object): def wait_until_installation_completed(self): WAIT_LOOP = 360 SLEEP_TIME = 10 - CMD = 'ps -ef | grep %s | grep -v grep' % BOOTSTRAP_ADMIN + CMD = 'pgrep -f %s' % BOOTSTRAP_ADMIN install_completed = False with self.ssh: diff --git a/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml index e6aef2aba..c4789484c 100644 --- a/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/ha_odl-l2_sfc_heat_ceilometer_scenario.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2015 Ericsson AB and others. +# Copyright (c) 2015,2016 Ericsson AB and others. # jonas.bjurel@ericsson.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -22,7 +22,7 @@ # deployment configuration meta-data deployment-scenario-metadata: title: ODL-L2 SFC HA deployment - version: 0.0.1 + version: 0.0.2 created: Feb 10 2016 comment: Rebased to Fuel9 @@ -52,6 +52,9 @@ stack-extensions: value: true metadata: plugin_version: 0.9.0 + - module: tacker + module-config-name: fuel-tacker + module-config-version: 0.9.0 # Note that the module substitionion does not support arrays # This is a quick fix # - module: opendaylight @@ -78,7 +81,7 @@ dea-override-config: role: controller,opendaylight - id: 2 interfaces: interfaces_1 - role: mongo,controller + role: mongo,controller,tacker - id: 3 interfaces: interfaces_1 role: ceph-osd,controller diff --git a/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml b/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml index 0dda3b3ca..967e7d295 100644 --- a/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/ha_odl-l3_heat_ceilometer_scenario.yaml @@ -44,6 +44,11 @@ stack-extensions: value: true metadata: plugin_version: 0.9.0 + + - module: fuel-plugin-ovs + module-config-name: fuel-nshovs + module-config-version: 0.9.0 + # - module: opendaylight # module-config-name: fuel-odl # module-config-version: 0.0.2 diff --git a/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml index a8d9ed848..90a45d577 100644 --- a/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/no-ha_odl-l2_sfc_heat_ceilometer_scenario.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2015 Ericsson AB and others. +# Copyright (c) 2015,2016 Ericsson AB and others. # jonas.bjurel@ericsson.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -22,7 +22,7 @@ # deployment configuration meta-data deployment-scenario-metadata: title: ODL-L2-SFC No-HA deployment - version: 0.0.2 + version: 0.0.3 created: Feb 10 2016 comment: Fuel ODL-L2 SFC No HA with Ceph, Ceilometer and Heat Rebased for Fuel9 @@ -52,6 +52,9 @@ stack-extensions: value: true metadata: plugin_version: 0.9.0 + - module: tacker + module-config-name: fuel-tacker + module-config-version: 0.9.0 # Note that the module substitionion does not support arrays # This is a quick fix @@ -76,7 +79,7 @@ dea-override-config: nodes: - id: 1 interfaces: interfaces_1 - role: mongo,controller + role: mongo,controller,tacker - id: 2 interfaces: interfaces_1 role: ceph-osd,opendaylight diff --git a/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml b/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml index 5f800abad..0c8415f78 100644 --- a/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml +++ b/deploy/scenario/no-ha_odl-l3_heat_ceilometer_scenario.yaml @@ -44,6 +44,11 @@ stack-extensions: value: true metadata: plugin_version: 0.9.0 + + - module: fuel-plugin-ovs + module-config-name: fuel-nshovs + module-config-version: 0.9.0 + # - module: opendaylight # module-config-name: fuel-odl # module-config-version: 0.0.2 diff --git a/deploy/templater.py b/deploy/templater.py index 6b41e1f3c..bda60c7fe 100755 --- a/deploy/templater.py +++ b/deploy/templater.py @@ -12,6 +12,7 @@ import io import re import yaml +import urllib2 from common import( err, ArgParser, @@ -29,10 +30,29 @@ class Templater(object): self.output_file = output_file self.base = self.load_yaml(base_file) - def load_yaml(self, filename): + def is_url(self, filespec): + regex = re.compile('^([^/:]+)://') + return re.search(regex, filespec) + + def load_template(self, filespec): + try: + if(self.is_url(filespec)): + response = urllib2.urlopen(filespec) + return response.read() + else: + with io.open(filespec) as f: + return f.readlines() + except Exception as error: + err('Error opening template file: %s' % error) + + def load_yaml(self, filespec): try: - with io.open(filename) as yaml_file: - return yaml.load(yaml_file) + if(self.is_url(filespec)): + response = urllib2.urlopen(filespec) + return yaml.load(response) + else: + with io.open(filespec) as f: + return yaml.load(f) except Exception as error: err('Error opening YAML file: %s' % error) @@ -147,12 +167,11 @@ class Templater(object): regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END), flags=re.IGNORECASE) - with io.open(self.template_file) as f: - for line in f: - indent = self.get_indent(line) - result += re.sub(regex, - lambda match: self.parse_tag(match.group(1), indent), - line) + for line in self.load_template(self.template_file): + indent = self.get_indent(line) + result += re.sub(regex, + lambda match: self.parse_tag(match.group(1), indent), + line) self.save_yaml(self.output_file, result) @@ -164,9 +183,9 @@ template variable substitution and write the results to 'output_file'.''' parser = ArgParser(prog='python %s' % __file__, description=description) parser.add_argument('base_file', - help='Base YAML filename') + help='Base YAML file or URL') parser.add_argument('template_file', - help='Fragment filename') + help='Template file or URL') parser.add_argument('output_file', help='Output filename') diff --git a/deploy/templates/hardware_environment/vms/enea_lab/fuel.xml b/deploy/templates/hardware_environment/vms/enea_lab/fuel.xml new file mode 100644 index 000000000..15617f577 --- /dev/null +++ b/deploy/templates/hardware_environment/vms/enea_lab/fuel.xml @@ -0,0 +1,88 @@ +<domain type='kvm' id='1'> + <name>fuel</name> + <memory unit='KiB'>8290304</memory> + <currentMemory unit='KiB'>8290304</currentMemory> + <vcpu placement='static'>4</vcpu> + <resource> + <partition>/machine</partition> + </resource> + <os> + <type arch='x86_64' machine='pc-i440fx-rhel7.0.0'>hvm</type> + <boot dev='cdrom'/> + <boot dev='hd'/> + <bootmenu enable='no'/> + </os> + <features> + <acpi/> + <apic/> + <pae/> + </features> + <cpu mode='host-model'> + <model fallback='allow'/> + </cpu> + <clock offset='utc'> + <timer name='rtc' tickpolicy='catchup'/> + <timer name='pit' tickpolicy='delay'/> + <timer name='hpet' present='no'/> + </clock> + <on_poweroff>destroy</on_poweroff> + <on_reboot>restart</on_reboot> + <on_crash>restart</on_crash> + <pm> + <suspend-to-mem enabled='no'/> + <suspend-to-disk enabled='no'/> + </pm> + <devices> + <emulator>/usr/libexec/qemu-kvm</emulator> + <disk type='file' device='disk'> + <driver name='qemu' type='qcow2' cache='writeback'/> + <target dev='vda' bus='virtio'/> + </disk> + <disk type='block' device='cdrom'> + <driver name='qemu' type='raw'/> + <target dev='hdb' bus='ide'/> + <readonly/> + </disk> + <controller type='usb' index='0' model='ich9-ehci1'> + </controller> + <controller type='usb' index='0' model='ich9-uhci1'> + <master startport='0'/> + </controller> + <controller type='usb' index='0' model='ich9-uhci2'> + <master startport='2'/> + </controller> + <controller type='usb' index='0' model='ich9-uhci3'> + <master startport='4'/> + </controller> + <controller type='pci' index='0' model='pci-root'> + </controller> + <controller type='ide' index='0'> + </controller> + <controller type='virtio-serial' index='0'> + </controller> + <interface type='bridge'> + <model type='virtio'/> + </interface> + <interface type='bridge'> + <model type='virtio'/> + </interface> + <serial type='pty'> + <source path='/dev/pts/0'/> + <target port='0'/> + </serial> + <console type='pty' tty='/dev/pts/0'> + <source path='/dev/pts/0'/> + <target type='serial' port='0'/> + </console> + <input type='mouse' bus='ps2'/> + <input type='keyboard' bus='ps2'/> + <graphics type='vnc' port='5906' autoport='yes' listen='127.0.0.1'> + <listen type='address' address='127.0.0.1'/> + </graphics> + <video> + <model type='vga' vram='16384' heads='1'/> + </video> + <memballoon model='virtio'> + </memballoon> + </devices> +</domain> |