diff options
Diffstat (limited to 'deploy')
| -rw-r--r-- | deploy/common.py | 10 | ||||
| -rw-r--r-- | deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml | 9 | ||||
| -rw-r--r-- | deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml | 9 | ||||
| -rw-r--r-- | deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dha.yaml | 9 | ||||
| -rw-r--r-- | deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dha.yaml | 9 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-bgpvpn_0.0.2.yaml | 8 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-bgpvpn_0.0.3.yaml | 8 | ||||
| -rw-r--r-- | deploy/config/plugins/fuel-bgpvpn_0.9.0.yaml | 8 | ||||
| -rw-r--r-- | deploy/deploy-config.py | 475 | ||||
| -rwxr-xr-x | deploy/templater.py | 41 |
10 files changed, 335 insertions, 251 deletions
diff --git a/deploy/common.py b/deploy/common.py index 80832e201..07ad54372 100644 --- a/deploy/common.py +++ b/deploy/common.py @@ -74,13 +74,13 @@ def exec_cmd(cmd, check=True, attempts=1, delay=5, verbose=False, mask_args=[], if check: if return_code > 0: stderr = stderr.strip() - print "Failed command: " + str(masked_cmd) - print "Command returned response: " + str(stderr) - print "Command return code: " + str(return_code) + print("Failed command: " + str(masked_cmd)) + print("Command returned response: " + str(stderr)) + print("Command return code: " + str(return_code)) raise Exception(stderr) else: - print "Command: " + str(masked_cmd) - print str(response) + print("Command: " + str(masked_cmd)) + print(str(response)) return response return response, return_code diff --git a/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml index ad2f5ebde..0374c5d29 100644 --- a/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml +++ b/deploy/config/labs/devel-pipeline/default/fuel/config/dha.yaml @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## dha-pod-config-metadata: title: Deployment Hardware Adapter (DHA) for fuel development pipeline - default version version: 0.0.3 diff --git a/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml index 3c246c11c..c2352f924 100644 --- a/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml +++ b/deploy/config/labs/devel-pipeline/elx/fuel/config/dha.yaml @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## dha-pod-config-metadata: title: Deployment Hardware Adapter (DHA) for fuel development pipeline - Ericsson ELX version version: 0.0.3 diff --git a/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dha.yaml index f2aeabd4c..d8f904362 100644 --- a/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dha.yaml +++ b/deploy/config/labs/devel-pipeline/huawei-ch/fuel/config/dha.yaml @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## dha-pod-config-metadata: title: Deployment Hardware Adapter (DHA) for fuel development pipeline - default version version: 0.0.4 diff --git a/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dha.yaml b/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dha.yaml index 3657285e8..837293950 100644 --- a/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dha.yaml +++ b/deploy/config/labs/devel-pipeline/intel-sc/fuel/config/dha.yaml @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## dha-pod-config-metadata: title: Deployment Hardware Adapter (DHA) for fuel development pipeline - Intel Santa Clara, CA. USA version version: 0.0.1 diff --git a/deploy/config/plugins/fuel-bgpvpn_0.0.2.yaml b/deploy/config/plugins/fuel-bgpvpn_0.0.2.yaml index 21bcdf170..f7ab89b37 100644 --- a/deploy/config/plugins/fuel-bgpvpn_0.0.2.yaml +++ b/deploy/config/plugins/fuel-bgpvpn_0.0.2.yaml @@ -1,3 +1,11 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## plugin-config-metadata: title: BGPVPN fuel Plugin configuration template version: 0.2 diff --git a/deploy/config/plugins/fuel-bgpvpn_0.0.3.yaml b/deploy/config/plugins/fuel-bgpvpn_0.0.3.yaml index adca822fc..aeea9e450 100644 --- a/deploy/config/plugins/fuel-bgpvpn_0.0.3.yaml +++ b/deploy/config/plugins/fuel-bgpvpn_0.0.3.yaml @@ -1,3 +1,11 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## plugin-config-metadata: title: BGPVPN fuel Plugin configuration template version: 0.3 diff --git a/deploy/config/plugins/fuel-bgpvpn_0.9.0.yaml b/deploy/config/plugins/fuel-bgpvpn_0.9.0.yaml index bbdefaf6e..1aa1259ca 100644 --- a/deploy/config/plugins/fuel-bgpvpn_0.9.0.yaml +++ b/deploy/config/plugins/fuel-bgpvpn_0.9.0.yaml @@ -1,3 +1,11 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## plugin-config-metadata: title: BGPVPN fuel Plugin configuration template version: 0.9.0 diff --git a/deploy/deploy-config.py b/deploy/deploy-config.py index 8896080db..08cb36197 100644 --- a/deploy/deploy-config.py +++ b/deploy/deploy-config.py @@ -162,127 +162,6 @@ def merge_dicts(dict1, dict2): yield (k, dict1[k]) -setup_yaml() -kwargs = parse_arguments() - -# Generate final dea.yaml by merging following config files/fragments in revers priority order: -# "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override" -# and "deployment-scenario/dea-override" -print('Generating final dea.yaml configuration....') - -# Fetch dea-base, extract and purge meta-data -print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....") -response = urllib2.urlopen(kwargs["dea_base_uri"]) -dea_base_conf = yaml.load(response.read()) -dea_base_title = dea_base_conf['dea-base-config-metadata']['title'] -dea_base_version = dea_base_conf['dea-base-config-metadata']['version'] -dea_base_creation = dea_base_conf['dea-base-config-metadata']['created'] -dea_base_sha = sha_uri(kwargs["dea_base_uri"]) -dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment'] -dea_base_conf.pop('dea-base-config-metadata') -final_dea_conf = dea_base_conf -dea_pod_override_nodes = None - -# Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure -print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....") -response = urllib2.urlopen(kwargs["dea_pod_override_uri"]) -dea_pod_override_conf = yaml.load(response.read()) -if dea_pod_override_conf: - dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title'] - dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version'] - dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created'] - dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"]) - dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment'] - print('Merging dea-base and dea-pod-override configuration ....') - dea_pod_override_conf.pop('dea-pod-override-config-metadata') - # Copy the list of original nodes, which holds info on their transformations - if 'nodes' in dea_pod_override_conf: - dea_pod_override_nodes = list(dea_pod_override_conf['nodes']) - if dea_pod_override_conf: - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf)) - -# Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/ -# dea-override-configith previous dea data structure -print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....") - -response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml") -scenario_short_translation_conf = yaml.load(response.read()) -if kwargs["scenario"] in scenario_short_translation_conf: - scenario_uri = (kwargs["scenario_base_uri"] - + "/" - + scenario_short_translation_conf[kwargs["scenario"]]['configfile']) -else: - scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"] -response = urllib2.urlopen(scenario_uri) -deploy_scenario_conf = yaml.load(response.read()) - -if deploy_scenario_conf: - deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title'] - deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version'] - deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created'] - deploy_scenario_sha = sha_uri(scenario_uri) - deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment'] - deploy_scenario_conf.pop('deployment-scenario-metadata') -else: - print("Deployment scenario file not found or is empty") - print("Cannot continue, exiting ....") - sys.exit(1) - -dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"] -if dea_scenario_override_conf: - print('Merging dea-base-, dea-pod-override- and deployment-scenario ' - 'configuration into final dea.yaml configuration....') - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf)) - -# Fetch plugin-configuration configuration files, extract and purge meta-data, -# merge/append with previous dea data structure, override plugin-configuration with -# deploy-scenario/module-config-override -modules = [] -module_uris = [] -module_titles = [] -module_versions = [] -module_creations = [] -module_shas = [] -module_comments = [] -if deploy_scenario_conf["stack-extensions"]: - for module in deploy_scenario_conf["stack-extensions"]: - print('Loading configuration for module: ' - + module["module"] - + ' and merging it to final dea.yaml configuration....') - response = urllib2.urlopen(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml') - module_conf = yaml.load(response.read()) - modules.append(module["module"]) - module_uris.append(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml') - module_titles.append(str(module_conf['plugin-config-metadata']['title'])) - module_versions.append(str(module_conf['plugin-config-metadata']['version'])) - module_creations.append(str(module_conf['plugin-config-metadata']['created'])) - module_shas.append(sha_uri(kwargs["plugins_uri"] - + '/' - + module["module-config-name"] - + '_' - + module["module-config-version"] - + '.yaml')) - module_comments.append(str(module_conf['plugin-config-metadata']['comment'])) - module_conf.pop('plugin-config-metadata') - final_dea_conf['settings']['editable'].update(module_conf) - scenario_module_override_conf = module.get('module-config-override') - if scenario_module_override_conf: - dea_scenario_module_override_conf = {} - dea_scenario_module_override_conf['settings'] = {} - dea_scenario_module_override_conf['settings']['editable'] = {} - dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf - final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf)) - def get_node_ifaces_and_trans(nodes, nid): for node in nodes: if node['id'] == nid: @@ -293,117 +172,243 @@ def get_node_ifaces_and_trans(nodes, nid): return None -if dea_pod_override_nodes: - for node in final_dea_conf['nodes']: - data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id']) - if data: - print ("Honoring original interfaces and transformations for " - "node %d to %s, %s" % (node['id'], data[0], data[1])) - node['interfaces'] = data[0] - node['transformations'] = data[1] - -# Dump final dea.yaml including configuration management meta-data to argument provided -# directory -if not os.path.exists(kwargs["output_path"]): - os.makedirs(kwargs["output_path"]) -print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....') -with open(kwargs["output_path"] + '/dea.yaml', "w") as f: - f.write("\n".join([("title: DEA.yaml file automatically generated from the" - 'configuration files stated in the "configuration-files"' - "fragment below"), - "version: " + str(calendar.timegm(time.gmtime())), - "created: " + str(time.strftime("%d/%m/%Y")) + " " - + str(time.strftime("%H:%M:%S")), - "comment: none\n"])) - - f.write("\n".join(["configuration-files:", - " dea-base:", - " uri: " + kwargs["dea_base_uri"], - " title: " + str(dea_base_title), - " version: " + str(dea_base_version), - " created: " + str(dea_base_creation), - " sha1: " + str(dea_base_sha), - " comment: " + str(dea_base_comment) + "\n"])) - - f.write("\n".join([" pod-override:", - " uri: " + kwargs["dea_pod_override_uri"], - " title: " + str(dea_pod_title), - " version: " + str(dea_pod_version), - " created: " + str(dea_pod_creation), - " sha1: " + str(dea_pod_sha), - " comment: " + str(dea_pod_comment) + "\n"])) - - f.write("\n".join([" deployment-scenario:", - " uri: " + str(scenario_uri), - " title: " + str(deploy_scenario_title), - " version: " + str(deploy_scenario_version), - " created: " + str(deploy_scenario_creation), - " sha1: " + str(deploy_scenario_sha), - " comment: " + str(deploy_scenario_comment) + "\n"])) - - f.write(" plugin-modules:\n") - for k, _ in enumerate(modules): - f.write("\n".join([" - module: " + modules[k], - " uri: " + module_uris[k], - " title: " + module_titles[k], - " version: " + module_versions[k], - " created: " + module_creations[k], - " sha-1: " + module_shas[k], - " comment: " + module_comments[k] + "\n"])) - - yaml.dump(final_dea_conf, f, default_flow_style=False) - -# Load POD dha and override it with "deployment-scenario/dha-override-config" section -print('Generating final dha.yaml configuration....') -print('Parsing dha-pod yaml configuration....') -response = urllib2.urlopen(kwargs["dha_uri"]) -dha_pod_conf = yaml.load(response.read()) -dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title'] -dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version'] -dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created'] -dha_pod_sha = sha_uri(kwargs["dha_uri"]) -dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment'] -dha_pod_conf.pop('dha-pod-config-metadata') -final_dha_conf = dha_pod_conf - -dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"] -# Only virtual deploy scenarios can override dha.yaml since there -# is no way to programatically override a physical environment: -# wireing, IPMI set-up, etc. -# For Physical environments, dha.yaml overrides will be silently ignored -if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' - or final_dha_conf['adapter'] == 'esxi' - or final_dha_conf['adapter'] == 'vbox'): - print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....') - final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf)) - -# Dump final dha.yaml to argument provided directory -print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....') -with open(kwargs["output_path"] + '/dha.yaml', "w") as f: - f.write("\n".join([("title: DHA.yaml file automatically generated from" - "the configuration files stated in the" - '"configuration-files" fragment below'), - "version: " + str(calendar.timegm(time.gmtime())), - "created: " + str(time.strftime("%d/%m/%Y")) + " " - + str(time.strftime("%H:%M:%S")), - "comment: none\n"])) - - f.write("configuration-files:\n") - - f.write("\n".join([" dha-pod-configuration:", - " uri: " + kwargs["dha_uri"], - " title: " + str(dha_pod_title), - " version: " + str(dha_pod_version), - " created: " + str(dha_pod_creation), - " sha-1: " + str(dha_pod_sha), - " comment: " + str(dha_pod_comment) + "\n"])) - - f.write("\n".join([" deployment-scenario:", - " uri: " + str(scenario_uri), - " title: " + str(deploy_scenario_title), - " version: " + str(deploy_scenario_version), - " created: " + str(deploy_scenario_creation), - " sha-1: " + str(deploy_scenario_sha), - " comment: " + str(deploy_scenario_comment) + "\n"])) - - yaml.dump(final_dha_conf, f, default_flow_style=False) + +def main(): + setup_yaml() + kwargs = parse_arguments() + + # Generate final dea.yaml by merging following config files/fragments in revers priority order: + # "dea-base", "dea-pod-override", "deplyment-scenario/module-config-override" + # and "deployment-scenario/dea-override" + print('Generating final dea.yaml configuration....') + + # Fetch dea-base, extract and purge meta-data + print('Parsing dea-base from: ' + kwargs["dea_base_uri"] + "....") + response = urllib2.urlopen(kwargs["dea_base_uri"]) + dea_base_conf = yaml.load(response.read()) + dea_base_title = dea_base_conf['dea-base-config-metadata']['title'] + dea_base_version = dea_base_conf['dea-base-config-metadata']['version'] + dea_base_creation = dea_base_conf['dea-base-config-metadata']['created'] + dea_base_sha = sha_uri(kwargs["dea_base_uri"]) + dea_base_comment = dea_base_conf['dea-base-config-metadata']['comment'] + dea_base_conf.pop('dea-base-config-metadata') + final_dea_conf = dea_base_conf + dea_pod_override_nodes = None + + # Fetch dea-pod-override, extract and purge meta-data, merge with previous dea data structure + print('Parsing the dea-pod-override from: ' + kwargs["dea_pod_override_uri"] + "....") + response = urllib2.urlopen(kwargs["dea_pod_override_uri"]) + dea_pod_override_conf = yaml.load(response.read()) + if dea_pod_override_conf: + dea_pod_title = dea_pod_override_conf['dea-pod-override-config-metadata']['title'] + dea_pod_version = dea_pod_override_conf['dea-pod-override-config-metadata']['version'] + dea_pod_creation = dea_pod_override_conf['dea-pod-override-config-metadata']['created'] + dea_pod_sha = sha_uri(kwargs["dea_pod_override_uri"]) + dea_pod_comment = dea_pod_override_conf['dea-pod-override-config-metadata']['comment'] + print('Merging dea-base and dea-pod-override configuration ....') + dea_pod_override_conf.pop('dea-pod-override-config-metadata') + # Copy the list of original nodes, which holds info on their transformations + if 'nodes' in dea_pod_override_conf: + dea_pod_override_nodes = list(dea_pod_override_conf['nodes']) + if dea_pod_override_conf: + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_pod_override_conf)) + + # Fetch deployment-scenario, extract and purge meta-data, merge deployment-scenario/ + # dea-override-configith previous dea data structure + print('Parsing deployment-scenario from: ' + kwargs["scenario"] + "....") + + response = urllib2.urlopen(kwargs["scenario_base_uri"] + "/scenario.yaml") + scenario_short_translation_conf = yaml.load(response.read()) + if kwargs["scenario"] in scenario_short_translation_conf: + scenario_uri = (kwargs["scenario_base_uri"] + + "/" + + scenario_short_translation_conf[kwargs["scenario"]]['configfile']) + else: + scenario_uri = kwargs["scenario_base_uri"] + "/" + kwargs["scenario"] + response = urllib2.urlopen(scenario_uri) + deploy_scenario_conf = yaml.load(response.read()) + + if deploy_scenario_conf: + deploy_scenario_title = deploy_scenario_conf['deployment-scenario-metadata']['title'] + deploy_scenario_version = deploy_scenario_conf['deployment-scenario-metadata']['version'] + deploy_scenario_creation = deploy_scenario_conf['deployment-scenario-metadata']['created'] + deploy_scenario_sha = sha_uri(scenario_uri) + deploy_scenario_comment = deploy_scenario_conf['deployment-scenario-metadata']['comment'] + deploy_scenario_conf.pop('deployment-scenario-metadata') + else: + print("Deployment scenario file not found or is empty") + print("Cannot continue, exiting ....") + sys.exit(1) + + dea_scenario_override_conf = deploy_scenario_conf["dea-override-config"] + if dea_scenario_override_conf: + print('Merging dea-base-, dea-pod-override- and deployment-scenario ' + 'configuration into final dea.yaml configuration....') + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_override_conf)) + + # Fetch plugin-configuration configuration files, extract and purge meta-data, + # merge/append with previous dea data structure, override plugin-configuration with + # deploy-scenario/module-config-override + modules = [] + module_uris = [] + module_titles = [] + module_versions = [] + module_creations = [] + module_shas = [] + module_comments = [] + if deploy_scenario_conf["stack-extensions"]: + for module in deploy_scenario_conf["stack-extensions"]: + print('Loading configuration for module: ' + + module["module"] + + ' and merging it to final dea.yaml configuration....') + response = urllib2.urlopen(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') + module_conf = yaml.load(response.read()) + modules.append(module["module"]) + module_uris.append(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml') + module_titles.append(str(module_conf['plugin-config-metadata']['title'])) + module_versions.append(str(module_conf['plugin-config-metadata']['version'])) + module_creations.append(str(module_conf['plugin-config-metadata']['created'])) + module_shas.append(sha_uri(kwargs["plugins_uri"] + + '/' + + module["module-config-name"] + + '_' + + module["module-config-version"] + + '.yaml')) + module_comments.append(str(module_conf['plugin-config-metadata']['comment'])) + module_conf.pop('plugin-config-metadata') + final_dea_conf['settings']['editable'].update(module_conf) + scenario_module_override_conf = module.get('module-config-override') + if scenario_module_override_conf: + dea_scenario_module_override_conf = {} + dea_scenario_module_override_conf['settings'] = {} + dea_scenario_module_override_conf['settings']['editable'] = {} + dea_scenario_module_override_conf['settings']['editable'][module["module"]] = scenario_module_override_conf + final_dea_conf = dict(merge_dicts(final_dea_conf, dea_scenario_module_override_conf)) + + if dea_pod_override_nodes: + for node in final_dea_conf['nodes']: + data = get_node_ifaces_and_trans(dea_pod_override_nodes, node['id']) + if data: + print ("Honoring original interfaces and transformations for " + "node %d to %s, %s" % (node['id'], data[0], data[1])) + node['interfaces'] = data[0] + node['transformations'] = data[1] + + # Dump final dea.yaml including configuration management meta-data to argument provided + # directory + if not os.path.exists(kwargs["output_path"]): + os.makedirs(kwargs["output_path"]) + print('Dumping final dea.yaml to ' + kwargs["output_path"] + '/dea.yaml....') + with open(kwargs["output_path"] + '/dea.yaml', "w") as f: + f.write("\n".join([("title: DEA.yaml file automatically generated from the" + 'configuration files stated in the "configuration-files"' + "fragment below"), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + str(time.strftime("%d/%m/%Y")) + " " + + str(time.strftime("%H:%M:%S")), + "comment: none\n"])) + + f.write("\n".join(["configuration-files:", + " dea-base:", + " uri: " + kwargs["dea_base_uri"], + " title: " + str(dea_base_title), + " version: " + str(dea_base_version), + " created: " + str(dea_base_creation), + " sha1: " + str(dea_base_sha), + " comment: " + str(dea_base_comment) + "\n"])) + + f.write("\n".join([" pod-override:", + " uri: " + kwargs["dea_pod_override_uri"], + " title: " + str(dea_pod_title), + " version: " + str(dea_pod_version), + " created: " + str(dea_pod_creation), + " sha1: " + str(dea_pod_sha), + " comment: " + str(dea_pod_comment) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + str(scenario_uri), + " title: " + str(deploy_scenario_title), + " version: " + str(deploy_scenario_version), + " created: " + str(deploy_scenario_creation), + " sha1: " + str(deploy_scenario_sha), + " comment: " + str(deploy_scenario_comment) + "\n"])) + + f.write(" plugin-modules:\n") + for k, _ in enumerate(modules): + f.write("\n".join([" - module: " + modules[k], + " uri: " + module_uris[k], + " title: " + module_titles[k], + " version: " + module_versions[k], + " created: " + module_creations[k], + " sha-1: " + module_shas[k], + " comment: " + module_comments[k] + "\n"])) + + yaml.dump(final_dea_conf, f, default_flow_style=False) + + # Load POD dha and override it with "deployment-scenario/dha-override-config" section + print('Generating final dha.yaml configuration....') + print('Parsing dha-pod yaml configuration....') + response = urllib2.urlopen(kwargs["dha_uri"]) + dha_pod_conf = yaml.load(response.read()) + dha_pod_title = dha_pod_conf['dha-pod-config-metadata']['title'] + dha_pod_version = dha_pod_conf['dha-pod-config-metadata']['version'] + dha_pod_creation = dha_pod_conf['dha-pod-config-metadata']['created'] + dha_pod_sha = sha_uri(kwargs["dha_uri"]) + dha_pod_comment = dha_pod_conf['dha-pod-config-metadata']['comment'] + dha_pod_conf.pop('dha-pod-config-metadata') + final_dha_conf = dha_pod_conf + + dha_scenario_override_conf = deploy_scenario_conf["dha-override-config"] + # Only virtual deploy scenarios can override dha.yaml since there + # is no way to programatically override a physical environment: + # wireing, IPMI set-up, etc. + # For Physical environments, dha.yaml overrides will be silently ignored + if dha_scenario_override_conf and (final_dha_conf['adapter'] == 'libvirt' + or final_dha_conf['adapter'] == 'esxi' + or final_dha_conf['adapter'] == 'vbox'): + print('Merging dha-pod and deployment-scenario override information to final dha.yaml configuration....') + final_dha_conf = dict(merge_dicts(final_dha_conf, dha_scenario_override_conf)) + + # Dump final dha.yaml to argument provided directory + print('Dumping final dha.yaml to ' + kwargs["output_path"] + '/dha.yaml....') + with open(kwargs["output_path"] + '/dha.yaml', "w") as f: + f.write("\n".join([("title: DHA.yaml file automatically generated from" + "the configuration files stated in the" + '"configuration-files" fragment below'), + "version: " + str(calendar.timegm(time.gmtime())), + "created: " + str(time.strftime("%d/%m/%Y")) + " " + + str(time.strftime("%H:%M:%S")), + "comment: none\n"])) + + f.write("configuration-files:\n") + + f.write("\n".join([" dha-pod-configuration:", + " uri: " + kwargs["dha_uri"], + " title: " + str(dha_pod_title), + " version: " + str(dha_pod_version), + " created: " + str(dha_pod_creation), + " sha-1: " + str(dha_pod_sha), + " comment: " + str(dha_pod_comment) + "\n"])) + + f.write("\n".join([" deployment-scenario:", + " uri: " + str(scenario_uri), + " title: " + str(deploy_scenario_title), + " version: " + str(deploy_scenario_version), + " created: " + str(deploy_scenario_creation), + " sha-1: " + str(deploy_scenario_sha), + " comment: " + str(deploy_scenario_comment) + "\n"])) + + yaml.dump(final_dha_conf, f, default_flow_style=False) + +if __name__ == '__main__': + main() diff --git a/deploy/templater.py b/deploy/templater.py index 6b41e1f3c..bda60c7fe 100755 --- a/deploy/templater.py +++ b/deploy/templater.py @@ -12,6 +12,7 @@ import io import re import yaml +import urllib2 from common import( err, ArgParser, @@ -29,10 +30,29 @@ class Templater(object): self.output_file = output_file self.base = self.load_yaml(base_file) - def load_yaml(self, filename): + def is_url(self, filespec): + regex = re.compile('^([^/:]+)://') + return re.search(regex, filespec) + + def load_template(self, filespec): + try: + if(self.is_url(filespec)): + response = urllib2.urlopen(filespec) + return response.read() + else: + with io.open(filespec) as f: + return f.readlines() + except Exception as error: + err('Error opening template file: %s' % error) + + def load_yaml(self, filespec): try: - with io.open(filename) as yaml_file: - return yaml.load(yaml_file) + if(self.is_url(filespec)): + response = urllib2.urlopen(filespec) + return yaml.load(response) + else: + with io.open(filespec) as f: + return yaml.load(f) except Exception as error: err('Error opening YAML file: %s' % error) @@ -147,12 +167,11 @@ class Templater(object): regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END), flags=re.IGNORECASE) - with io.open(self.template_file) as f: - for line in f: - indent = self.get_indent(line) - result += re.sub(regex, - lambda match: self.parse_tag(match.group(1), indent), - line) + for line in self.load_template(self.template_file): + indent = self.get_indent(line) + result += re.sub(regex, + lambda match: self.parse_tag(match.group(1), indent), + line) self.save_yaml(self.output_file, result) @@ -164,9 +183,9 @@ template variable substitution and write the results to 'output_file'.''' parser = ArgParser(prog='python %s' % __file__, description=description) parser.add_argument('base_file', - help='Base YAML filename') + help='Base YAML file or URL') parser.add_argument('template_file', - help='Fragment filename') + help='Template file or URL') parser.add_argument('output_file', help='Output filename') |