diff options
25 files changed, 643 insertions, 22 deletions
@@ -44,3 +44,18 @@ doc/_build # Built by pbr (python setup.py sdist): AUTHORS ChangeLog + +extraconfig/all_nodes/mac_hostname.yaml +extraconfig/all_nodes/random_string.yaml +extraconfig/all_nodes/swap-partition.yaml +extraconfig/all_nodes/swap.yaml +extraconfig/tasks/major_upgrade_pacemaker_init.yaml +network/service_net_map.yaml +overcloud-resource-registry-puppet.yaml +overcloud.yaml +puppet/blockstorage-config.yaml +puppet/cephstorage-config.yaml +puppet/compute-config.yaml +puppet/controller-config.yaml +puppet/objectstorage-config.yaml +puppet/post.yaml diff --git a/ci/environments/scenario003-multinode.yaml b/ci/environments/scenario003-multinode.yaml index e540bc55..25fe1697 100644 --- a/ci/environments/scenario003-multinode.yaml +++ b/ci/environments/scenario003-multinode.yaml @@ -1,11 +1,11 @@ resource_registry: OS::TripleO::Controller::Net::SoftwareConfig: ../common/net-config-multinode.yaml OS::TripleO::Compute::Net::SoftwareConfig: ../common/net-config-multinode.yaml - OS::TripleO::Services::SaharaApi: /usr/share/openstack-tripleo-heat-templates/puppet/services/sahara-api.yaml - OS::TripleO::Services::SaharaEngine: /usr/share/openstack-tripleo-heat-templates/puppet/services/sahara-engine.yaml - OS::TripleO::Services::MistralApi: /usr/share/openstack-tripleo-heat-templates/puppet/services/mistral-api.yaml - OS::TripleO::Services::MistralEngine: /usr/share/openstack-tripleo-heat-templates/puppet/services/mistral-engine.yaml - OS::TripleO::Services::MistralExecutor: /usr/share/openstack-tripleo-heat-templates/puppet/services/mistral-executor.yaml + OS::TripleO::Services::SaharaApi: ../../puppet/services/sahara-api.yaml + OS::TripleO::Services::SaharaEngine: ../../puppet/services/sahara-engine.yaml + OS::TripleO::Services::MistralApi: ../../puppet/services/mistral-api.yaml + OS::TripleO::Services::MistralEngine: ../../puppet/services/mistral-engine.yaml + OS::TripleO::Services::MistralExecutor: ../../puppet/services/mistral-executor.yaml parameter_defaults: ControllerServices: diff --git a/ci/pingtests/scenario001-multinode.yaml b/ci/pingtests/scenario001-multinode.yaml index 9dcbd390..7374846f 100644 --- a/ci/pingtests/scenario001-multinode.yaml +++ b/ci/pingtests/scenario001-multinode.yaml @@ -118,19 +118,21 @@ resources: ram: 512 vcpus: 1 - gnocchi_res_alarm: - type: OS::Aodh::GnocchiResourcesAlarm - properties: - description: Do stuff with gnocchi - metric: cpu_util - aggregation_method: mean - granularity: 60 - evaluation_periods: 1 - threshold: 50 - alarm_actions: [] - resource_type: instance - resource_id: { get_resource: server1 } - comparison_operator: gt +# Disabling this resource now +# https://bugs.launchpad.net/tripleo/+bug/1646506 +# gnocchi_res_alarm: +# type: OS::Aodh::GnocchiResourcesAlarm +# properties: +# description: Do stuff with gnocchi +# metric: cpu_util +# aggregation_method: mean +# granularity: 60 +# evaluation_periods: 1 +# threshold: 50 +# alarm_actions: [] +# resource_type: instance +# resource_id: { get_resource: server1 } +# comparison_operator: gt asg: type: OS::Heat::AutoScalingGroup diff --git a/environments/major-upgrade-composable-steps.yaml b/environments/major-upgrade-composable-steps.yaml new file mode 100644 index 00000000..7e10014b --- /dev/null +++ b/environments/major-upgrade-composable-steps.yaml @@ -0,0 +1,3 @@ +resource_registry: + OS::TripleO::UpgradeSteps: ../puppet/major_upgrade_steps.yaml + OS::TripleO::PostDeploySteps: OS::Heat::None diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 30b9f2b9..21013bdc 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -100,6 +100,10 @@ resource_registry: # validation resources OS::TripleO::AllNodes::Validation: all-nodes-validation.yaml + # Upgrade resources + OS::TripleO::UpgradeConfig: puppet/upgrade_config.yaml + OS::TripleO::UpgradeSteps: OS::Heat::None + # services OS::TripleO::Services: puppet/services/services.yaml OS::TripleO::Services::Apache: puppet/services/apache.yaml diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index 6146a48a..39a092b1 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -604,6 +604,21 @@ resources: {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} {% endfor %} + # Upgrade steps for all roles + AllNodesUpgradeSteps: + type: OS::TripleO::UpgradeSteps + depends_on: AllNodesDeploySteps + properties: + servers: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}, attributes, nova_server_resource]} +{% endfor %} + role_data: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} +{% endfor %} + + outputs: ManagedEndpoints: description: Asserts that the keystone endpoints have been provisioned. @@ -636,3 +651,9 @@ outputs: {% for role in roles %} {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]} {% endfor %} + RoleData: + description: The configuration data associated with each role + value: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}ServiceChain, role_data]} +{% endfor %} diff --git a/puppet/blockstorage-role.yaml b/puppet/blockstorage-role.yaml index 45552e05..c9bf894f 100644 --- a/puppet/blockstorage-role.yaml +++ b/puppet/blockstorage-role.yaml @@ -266,6 +266,42 @@ resources: extraconfig: {get_param: ExtraConfig} volume: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [BlockStorage, name]} + - ctlplane + - {get_param: CloudDomain} # Resource for site-specific injection of root certificate NodeTLSCAData: diff --git a/puppet/cephstorage-role.yaml b/puppet/cephstorage-role.yaml index 03b57e2b..18787a21 100644 --- a/puppet/cephstorage-role.yaml +++ b/puppet/cephstorage-role.yaml @@ -271,6 +271,42 @@ resources: extraconfig: {get_param: ExtraConfig} ceph: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [CephStorage, name]} + - ctlplane + - {get_param: CloudDomain} # Resource for site-specific injection of root certificate NodeTLSCAData: diff --git a/puppet/compute-role.yaml b/puppet/compute-role.yaml index 13464339..f359bf70 100644 --- a/puppet/compute-role.yaml +++ b/puppet/compute-role.yaml @@ -284,6 +284,42 @@ resources: extraconfig: {get_param: ExtraConfig} compute: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [NovaCompute, name]} + - ctlplane + - {get_param: CloudDomain} NovaComputeDeployment: type: OS::TripleO::SoftwareDeployment diff --git a/puppet/controller-role.yaml b/puppet/controller-role.yaml index 09e10eae..77b54ff3 100644 --- a/puppet/controller-role.yaml +++ b/puppet/controller-role.yaml @@ -344,6 +344,42 @@ resources: # Misc tripleo::haproxy::service_certificate: {get_attr: [NodeTLSData, deployed_ssl_certificate_path]} tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [Controller, name]} + - ctlplane + - {get_param: CloudDomain} # Hook for site-specific additional pre-deployment config, e.g extra hieradata ControllerExtraConfigPre: diff --git a/puppet/major_upgrade_steps.j2.yaml b/puppet/major_upgrade_steps.j2.yaml new file mode 100644 index 00000000..f8dad433 --- /dev/null +++ b/puppet/major_upgrade_steps.j2.yaml @@ -0,0 +1,98 @@ +heat_template_version: 2016-10-14 +description: 'Upgrade steps for all roles' + +parameters: + servers: + type: json + + role_data: + type: json + description: Mapping of Role name e.g Controller to the per-role data + + UpdateIdentifier: + type: string + description: > + Setting to a previously unused value during stack-update will trigger + the Upgrade resources to re-run on all roles. + + UpgradeInitCommand: + type: string + description: | + Command or script snippet to run on all overcloud nodes to + initialize the upgrade process. E.g. a repository switch. + default: '' + +resources: + + # For the UpgradeInit also rename /etc/resolv.conf.save for +bug/1567004 + UpgradeInitConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - "#!/bin/bash\n\n" + - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n" + - get_param: UpgradeInitCommand + +{% for role in roles %} + {{role.name}}Upgrade_Init: + type: OS::Heat::StructuredDeploymentGroup + properties: + name: {{role.name}}Upgrade_Init + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: UpgradeInitConfig} +{% endfor %} + +# Upgrade Steps for all roles +# FIXME(shardy): would be nice to make the number of steps configurable +{% for step in range(1, 8) %} + {% for role in roles %} + # Step {{step}} resources + {{role.name}}UpgradeConfig_Step{{step}}: + type: OS::TripleO::UpgradeConfig + # The UpgradeConfig resources could actually be created without + # serialization, but the event output is easier to follow if we + # do, and there should be minimal performance hit (creating the + # config is cheap compared to the time to apply the deployment). + depends_on: + {% if step == 1 %} + - {{role.name}}Upgrade_Init + {% else %} + {% for dep in roles %} + - {{dep.name}}Upgrade_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + UpgradeStepConfig: {get_param: [role_data, {{role.name}}, upgrade_tasks]} + step: {{step}} + + {{role.name}}Upgrade_Step{{step}}: + type: OS::Heat::StructuredDeploymentGroup + {% if step > 1 %} + depends_on: + {% for dep in roles %} + - {{dep.name}}Upgrade_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + name: {{role.name}}Upgrade_Step{{step}} + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: {{role.name}}UpgradeConfig_Step{{step}}} + input_values: + role: {{role.name}} + update_identifier: {get_param: UpdateIdentifier} + {% endfor %} +{% endfor %} + +outputs: + # Output the config for each role, just use Step1 as the config should be + # the same for all steps (only the tag provided differs) + upgrade_configs: + description: The per-role upgrade configuration used + value: +{% for role in roles %} + {{role.name.lower()}}: {get_attr: [{{role.name}}UpgradeConfig_Step1, upgrade_config]} +{% endfor %} + diff --git a/puppet/objectstorage-role.yaml b/puppet/objectstorage-role.yaml index 533cd2c1..60c12c3b 100644 --- a/puppet/objectstorage-role.yaml +++ b/puppet/objectstorage-role.yaml @@ -255,6 +255,43 @@ resources: extraconfig: {get_param: ExtraConfig} object: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [SwiftStorage, name]} + - ctlplane + - {get_param: CloudDomain} + SwiftStorageHieraDeploy: type: OS::Heat::StructuredDeployment diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml index c2c322d8..587ff58d 100644 --- a/puppet/role.role.j2.yaml +++ b/puppet/role.role.j2.yaml @@ -5,13 +5,17 @@ parameters: description: Flavor for the {{role}} node. default: baremetal type: string +{% if disable_constraints is not defined %} constraints: - custom_constraint: nova.flavor +{% endif %} {{role}}Image: type: string default: overcloud-full +{% if disable_constraints is not defined %} constraints: - custom_constraint: glance.image +{% endif %} ImageUpdatePolicy: default: 'REBUILD_PRESERVE_EPHEMERAL' description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt. @@ -20,8 +24,10 @@ parameters: description: Name of an existing Nova key pair to enable SSH access to the instances type: string default: default +{% if disable_constraints is not defined %} constraints: - custom_constraint: nova.keypair +{% endif %} ServiceNetMap: default: {} description: Mapping of service_name -> network name. Typically set @@ -279,6 +285,42 @@ resources: tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade} tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources} tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups} + fqdn_internal_api: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - internalapi + - {get_param: CloudDomain} + fqdn_storage: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - storage + - {get_param: CloudDomain} + fqdn_storage_mgmt: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - storagemgmt + - {get_param: CloudDomain} + fqdn_tenant: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - tenant + - {get_param: CloudDomain} + fqdn_management: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - management + - {get_param: CloudDomain} + fqdn_ctlplane: + list_join: + - '.' + - - {get_attr: [{{role}}, name]} + - ctlplane + - {get_param: CloudDomain} # Resource for site-specific injection of root certificate NodeTLSCAData: diff --git a/puppet/services/README.rst b/puppet/services/README.rst index 3accff36..856b306e 100644 --- a/puppet/services/README.rst +++ b/puppet/services/README.rst @@ -22,8 +22,8 @@ Config Settings Each service may define a config_settings output variable which returns Hiera settings to be configured. -Steps ------ +Deployment Steps +---------------- Each service may define an output variable which returns a puppet manifest snippet that will run at each of the following steps. Earlier manifests @@ -48,3 +48,29 @@ are re-asserted when applying latter ones. 4) General OpenStack Services 5) Service activation (Pacemaker) + +Upgrade Steps +------------- + +Each service template may optionally define a `upgrade_tasks` key, which is a +list of ansible tasks to be performed during the upgrade process. + +Similar to the step_config, we allow a series of steps for the per-service +upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first +step, "step2" for the second, etc. + + Steps/tages correlate to the following: + + 1) Quiesce the control-plane, e.g disable LoadBalancer, stop pacemaker cluster + + 2) Stop all control-plane services, ready for upgrade + + 3) Perform a package update, (either specific packages or the whole system) + + 4) Start services needed for migration tasks (e.g DB) + + 5) Perform any migration tasks, e.g DB sync commands + + 6) Start control-plane services + + 7) Any additional online migration tasks (e.g data migrations) diff --git a/puppet/services/database/mysql.yaml b/puppet/services/database/mysql.yaml index bed8f7d3..cacf6db0 100644 --- a/puppet/services/database/mysql.yaml +++ b/puppet/services/database/mysql.yaml @@ -92,3 +92,11 @@ outputs: $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]} step_config: | include ::tripleo::profile::base::database::mysql + upgrade_tasks: + - name: Stop service + tags: step2 + service: name=mariadb state=stopped + - name: Start service + tags: step4 + service: name=mariadb state=started + diff --git a/puppet/services/haproxy.yaml b/puppet/services/haproxy.yaml index c8edade5..675a79ec 100644 --- a/puppet/services/haproxy.yaml +++ b/puppet/services/haproxy.yaml @@ -77,3 +77,10 @@ outputs: - get_attr: [HAProxyInternalTLS, role_data, certificates_specs] step_config: | include ::tripleo::profile::base::haproxy + upgrade_tasks: + - name: Stop haproxy service + tags: step1 + service: name=haproxy state=stopped + - name: Start haproxy service + tags: step4 # Needed at step 4 for mysql + service: name=haproxy state=started diff --git a/puppet/services/keepalived.yaml b/puppet/services/keepalived.yaml index 6f2c44ec..b4f1a100 100644 --- a/puppet/services/keepalived.yaml +++ b/puppet/services/keepalived.yaml @@ -64,3 +64,11 @@ outputs: - tripleo::keepalived::public_virtual_interface: {get_param: PublicVirtualInterface} step_config: | include ::tripleo::profile::base::keepalived + upgrade_tasks: + - name: Stop keepalived service + tags: step1 + service: name=keepalived state=stopped + - name: Start keepalived service + tags: step4 # Needed at step 4 for mysql + service: name=keepalived state=started + diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml index c2a282d4..f021e18b 100644 --- a/puppet/services/keystone.yaml +++ b/puppet/services/keystone.yaml @@ -248,3 +248,14 @@ outputs: keystone::db::mysql::allowed_hosts: - '%' - "%{hiera('mysql_bind_host')}" + # Ansible tasks to handle upgrade + upgrade_tasks: + - name: Stop keystone service (running under httpd) + tags: step2 + service: name=httpd state=stopped + - name: Sync keystone DB + tags: step5 + command: keystone-manage db_sync + - name: Start keystone service (running under httpd) + tags: step6 + service: name=httpd state=started diff --git a/puppet/services/rabbitmq.yaml b/puppet/services/rabbitmq.yaml index 44a09a42..b77e0a91 100644 --- a/puppet/services/rabbitmq.yaml +++ b/puppet/services/rabbitmq.yaml @@ -97,6 +97,13 @@ outputs: # internal_api_subnet - > IP/CIDR rabbitmq::node_ip_address: {get_param: [ServiceNetMap, RabbitmqNetwork]} rabbitmq::nr_ha_queues: {get_param: RabbitHAQueues} - step_config: | include ::tripleo::profile::base::rabbitmq + upgrade_tasks: + - name: Stop rabbitmq service + tags: step2 + service: name=rabbitmq-server state=stopped + - name: Start rabbitmq service + tags: step6 + service: name=rabbitmq-server state=started + diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml index ffe2d2d4..13df5bbe 100644 --- a/puppet/services/services.yaml +++ b/puppet/services/services.yaml @@ -108,3 +108,8 @@ outputs: expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {}) data: {role_data: {get_attr: [ServiceChain, role_data]}} step_config: {list_join: ["\n", {get_attr: [ServiceChain, role_data, step_config]}]} + upgrade_tasks: + yaql: + # Note we use distinct() here to filter any identical tasks, e.g yum update for all services + expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct() + data: {get_attr: [ServiceChain, role_data]} diff --git a/puppet/services/tripleo-packages.yaml b/puppet/services/tripleo-packages.yaml index 124f5fe8..69912fa5 100644 --- a/puppet/services/tripleo-packages.yaml +++ b/puppet/services/tripleo-packages.yaml @@ -32,3 +32,7 @@ outputs: tripleo::packages::enable_install: {get_param: EnablePackageInstall} step_config: | include ::tripleo::packages + upgrade_tasks: + - name: Update all packages + tags: step3 + yum: name=* state=latest diff --git a/puppet/upgrade_config.yaml b/puppet/upgrade_config.yaml new file mode 100644 index 00000000..c67e10b3 --- /dev/null +++ b/puppet/upgrade_config.yaml @@ -0,0 +1,48 @@ +heat_template_version: 2016-10-14 +description: 'Upgrade for via ansible by applying a step related tag' + +parameters: + UpgradeStepConfig: + type: json + description: Config (ansible yaml) that will be used to step through the deployment. + default: '' + + step: + type: string + description: Step number of the upgrade + +resources: + + AnsibleConfig: + type: OS::Heat::Value + properties: + value: + str_replace: + template: CONFIG + params: + CONFIG: + - hosts: localhost + connection: local + tasks: {get_param: UpgradeStepConfig} + + AnsibleUpgradeConfigImpl: + type: OS::Heat::SoftwareConfig + properties: + group: ansible + options: + tags: + str_replace: + template: "stepSTEP" + params: + STEP: {get_param: step} + inputs: + - name: role + config: {get_attr: [AnsibleConfig, value]} + +outputs: + OS::stack_id: + description: The software config which runs ansible with tags + value: {get_resource: AnsibleUpgradeConfigImpl} + upgrade_config: + description: The configuration file used for upgrade + value: {get_attr: [AnsibleConfig, value]} diff --git a/requirements.txt b/requirements.txt index 4e46b891..9c4a708a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pbr>=0.5.21,<1.0 +Jinja2>=2.8 # BSD License (3 clause) diff --git a/tools/process-templates.py b/tools/process-templates.py new file mode 100755 index 00000000..a15b00e2 --- /dev/null +++ b/tools/process-templates.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import jinja2 +import os +import sys +import yaml + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Configure host network interfaces using a JSON' + ' config file format.') + parser.add_argument('-p', '--base_path', metavar='BASE_PATH', + help="""base path of templates to process.""", + default='.') + parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA', + help="""relative path to the roles_data.yaml file.""", + default='roles_data.yaml') + parser.add_argument('--safe', + action='store_true', + help="""Enable safe mode (do not overwrite files).""", + default=False) + opts = parser.parse_args(argv[1:]) + + return opts + + +def _j2_render_to_file(j2_template, j2_data, outfile_name=None, + overwrite=True): + yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml') + print('rendering j2 template to file: %s' % outfile_name) + + if not overwrite and os.path.exists(outfile_name): + print('ERROR: path already exists for file: %s' % outfile_name) + sys.exit(1) + + try: + # Render the j2 template + template = jinja2.Environment().from_string(j2_template) + r_template = template.render(**j2_data) + except jinja2.exceptions.TemplateError as ex: + error_msg = ("Error rendering template %s : %s" + % (yaml_f, six.text_type(ex))) + print(error_msg) + raise Exception(error_msg) + with open(outfile_name, 'w') as out_f: + out_f.write(r_template) + + +def process_templates(template_path, role_data_path, overwrite): + + with open(role_data_path) as role_data_file: + role_data = yaml.safe_load(role_data_file) + + j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml') + with open(j2_excludes_path) as role_data_file: + j2_excludes = yaml.safe_load(role_data_file) + + role_names = [r.get('name') for r in role_data] + r_map = {} + for r in role_data: + r_map[r.get('name')] = r + excl_templates = ['%s/%s' % (template_path, e) + for e in j2_excludes.get('name')] + + if os.path.isdir(template_path): + for subdir, dirs, files in os.walk(template_path): + for f in files: + file_path = os.path.join(subdir, f) + # We do two templating passes here: + # 1. *.role.j2.yaml - we template just the role name + # and create multiple files (one per role) + # 2. *.j2.yaml - we template with all roles_data, + # and create one file common to all roles + if f.endswith('.role.j2.yaml'): + print("jinja2 rendering role template %s" % f) + with open(file_path) as j2_template: + template_data = j2_template.read() + print("jinja2 rendering roles %s" % "," + .join(role_names)) + for role in role_names: + j2_data = {'role': role} + # (dprince) For the undercloud installer we don't + # want to have heat check nova/glance API's + if r_map[role].get('disable_constraints', False): + j2_data['disable_constraints'] = True + out_f = "-".join( + [role.lower(), + os.path.basename(f).replace('.role.j2.yaml', + '.yaml')]) + out_f_path = os.path.join(subdir, out_f) + if not (out_f_path in excl_templates): + _j2_render_to_file(template_data, j2_data, + out_f_path, overwrite) + else: + print('skipping rendering of %s' % out_f_path) + elif f.endswith('.j2.yaml'): + print("jinja2 rendering normal template %s" % f) + with open(file_path) as j2_template: + template_data = j2_template.read() + j2_data = {'roles': role_data} + out_f = file_path.replace('.j2.yaml', '.yaml') + _j2_render_to_file(template_data, j2_data, out_f, + overwrite) + + else: + print('Unexpected argument %s' % template_path) + +opts = parse_opts(sys.argv) + +role_data_path = os.path.join(opts.base_path, opts.roles_data) + +process_templates(opts.base_path, role_data_path, (not opts.safe)) @@ -11,4 +11,9 @@ deps = -r{toxinidir}/requirements.txt commands = {posargs} [testenv:pep8] -commands = python ./tools/yaml-validate.py . +commands = + python ./tools/process-templates.py + python ./tools/yaml-validate.py . + +[testenv:templates] +commands = python ./tools/process-templates.py |