diff options
47 files changed, 854 insertions, 93 deletions
diff --git a/ci/environments/scenario002-multinode-containers.yaml b/ci/environments/scenario002-multinode-containers.yaml index 7191deae..92c834b6 100644 --- a/ci/environments/scenario002-multinode-containers.yaml +++ b/ci/environments/scenario002-multinode-containers.yaml @@ -6,9 +6,10 @@ resource_registry: OS::TripleO::Controller::Net::SoftwareConfig: ../common/net-config-multinode.yaml OS::TripleO::Compute::Net::SoftwareConfig: ../common/net-config-multinode.yaml - OS::TripleO::Services::BarbicanApi: ../../puppet/services/barbican-api.yaml - OS::TripleO::Services::Zaqar: ../../puppet/services/zaqar.yaml - OS::TripleO::Services::Ec2Api: ../../puppet/services/ec2-api.yaml + # TODO: Barbican is not yet containerized: https://review.openstack.org/#/c/474327 + # OS::TripleO::Services::BarbicanApi: ../../docker/services/barbican-api.yaml + OS::TripleO::Services::Zaqar: ../../docker/services/zaqar.yaml + OS::TripleO::Services::Ec2Api: ../../docker/services/ec2-api.yaml # NOTE: This is needed because of upgrades from Ocata to Pike. We # deploy the initial environment with Ocata templates, and # overcloud-resource-registry.yaml there doesn't have this Docker diff --git a/ci/environments/scenario004-multinode-containers.yaml b/ci/environments/scenario004-multinode-containers.yaml index 7a6724de..1d6d5917 100644 --- a/ci/environments/scenario004-multinode-containers.yaml +++ b/ci/environments/scenario004-multinode-containers.yaml @@ -6,6 +6,7 @@ resource_registry: OS::TripleO::Controller::Net::SoftwareConfig: ../common/net-config-multinode.yaml OS::TripleO::Compute::Net::SoftwareConfig: ../common/net-config-multinode.yaml + # TODO deploy ceph with ceph-ansible: https://review.openstack.org/#/c/465066/ OS::TripleO::Services::CephMds: ../../puppet/services/ceph-mds.yaml OS::TripleO::Services::CephMon: ../../puppet/services/ceph-mon.yaml OS::TripleO::Services::CephOSD: ../../puppet/services/ceph-osd.yaml @@ -13,10 +14,12 @@ resource_registry: OS::TripleO::Services::SwiftProxy: OS::Heat::None OS::TripleO::Services::SwiftStorage: OS::Heat::None OS::TripleO::Services::SwiftRingBuilder: OS::Heat::None - OS::TripleO::Services::ManilaApi: ../../puppet/services/manila-api.yaml - OS::TripleO::Services::ManilaScheduler: ../../puppet/services/manila-scheduler.yaml + OS::TripleO::Services::ManilaApi: ../../docker/services/manila-api.yaml + OS::TripleO::Services::ManilaScheduler: ../../docker/services/manila-scheduler.yaml + # NOTE: being containerized here: https://review.openstack.org/#/c/471527/ OS::TripleO::Services::ManilaShare: ../../puppet/services/manila-share.yaml OS::TripleO::Services::ManilaBackendCephFs: ../../puppet/services/manila-backend-cephfs.yaml + # TODO: containerize NeutronBgpVpnApi OS::TripleO::Services::NeutronBgpVpnApi: ../../puppet/services/neutron-bgpvpn-api.yaml # NOTE: This is needed because of upgrades from Ocata to Pike. We # deploy the initial environment with Ocata templates, and diff --git a/deployed-server/deployed-server.yaml b/deployed-server/deployed-server.yaml index 0847bfbf..16deb7d6 100644 --- a/deployed-server/deployed-server.yaml +++ b/deployed-server/deployed-server.yaml @@ -44,6 +44,9 @@ parameters: Command or script snippet to run on all overcloud nodes to initialize the upgrade process. E.g. a repository switch. default: '' + deployment_swift_data: + type: json + default: {} resources: deployed-server: @@ -51,6 +54,7 @@ resources: properties: name: {get_param: name} software_config_transport: {get_param: software_config_transport} + deployment_swift_data: {get_param: deployment_swift_data} UpgradeInitConfig: type: OS::Heat::SoftwareConfig @@ -133,3 +137,5 @@ outputs: - {get_attr: [ControlPlanePort, fixed_ips, 0, ip_address]} name: value: {get_attr: [HostsEntryDeployment, hostname]} + os_collect_config: + value: {get_attr: [deployed-server, os_collect_config]} diff --git a/docker/docker-steps.j2 b/docker/docker-steps.j2 index 3dd963b9..83772028 100644 --- a/docker/docker-steps.j2 +++ b/docker/docker-steps.j2 @@ -21,6 +21,9 @@ parameters: servers: type: json description: Mapping of Role name e.g Controller to a list of servers + stack_name: + type: string + description: Name of the topmost stack role_data: type: json description: Mapping of Role name e.g Controller to the per-role data @@ -35,6 +38,21 @@ parameters: description: Mapping of service endpoint -> protocol. Typically set via parameter_defaults in the resource registry. type: json + ctlplane_service_ips: + type: json + +conditions: +{% for step in range(1, deploy_steps_max) %} + WorkflowTasks_Step{{step}}_Enabled: + or: + {% for role in roles %} + - not: + equals: + - get_param: [role_data, {{role.name}}, service_workflow_tasks, step{{step}}] + - '' + - False + {% endfor %} +{% endfor %} resources: @@ -68,6 +86,53 @@ resources: - name: bootstrap_server_id config: {get_file: deploy-steps-playbook.yaml} +{%- for step in range(1, deploy_steps_max) %} +# BEGIN service_workflow_tasks handling + WorkflowTasks_Step{{step}}: + type: OS::Mistral::Workflow + condition: WorkflowTasks_Step{{step}}_Enabled + depends_on: + {% if step == 1 %} + {% for dep in roles %} + - {{dep.name}}PreConfig + - {{dep.name}}ArtifactsDeploy + {% endfor %} + {% else %} + {% for dep in roles %} + - {{dep.name}}Deployment_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + name: {list_join: [".", ["tripleo", {get_param: stack_name}, "workflowtasks", "step{{step}}"]]} + type: direct + tasks: + yaql: + expression: $.data.where($ != '').select($.get('step{{step}}')).where($ != null).flatten() + data: + {% for role in roles %} + - get_param: [role_data, {{role.name}}, service_workflow_tasks] + {% endfor %} + + WorkflowTasks_Step{{step}}_Execution: + type: OS::Mistral::ExternalResource + condition: WorkflowTasks_Step{{step}}_Enabled + depends_on: WorkflowTasks_Step{{step}} + properties: + actions: + CREATE: + workflow: { get_resource: WorkflowTasks_Step{{step}} } + params: + env: + service_ips: { get_param: ctlplane_service_ips } + UPDATE: + workflow: { get_resource: WorkflowTasks_Step{{step}} } + params: + env: + service_ips: { get_param: ctlplane_service_ips } + always_update: true +# END service_workflow_tasks handling +{% endfor %} + {% for role in roles %} # Post deployment steps for all roles # A single config is re-applied with an incrementing step number @@ -195,14 +260,23 @@ resources: {{role.name}}Deployment_Step{{step}}: type: OS::Heat::StructuredDeploymentGroup - {% if step == 1 %} - depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy] - {% else %} depends_on: - {% for dep in roles %} + - WorkflowTasks_Step{{step}}_Execution + # TODO(gfidente): the following if/else condition + # replicates what is already defined for the + # WorkflowTasks_StepX resource and can be remove + # if https://bugs.launchpad.net/heat/+bug/1700569 + # is fixed. + {% if step == 1 %} + {% for dep in roles %} + - {{dep.name}}PreConfig + - {{dep.name}}ArtifactsDeploy + {% endfor %} + {% else %} + {% for dep in roles %} - {{dep.name}}Deployment_Step{{step -1}} - {% endfor %} - {% endif %} + {% endfor %} + {% endif %} properties: name: {{role.name}}Deployment_Step{{step}} servers: {get_param: [servers, {{role.name}}]} diff --git a/docker/services/collectd.yaml b/docker/services/collectd.yaml index 7354898b..6c58a589 100644 --- a/docker/services/collectd.yaml +++ b/docker/services/collectd.yaml @@ -55,7 +55,11 @@ outputs: description: Role data for the collectd role. value: service_name: {get_attr: [CollectdBase, role_data, service_name]} - config_settings: {get_attr: [CollectdBase, role_data, config_settings]} + config_settings: + map_merge: + - get_attr: [CollectdBase, role_data, config_settings] + - tripleo::profile::base::metrics::collectd::enable_file_logging: true + collectd::plugin::logfile::log_file: /var/log/collectd/collectd.log step_config: &step_config get_attr: [CollectdBase, role_data, step_config] service_config_settings: {get_attr: [CollectdBase, role_data, service_config_settings]} @@ -71,6 +75,10 @@ outputs: kolla_config: /var/lib/kolla/config_files/collectd.json: command: /usr/sbin/collectd -f + permissions: + - path: /var/log/collectd + owner: collectd:collectd + recurse: true docker_config: step_3: collectd: @@ -84,11 +92,17 @@ outputs: - - /var/run/docker.sock:/var/run/docker.sock:rw - /var/lib/kolla/config_files/collectd.json:/var/lib/kolla/config_files/config.json:ro - - /var/lib/config-data/collectd/etc/collectd/:/etc/collectd/:ro + - /var/lib/config-data/collectd/etc/collectd.conf:/etc/collectd.conf:ro + - /var/lib/config-data/collectd/etc/collectd.d:/etc/collectd.d:ro + - /var/log/containers/collectd:/var/log/collectd:rw environment: - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS + host_prep_tasks: + - name: create persistent logs directory + file: + path: /var/log/containers/collectd + state: directory upgrade_tasks: - name: Stop and disable collectd service tags: step2 service: name=collectd.service state=stopped enabled=no - diff --git a/docker/services/database/mysql-client.yaml b/docker/services/database/mysql-client.yaml new file mode 100644 index 00000000..b0ad3760 --- /dev/null +++ b/docker/services/database/mysql-client.yaml @@ -0,0 +1,66 @@ +heat_template_version: pike + +description: > + Configuration for containerized MySQL clients + +parameters: + DockerNamespace: + description: namespace + default: 'tripleoupstream' + type: string + DockerMysqlImage: + description: image + default: 'centos-binary-mariadb:latest' + type: string + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + DefaultPasswords: + default: {} + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + default: {} + description: Parameters specific to the role + type: json + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + EnableInternalTLS: + type: boolean + default: false + InternalTLSCAFile: + default: '/etc/ipa/ca.crt' + type: string + description: Specifies the default CA cert to use if TLS is used for + services in the internal network. + +outputs: + role_data: + description: Role for setting mysql client parameters + value: + service_name: mysql_client + config_settings: + tripleo::profile::base::database::mysql::client::mysql_client_bind_address: {get_param: [ServiceNetMap, MysqlNetwork]} + tripleo::profile::base::database::mysql::client::enable_ssl: {get_param: EnableInternalTLS} + tripleo::profile::base::database::mysql::client::ssl_ca: {get_param: InternalTLSCAFile} + # BEGIN DOCKER SETTINGS # + step_config: "" + puppet_config: + config_volume: mysql_client + puppet_tags: file # set this even though file is the default + step_config: "include ::tripleo::profile::base::database::mysql::client" + config_image: + list_join: + - '/' + - [ {get_param: DockerNamespace}, {get_param: DockerMysqlImage} ] + # no need for a docker config, this service only generates configuration files + docker_config: {} diff --git a/docker/services/manila-api.yaml b/docker/services/manila-api.yaml index a203d436..62fdaaf0 100644 --- a/docker/services/manila-api.yaml +++ b/docker/services/manila-api.yaml @@ -94,6 +94,7 @@ outputs: net: host detach: false volumes: + list_concat: - {get_attr: [ContainersCommon, volumes]} - - /var/lib/config-data/manila/etc/manila/:/etc/manila/:ro diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml new file mode 100644 index 00000000..7cac9d48 --- /dev/null +++ b/docker/services/pacemaker/cinder-backup.yaml @@ -0,0 +1,152 @@ +heat_template_version: pike + +description: > + OpenStack containerized Cinder Backup service + +parameters: + DockerNamespace: + description: namespace + default: 'tripleoupstream' + type: string + DockerCinderBackupImage: + description: image + default: 'centos-binary-cinder-backup:latest' + type: string + # we configure all cinder services in the same cinder base container + DockerCinderConfigImage: + description: image + default: 'centos-binary-cinder-api:latest' + type: string + CinderBackupBackend: + default: swift + description: The short name of the Cinder Backup backend to use. + type: string + constraints: + - allowed_values: ['swift', 'ceph'] + CinderBackupRbdPoolName: + default: backups + type: string + CephClientUserName: + default: openstack + type: string + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + DefaultPasswords: + default: {} + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + default: {} + description: Parameters specific to the role + type: json + + +resources: + + CinderBackupBase: + type: ../../../puppet/services/cinder-backup.yaml + properties: + EndpointMap: {get_param: EndpointMap} + ServiceNetMap: {get_param: ServiceNetMap} + DefaultPasswords: {get_param: DefaultPasswords} + RoleName: {get_param: RoleName} + RoleParameters: {get_param: RoleParameters} + CinderBackupBackend: {get_param: CinderBackupBackend} + CinderBackupRbdPoolName: {get_param: CinderBackupRbdPoolName} + CephClientUserName: {get_param: CephClientUserName} + +outputs: + role_data: + description: Role data for the Cinder Backup role. + value: + service_name: {get_attr: [CinderBackupBase, role_data, service_name]} + config_settings: + map_merge: + - get_attr: [CinderBackupBase, role_data, config_settings] + - tripleo::profile::pacemaker::cinder::backup_bundle::cinder_backup_docker_image: &cinder_backup_image + list_join: + - '/' + - [ {get_param: DockerNamespace}, {get_param: DockerCinderBackupImage} ] + cinder::backup::manage_service: false + cinder::backup::enabled: false + step_config: "" + service_config_settings: {get_attr: [CinderBackupBase, role_data, service_config_settings]} + # BEGIN DOCKER SETTINGS + puppet_config: + config_volume: cinder + puppet_tags: cinder_config,file,concat,file_line + step_config: {get_attr: [CinderBackupBase, role_data, step_config]} + config_image: + list_join: + - '/' + - [ {get_param: DockerNamespace}, {get_param: DockerCinderConfigImage} ] + kolla_config: + /var/lib/kolla/config_files/cinder_backup.json: + command: /usr/bin/cinder-backup --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf + permissions: + - path: /var/lib/cinder + owner: cinder:cinder + recurse: true + - path: /var/log/cinder + owner: cinder:cinder + recurse: true + docker_config: + step_3: + cinder_backup_init_logs: + start_order: 0 + image: *cinder_backup_image + privileged: false + user: root + volumes: + - /var/log/containers/cinder:/var/log/cinder + command: ['/bin/bash', '-c', 'chown -R cinder:cinder /var/log/cinder'] + step_5: + cinder_backup_init_bundle: + start_order: 1 + detach: false + net: host + user: root + command: + - '/bin/bash' + - '-c' + - str_replace: + template: + list_join: + - '; ' + - - "cp -a /tmp/puppet-etc/* /etc/puppet; echo '{\"step\": 5}' > /etc/puppet/hieradata/docker.json" + - "FACTER_uuid=docker puppet apply --tags file_line,concat,augeas,TAGS --debug -v -e 'CONFIG'" + params: + TAGS: 'pacemaker::resource::bundle,pacemaker::property,pacemaker::constraint::location' + CONFIG: 'include ::tripleo::profile::base::pacemaker;include ::tripleo::profile::pacemaker::cinder::backup_bundle' + image: *cinder_backup_image + volumes: + - /etc/hosts:/etc/hosts:ro + - /etc/localtime:/etc/localtime:ro + - /etc/puppet:/tmp/puppet-etc:ro + - /usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro + - /etc/corosync/corosync.conf:/etc/corosync/corosync.conf:ro + - /dev/shm:/dev/shm:rw + host_prep_tasks: + - name: create persistent directories + file: + path: "{{ item }}" + state: directory + with_items: + - /var/lib/cinder + - /var/log/containers/cinder + upgrade_tasks: + - name: Stop and disable cinder_backup service + tags: step2 + service: name=openstack-cinder-backup state=stopped enabled=no diff --git a/docker/services/sensu-client.yaml b/docker/services/sensu-client.yaml index e6bdf155..db6daf99 100644 --- a/docker/services/sensu-client.yaml +++ b/docker/services/sensu-client.yaml @@ -104,7 +104,11 @@ outputs: - [ {get_param: DockerNamespace}, {get_param: DockerSensuClientImage} ] kolla_config: /var/lib/kolla/config_files/sensu-client.json: - command: /usr/bin/sensu-client -d /etc/sensu/conf.d/ + command: /usr/bin/sensu-client -d /etc/sensu/conf.d/ -l /var/log/sensu/sensu-client.log + permissions: + - path: /var/log/sensu + owner: sensu:sensu + recurse: true docker_config: step_3: sensu_client: @@ -123,8 +127,14 @@ outputs: - /var/run/docker.sock:/var/run/docker.sock:rw - /var/lib/kolla/config_files/sensu-client.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/sensu/etc/sensu/:/etc/sensu/:ro + - /var/log/containers/sensu:/var/log/sensu:rw environment: - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS + host_prep_tasks: + - name: create persistent logs directory + file: + path: /var/log/containers/sensu + state: directory upgrade_tasks: - name: Stop and disable sensu-client service tags: step2 diff --git a/environments/docker.yaml b/environments/docker.yaml index 6a5ec87a..03713e83 100644 --- a/environments/docker.yaml +++ b/environments/docker.yaml @@ -30,6 +30,7 @@ resource_registry: OS::TripleO::Services::NeutronDhcpAgent: ../docker/services/neutron-dhcp.yaml OS::TripleO::Services::NeutronL3Agent: ../docker/services/neutron-l3.yaml OS::TripleO::Services::MySQL: ../docker/services/database/mysql.yaml + OS::TripleO::Services::MySQLClient: ../docker/services/database/mysql-client.yaml OS::TripleO::Services::RabbitMQ: ../docker/services/rabbitmq.yaml OS::TripleO::Services::MongoDb: ../docker/services/database/mongodb.yaml OS::TripleO::Services::Redis: ../docker/services/database/redis.yaml @@ -52,9 +53,9 @@ resource_registry: OS::TripleO::Services::Horizon: ../docker/services/horizon.yaml OS::TripleO::Services::Iscsid: ../docker/services/iscsid.yaml OS::TripleO::Services::Multipathd: ../docker/services/multipathd.yaml - OS::TripleO::Services::CinderApi: ../docker/services/cinder-api.yaml - OS::TripleO::Services::CinderScheduler: ../docker/services/cinder-scheduler.yaml # FIXME: Had to remove these to unblock containers CI. They should be put back when fixed. + # OS::TripleO::Services::CinderApi: ../docker/services/cinder-api.yaml + # OS::TripleO::Services::CinderScheduler: ../docker/services/cinder-scheduler.yaml # OS::TripleO::Services::CinderBackup: ../docker/services/cinder-backup.yaml # OS::TripleO::Services::CinderVolume: ../docker/services/cinder-volume.yaml diff --git a/environments/overcloud-baremetal.j2.yaml b/environments/overcloud-baremetal.j2.yaml new file mode 100644 index 00000000..668e28de --- /dev/null +++ b/environments/overcloud-baremetal.j2.yaml @@ -0,0 +1,19 @@ +resource_registry: + OS::TripleO::AllNodes::SoftwareConfig: OS::Heat::None + OS::TripleO::PostDeploySteps: OS::Heat::None + OS::TripleO::DefaultPasswords: OS::Heat::None + OS::TripleO::RandomString: OS::Heat::None + OS::TripleO::AllNodesDeployment: OS::Heat::None + +parameter_defaults: + # Deploy no services +{% for role in roles %} + {{role.name}}Services: [] +{% endfor %} + + # Consistent Hostname format + ControllerHostnameFormat: overcloud-controller-%index% + ComputeHostnameFormat: overcloud-novacompute-%index% + ObjectStorageHostnameFormat: overcloud-objectstorage-%index% + CephStorageHostnameFormat: overcloud-cephstorage-%index% + BlockStorageHostnameFormat: overcloud-blockstorage-%index% diff --git a/environments/overcloud-services.yaml b/environments/overcloud-services.yaml new file mode 100644 index 00000000..c409b899 --- /dev/null +++ b/environments/overcloud-services.yaml @@ -0,0 +1,7 @@ +parameter_defaults: + # Consistent Hostname format + ControllerDeployedServerHostnameFormat: overcloud-controller-%index% + ComputeDeployedServerHostnameFormat: overcloud-novacompute-%index% + ObjectStorageDeployedServerHostnameFormat: overcloud-objectstorage-%index% + CephStorageDeployedServerHostnameFormat: overcloud-cephstorage-%index% + BlockStorageDeployedServerHostnameFormat: overcloud-blockstorage-%index% diff --git a/environments/services/ironic.yaml b/environments/services/ironic.yaml index 8359f4a7..b81b0269 100644 --- a/environments/services/ironic.yaml +++ b/environments/services/ironic.yaml @@ -2,3 +2,5 @@ resource_registry: OS::TripleO::Services::IronicApi: ../../puppet/services/ironic-api.yaml OS::TripleO::Services::IronicConductor: ../../puppet/services/ironic-conductor.yaml OS::TripleO::Services::NovaIronic: ../../puppet/services/nova-ironic.yaml +parameter_defaults: + NovaSchedulerDiscoverHostsInCellsInterval: 15 diff --git a/net-config-bond.yaml b/net-config-bond.yaml index f92f9a13..95b47455 100644 --- a/net-config-bond.yaml +++ b/net-config-bond.yaml @@ -4,19 +4,14 @@ description: > parameters: BondInterfaceOvsOptions: default: '' - description: 'The ovs_options string for the bond interface. Set things like - - lacp=active and/or bond_mode=balance-slb using this option. - - ' + description: The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option. type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ - description: 'The balance-tcp bond mode is known to cause packet loss and - + description: The balance-tcp bond mode is known to cause packet loss and should not be used in BondInterfaceOvsOptions. - - ' ControlPlaneIp: default: '' description: IP address/subnet on the ctlplane network diff --git a/network/config/bond-with-vlans/ceph-storage.yaml b/network/config/bond-with-vlans/ceph-storage.yaml index 97177c41..9683456a 100644 --- a/network/config/bond-with-vlans/ceph-storage.yaml +++ b/network/config/bond-with-vlans/ceph-storage.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/cinder-storage.yaml b/network/config/bond-with-vlans/cinder-storage.yaml index 5456c2cc..3ad6d653 100644 --- a/network/config/bond-with-vlans/cinder-storage.yaml +++ b/network/config/bond-with-vlans/cinder-storage.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/compute-dpdk.yaml b/network/config/bond-with-vlans/compute-dpdk.yaml index 607d346f..095c4973 100644 --- a/network/config/bond-with-vlans/compute-dpdk.yaml +++ b/network/config/bond-with-vlans/compute-dpdk.yaml @@ -32,8 +32,9 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string ExternalNetworkVlanID: default: 10 diff --git a/network/config/bond-with-vlans/compute.yaml b/network/config/bond-with-vlans/compute.yaml index 448d4e2a..8fff1378 100644 --- a/network/config/bond-with-vlans/compute.yaml +++ b/network/config/bond-with-vlans/compute.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/controller-no-external.yaml b/network/config/bond-with-vlans/controller-no-external.yaml index 8ac5cda7..4901f94d 100644 --- a/network/config/bond-with-vlans/controller-no-external.yaml +++ b/network/config/bond-with-vlans/controller-no-external.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/controller-v6.yaml b/network/config/bond-with-vlans/controller-v6.yaml index 25796484..33c6fa65 100644 --- a/network/config/bond-with-vlans/controller-v6.yaml +++ b/network/config/bond-with-vlans/controller-v6.yaml @@ -34,16 +34,14 @@ parameters: type: string BondInterfaceOvsOptions: default: bond_mode=active-backup - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/controller.yaml b/network/config/bond-with-vlans/controller.yaml index e4b30120..100821b7 100644 --- a/network/config/bond-with-vlans/controller.yaml +++ b/network/config/bond-with-vlans/controller.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: bond_mode=active-backup - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: 'The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option.' type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ description: 'The balance-tcp bond mode is known to cause packet loss and - - should not be used in BondInterfaceOvsOptions. - - ' + should not be used in BondInterfaceOvsOptions.' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/config/bond-with-vlans/swift-storage.yaml b/network/config/bond-with-vlans/swift-storage.yaml index 6371ceb5..0ede081f 100644 --- a/network/config/bond-with-vlans/swift-storage.yaml +++ b/network/config/bond-with-vlans/swift-storage.yaml @@ -32,16 +32,14 @@ parameters: type: string BondInterfaceOvsOptions: default: '' - description: The ovs_options string for the bond interface. Set things like lacp=active and/or bond_mode=balance-slb using - this option. + description: The ovs_options or bonding_options string for the bond + interface. Set things like lacp=active and/or bond_mode=balance-slb + for OVS bonds or like mode=4 for Linux bonds using this option. type: string constraints: - allowed_pattern: ^((?!balance.tcp).)*$ - description: 'The balance-tcp bond mode is known to cause packet loss and - + description: The balance-tcp bond mode is known to cause packet loss and should not be used in BondInterfaceOvsOptions. - - ' ExternalNetworkVlanID: default: 10 description: Vlan ID for the external network traffic. diff --git a/network/ports/ctlplane_vip.yaml b/network/ports/ctlplane_vip.yaml index 386520cf..bb54ca62 100644 --- a/network/ports/ctlplane_vip.yaml +++ b/network/ports/ctlplane_vip.yaml @@ -34,7 +34,7 @@ parameters: resources: VipPort: - type: OS::Neutron::Port + type: OS::TripleO::Network::Ports::ControlPlaneVipPort properties: network: {get_param: ControlPlaneNetwork} name: {get_param: PortName} diff --git a/network/ports/net_ip_list_map.yaml b/network/ports/net_ip_list_map.yaml index c3734afe..a9111ed9 100644 --- a/network/ports/net_ip_list_map.yaml +++ b/network/ports/net_ip_list_map.yaml @@ -133,6 +133,20 @@ outputs: SERVICE: {get_attr: [EnabledServicesValue, value]} - values: {get_param: ServiceNetMap} - values: {get_attr: [NetIpMapValue, value]} + ctlplane_service_ips: + description: > + Map of enabled services to a list of their ctlplane IP addresses + value: + yaql: + expression: dict($.data.map.items().where(len($[1]) > 0)) + data: + map: + map_merge: + repeat: + template: + SERVICE_ctlplane_node_ips: {get_param: ControlPlaneIpList} + for_each: + SERVICE: {get_attr: [EnabledServicesValue, value]} service_hostnames: description: > Map of enabled services to a list of hostnames where they're running diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index 4aee571e..b1a35293 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -4,10 +4,12 @@ resource_registry: OS::TripleO::PostDeploySteps: puppet/post.yaml OS::TripleO::PostUpgradeSteps: puppet/post-upgrade.yaml OS::TripleO::AllNodes::SoftwareConfig: puppet/all-nodes-config.yaml + OS::TripleO::AllNodesDeployment: OS::Heat::StructuredDeployments OS::TripleO::Hosts::SoftwareConfig: hosts-config.yaml OS::TripleO::Ssh::HostPubKey: extraconfig/tasks/ssh/host_public_key.yaml OS::TripleO::Ssh::KnownHostsConfig: extraconfig/tasks/ssh/known_hosts_config.yaml OS::TripleO::DefaultPasswords: default_passwords.yaml + OS::TripleO::RandomString: OS::Heat::RandomString # Tasks (for internal TripleO usage) OS::TripleO::Tasks::UpdateWorkflow: OS::Heat::None diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml index e4c04b4e..1848e09a 100644 --- a/overcloud.j2.yaml +++ b/overcloud.j2.yaml @@ -242,15 +242,15 @@ resources: HOST: {get_param: CloudNameStorageManagement} HeatAuthEncryptionKey: - type: OS::Heat::RandomString + type: OS::TripleO::RandomString PcsdPassword: - type: OS::Heat::RandomString + type: OS::TripleO::RandomString properties: length: 16 HorizonSecret: - type: OS::Heat::RandomString + type: OS::TripleO::RandomString properties: length: 10 @@ -334,7 +334,7 @@ resources: servers: {get_attr: [{{role.name}}Servers, value]} {{role.name}}AllNodesDeployment: - type: OS::Heat::StructuredDeployments + type: OS::TripleO::AllNodesDeployment depends_on: {% for role_inner in roles %} - {{role_inner.name}}HostsDeployment @@ -575,12 +575,12 @@ resources: UpdateIdentifier: {get_param: UpdateIdentifier} MysqlRootPassword: - type: OS::Heat::RandomString + type: OS::TripleO::RandomString properties: length: 10 RabbitCookie: - type: OS::Heat::RandomString + type: OS::TripleO::RandomString properties: length: 20 salt: {get_param: RabbitCookieSalt} @@ -735,12 +735,34 @@ resources: {% for role in roles %} {{role.name}}: {get_attr: [{{role.name}}Servers, value]} {% endfor %} + stack_name: {get_param: 'OS::stack_name'} EndpointMap: {get_attr: [EndpointMap, endpoint_map]} + ctlplane_service_ips: + # Note (shardy) this somewhat complex yaql may be replaced + # with a map_deep_merge function in ocata. It merges the + # list of maps, but appends to colliding lists when a service + # is deployed on more than one role + yaql: + expression: dict($.data.l.where($ != null).selectMany($.items()).groupBy($[0], $[1], [$[0], $[1].flatten()])) + data: + l: +{% for role in roles %} + - {get_attr: [{{role.name}}IpListMap, ctlplane_service_ips]} +{% endfor %} role_data: {% for role in roles %} {{role.name}}: {get_attr: [{{role.name}}ServiceChainRoleData, value]} {% endfor %} + ServerOsCollectConfigData: + type: OS::Heat::Value + properties: + type: json + value: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}, attributes, os_collect_config]} +{% endfor %} + outputs: ManagedEndpoints: description: Asserts that the keystone endpoints have been provisioned. @@ -791,3 +813,15 @@ outputs: {% for role in roles %} {{role.name}}: {get_attr: [{{role.name}}NetworkHostnameMap, value]} {% endfor %} + ServerOsCollectConfigData: + description: The os-collect-config configuration associated with each server resource + value: +{% for role in roles %} + {{role.name}}: {get_attr: [{{role.name}}, attributes, os_collect_config]} +{% endfor %} + VipMap: + description: Mapping of each network to VIP addresses. Also includes the Redis VIP. + value: + map_merge: + - {get_attr: [VipMap, net_ip_map]} + - redis: {get_attr: [RedisVirtualIP, ip_address]} diff --git a/puppet/blockstorage-role.yaml b/puppet/blockstorage-role.yaml index b20cdbac..612a4a01 100644 --- a/puppet/blockstorage-role.yaml +++ b/puppet/blockstorage-role.yaml @@ -142,6 +142,24 @@ parameters: RoleParameters: type: json description: Role Specific Parameters + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 default: {} conditions: @@ -150,6 +168,12 @@ conditions: equals: - {get_param: [DeploymentServerBlacklistDict, {get_param: Hostname}]} - 1 + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" resources: BlockStorage: @@ -178,6 +202,12 @@ resources: - {get_param: BlockStorageServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: BlockStorageSchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -615,3 +645,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" management_ip_address: description: IP address of the server in the management network value: {get_attr: [ManagementPort, ip_address]} + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [BlockStorage, os_collect_config]} diff --git a/puppet/cephstorage-role.yaml b/puppet/cephstorage-role.yaml index f1320bc1..e7afcb40 100644 --- a/puppet/cephstorage-role.yaml +++ b/puppet/cephstorage-role.yaml @@ -148,6 +148,24 @@ parameters: RoleParameters: type: json description: Role Specific Parameters + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 default: {} conditions: @@ -156,6 +174,12 @@ conditions: equals: - {get_param: [DeploymentServerBlacklistDict, {get_param: Hostname}]} - 1 + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" resources: CephStorage: @@ -184,6 +208,12 @@ resources: - {get_param: CephStorageServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: CephStorageSchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -626,3 +656,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" management_ip_address: description: IP address of the server in the management network value: {get_attr: [ManagementPort, ip_address]} + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [CephStorage, os_collect_config]} diff --git a/puppet/compute-role.yaml b/puppet/compute-role.yaml index 6b0beab2..5a662e86 100644 --- a/puppet/compute-role.yaml +++ b/puppet/compute-role.yaml @@ -37,7 +37,7 @@ parameters: type: string NeutronPublicInterface: default: nic1 - description: A port to add to the NeutronPhysicalBridge. + description: Which interface to add to the NeutronPhysicalBridge. type: string NodeIndex: type: number @@ -160,9 +160,33 @@ parameters: RoleParameters: type: json description: Role Specific Parameters + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 default: {} conditions: + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" server_not_blacklisted: not: equals: @@ -198,6 +222,12 @@ resources: - {get_param: NovaComputeServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: NovaComputeSchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -652,3 +682,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" value: {get_resource: NovaCompute} condition: server_not_blacklisted + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [NovaCompute, os_collect_config]} diff --git a/puppet/controller-role.yaml b/puppet/controller-role.yaml index b5c38287..09e5b2b9 100644 --- a/puppet/controller-role.yaml +++ b/puppet/controller-role.yaml @@ -58,9 +58,13 @@ parameters: type: string constraints: - custom_constraint: nova.keypair + NeutronPhysicalBridge: + default: 'br-ex' + description: An OVS bridge to create for accessing external networks. + type: string NeutronPublicInterface: default: nic1 - description: What interface to bridge onto br-ex for network nodes. + description: Which interface to add to the NeutronPhysicalBridge. type: string ServiceNetMap: default: {} @@ -174,6 +178,24 @@ parameters: RoleParameters: type: json description: Role Specific Parameters + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 default: {} parameter_groups: @@ -188,7 +210,12 @@ conditions: equals: - {get_param: [DeploymentServerBlacklistDict, {get_param: Hostname}]} - 1 - + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" resources: @@ -218,6 +245,12 @@ resources: - {get_param: ControllerServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: ControllerSchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -432,7 +465,7 @@ resources: - {get_param: NetworkDeploymentActions} - [] input_values: - bridge_name: br-ex + bridge_name: {get_param: NeutronPhysicalBridge} interface_name: {get_param: NeutronPublicInterface} # Resource for site-specific injection of root certificate @@ -698,3 +731,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" tls_cert_modulus_md5: description: MD5 checksum of the TLS Certificate Modulus value: {get_attr: [NodeTLSData, cert_modulus_md5]} + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [Controller, os_collect_config]} diff --git a/puppet/deploy-artifacts.sh b/puppet/deploy-artifacts.sh index 6e7f0ec1..e4d20b49 100644 --- a/puppet/deploy-artifacts.sh +++ b/puppet/deploy-artifacts.sh @@ -10,7 +10,9 @@ if [ -n "$artifact_urls" ]; then for URL in $(echo $artifact_urls | sed -e "s| |\n|g" | sort -u); do curl --globoff -o $TMP_DATA/file_data "$URL" if file -b $TMP_DATA/file_data | grep RPM &>/dev/null; then - yum install -y $TMP_DATA/file_data + mv $TMP_DATA/file_data $TMP_DATA/file_data.rpm + yum install -y $TMP_DATA/file_data.rpm + rm $TMP_DATA/file_data.rpm elif file -b $TMP_DATA/file_data | grep 'gzip compressed data' &>/dev/null; then pushd / tar xvzf $TMP_DATA/file_data @@ -19,7 +21,9 @@ if [ -n "$artifact_urls" ]; then echo "ERROR: Unsupported file format: $URL" exit 1 fi - rm $TMP_DATA/file_data + if [ -f $TMP_DATA/file_data ]; then + rm $TMP_DATA/file_data + fi done else echo "No artifact_urls was set. Skipping..." diff --git a/puppet/objectstorage-role.yaml b/puppet/objectstorage-role.yaml index 85567462..4a1670f8 100644 --- a/puppet/objectstorage-role.yaml +++ b/puppet/objectstorage-role.yaml @@ -143,6 +143,25 @@ parameters: type: json description: Role Specific Parameters default: {} + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 + default: {} conditions: server_not_blacklisted: @@ -150,6 +169,12 @@ conditions: equals: - {get_param: [DeploymentServerBlacklistDict, {get_param: Hostname}]} - 1 + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" resources: @@ -178,6 +203,12 @@ resources: - {get_param: SwiftStorageServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: ObjectStorageSchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -614,3 +645,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" management_ip_address: description: IP address of the server in the management network value: {get_attr: [ManagementPort, ip_address]} + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [SwiftStorage, os_collect_config]} diff --git a/puppet/post.j2.yaml b/puppet/post.j2.yaml index 3a15cec6..67e1ecfd 100644 --- a/puppet/post.j2.yaml +++ b/puppet/post.j2.yaml @@ -8,7 +8,9 @@ parameters: servers: type: json description: Mapping of Role name e.g Controller to a list of servers - + stack_name: + type: string + description: Name of the topmost stack role_data: type: json description: Mapping of Role name e.g Controller to the per-role data @@ -23,6 +25,7 @@ parameters: description: > Setting this to a unique value will re-run any deployment tasks which perform configuration on a Heat stack-update. + ctlplane_service_ips: + type: json -resources: {% include 'puppet-steps.j2' %} diff --git a/puppet/puppet-steps.j2 b/puppet/puppet-steps.j2 index 5567d65d..82c6171e 100644 --- a/puppet/puppet-steps.j2 +++ b/puppet/puppet-steps.j2 @@ -1,3 +1,19 @@ +{% set deploy_steps_max = 6 %} + +conditions: +{% for step in range(1, deploy_steps_max) %} + WorkflowTasks_Step{{step}}_Enabled: + or: + {% for role in roles %} + - not: + equals: + - get_param: [role_data, {{role.name}}, service_workflow_tasks, step{{step}}] + - '' + - False + {% endfor %} +{% endfor %} + +resources: # Post deployment steps for all roles # A single config is re-applied with an incrementing step number {% for role in roles %} @@ -24,17 +40,26 @@ StepConfig: {list_join: ["\n", {get_param: [role_data, {{role.name}}, step_config]}]} # Step through a series of configuration steps -{% for step in range(1, 6) %} +{% for step in range(1, deploy_steps_max) %} {{role.name}}Deployment_Step{{step}}: type: OS::Heat::StructuredDeploymentGroup - {% if step == 1 %} - depends_on: [{{role.name}}PreConfig, {{role.name}}ArtifactsDeploy] - {% else %} depends_on: + - WorkflowTasks_Step{{step}}_Execution + # TODO(gfidente): the following if/else condition + # replicates what is already defined for the + # WorkflowTasks_StepX resource and can be remove + # if https://bugs.launchpad.net/heat/+bug/1700569 + # is fixed. + {% if step == 1 %} + {% for dep in roles %} + - {{dep.name}}PreConfig + - {{dep.name}}ArtifactsDeploy + {% endfor %} + {% else %} {% for dep in roles %} - {{dep.name}}Deployment_Step{{step -1}} {% endfor %} - {% endif %} + {% endif %} properties: name: {{role.name}}Deployment_Step{{step}} servers: {get_param: [servers, {{role.name}}]} @@ -72,3 +97,50 @@ {% endfor %} + +# BEGIN service_workflow_tasks handling +{% for step in range(1, deploy_steps_max) %} + WorkflowTasks_Step{{step}}: + type: OS::Mistral::Workflow + condition: WorkflowTasks_Step{{step}}_Enabled + depends_on: + {% if step == 1 %} + {% for dep in roles %} + - {{dep.name}}PreConfig + - {{dep.name}}ArtifactsDeploy + {% endfor %} + {% else %} + {% for dep in roles %} + - {{dep.name}}Deployment_Step{{step -1}} + {% endfor %} + {% endif %} + properties: + name: {list_join: [".", ["tripleo", {get_param: stack_name}, "workflowtasks", "step{{step}}"]]} + type: direct + tasks: + yaql: + expression: $.data.where($ != '').select($.get('step{{step}}')).where($ != null).flatten() + data: + {% for role in roles %} + - get_param: [role_data, {{role.name}}, service_workflow_tasks] + {% endfor %} + + WorkflowTasks_Step{{step}}_Execution: + type: OS::Mistral::ExternalResource + condition: WorkflowTasks_Step{{step}}_Enabled + depends_on: WorkflowTasks_Step{{step}} + properties: + actions: + CREATE: + workflow: { get_resource: WorkflowTasks_Step{{step}} } + params: + env: + service_ips: { get_param: ctlplane_service_ips } + UPDATE: + workflow: { get_resource: WorkflowTasks_Step{{step}} } + params: + env: + service_ips: { get_param: ctlplane_service_ips } + always_update: true +{% endfor %} +# END service_workflow_tasks handling diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml index 0a47d50b..b45736c1 100644 --- a/puppet/role.role.j2.yaml +++ b/puppet/role.role.j2.yaml @@ -28,9 +28,13 @@ parameters: constraints: - custom_constraint: nova.keypair {% endif %} + NeutronPhysicalBridge: + default: 'br-ex' + description: An OVS bridge to create for accessing tenant networks. + type: string NeutronPublicInterface: default: nic1 - description: What interface to bridge onto br-ex for network nodes. + description: Which interface to add to the NeutronPhysicalBridge. type: string ServiceNetMap: default: {} @@ -164,6 +168,24 @@ parameters: RoleParameters: type: json description: Role Specific Parameters + DeploymentSwiftDataMap: + type: json + description: | + Map of servers to Swift container and object for storing deployment data. + The keys are the Heat assigned hostnames, and the value is a map of the + container/object name in Swift. Example value: + overcloud-controller-0: + container: overcloud-controller + object: 0 + overcloud-controller-1: + container: overcloud-controller + object: 1 + overcloud-controller-2: + container: overcloud-controller + object: 2 + overcloud-novacompute-0: + container: overcloud-compute + object: 0 default: {} conditions: @@ -172,10 +194,16 @@ conditions: equals: - {get_param: [DeploymentServerBlacklistDict, {get_param: Hostname}]} - 1 + deployment_swift_data_map_unset: + equals: + - get_param: + - DeploymentSwiftDataMap + - {get_param: Hostname} + - "" resources: {{role}}: - type: OS::TripleO::{{role.name}}Server + type: OS::TripleO::{{role}}Server metadata: os-collect-config: command: {get_param: ConfigCommand} @@ -200,6 +228,12 @@ resources: - {get_param: {{role}}ServerMetadata} - {get_param: ServiceMetadataSettings} scheduler_hints: {get_param: {{role}}SchedulerHints} + deployment_swift_data: + if: + - deployment_swift_data_map_unset + - {} + - {get_param: [DeploymentSwiftDataMap, + {get_param: Hostname}]} # Combine the NodeAdminUserData and NodeUserData mime archives UserData: @@ -410,7 +444,7 @@ resources: server: {get_resource: {{role}}} actions: {get_param: NetworkDeploymentActions} input_values: - bridge_name: br-ex + bridge_name: {get_param: NeutronPhysicalBridge} interface_name: {get_param: NeutronPublicInterface} actions: if: @@ -649,3 +683,6 @@ CTLPLANEIP,CTLPLANEHOST.DOMAIN,CTLPLANEHOST HOSTSSHPUBKEY" management_ip_address: description: IP address of the server in the management network value: {get_attr: [ManagementPort, ip_address]} + os_collect_config: + description: The os-collect-config configuration associated with this server resource + value: {get_attr: [{{role}}, os_collect_config]} diff --git a/puppet/services/README.rst b/puppet/services/README.rst index 7a18ef0c..d55414b7 100644 --- a/puppet/services/README.rst +++ b/puppet/services/README.rst @@ -95,6 +95,30 @@ are re-asserted when applying latter ones. 5) Service activation (Pacemaker) +It is also possible to use Mistral actions or workflows together with +a deployment step, these are executed before the main configuration run. +To describe actions or workflows from within a service use: + + * service_workflow_tasks: One or more workflow task properties + +which expects a map where the key is the step and the value a list of +dictionaries descrbing each a workflow task, for example:: + + service_workflow_tasks: + step2: + - name: echo + action: std.echo output=Hello + step3: + - name: external + workflow: my-pre-existing-workflow-name + input: + workflow_param1: value + workflow_param2: value + +The Heat guide for the `OS::Mistral::Workflow task property +<https://docs.openstack.org/developer/heat/template_guide/openstack.html#OS::Mistral::Workflow-prop-tasks>`_ +has more details about the expected dictionary. + Batch Upgrade Steps ------------------- diff --git a/puppet/services/haproxy.yaml b/puppet/services/haproxy.yaml index 619cf131..5bdc3b88 100644 --- a/puppet/services/haproxy.yaml +++ b/puppet/services/haproxy.yaml @@ -38,6 +38,10 @@ parameters: default: /dev/log description: Syslog address where HAproxy will send its log type: string + HAProxyStatsEnabled: + default: true + description: Whether or not to enable the HAProxy stats interface. + type: boolean RedisPassword: description: The password for Redis type: string @@ -95,6 +99,7 @@ outputs: tripleo::haproxy::redis_password: {get_param: RedisPassword} tripleo::haproxy::ca_bundle: {get_param: InternalTLSCAFile} tripleo::haproxy::crl_file: {get_param: InternalTLSCRLPEMFile} + tripleo::haproxy::haproxy_stats: {get_param: HAProxyStatsEnabled} tripleo::profile::base::haproxy::certificates_specs: map_merge: - get_attr: [HAProxyPublicTLS, role_data, certificates_specs] diff --git a/puppet/services/nova-scheduler.yaml b/puppet/services/nova-scheduler.yaml index 5da6d43e..72a1fce7 100644 --- a/puppet/services/nova-scheduler.yaml +++ b/puppet/services/nova-scheduler.yaml @@ -45,6 +45,14 @@ parameters: default: tag: openstack.nova.scheduler path: /var/log/nova/nova-scheduler.log + NovaSchedulerDiscoverHostsInCellsInterval: + type: number + default: -1 + description: > + This value controls how often (in seconds) the scheduler should + attempt to discover new hosts that have been added to cells. + The default value of -1 disables the periodic task completely. + It is recommended to set this parameter for deployments using Ironic. resources: NovaBase: @@ -71,6 +79,7 @@ outputs: - nova::ram_allocation_ratio: '1.0' nova::scheduler::filter::scheduler_available_filters: {get_param: NovaSchedulerAvailableFilters} nova::scheduler::filter::scheduler_default_filters: {get_param: NovaSchedulerDefaultFilters} + nova::scheduler::discover_hosts_in_cells_interval: {get_param: NovaSchedulerDiscoverHostsInCellsInterval} step_config: | include tripleo::profile::base::nova::scheduler upgrade_tasks: diff --git a/releasenotes/notes/Make-exposing-haproxy-stats-interface-configurable-2b634793c4f13950.yaml b/releasenotes/notes/Make-exposing-haproxy-stats-interface-configurable-2b634793c4f13950.yaml new file mode 100644 index 00000000..193154d0 --- /dev/null +++ b/releasenotes/notes/Make-exposing-haproxy-stats-interface-configurable-2b634793c4f13950.yaml @@ -0,0 +1,4 @@ +--- +features: + - The HAProxy stats interface can now be enabled/disabled with the + HAProxyStatsEnabled flag. Note that it's still enabled by default. diff --git a/releasenotes/notes/add-deploymentswiftdatamap-parameter-351ee63800016e4d.yaml b/releasenotes/notes/add-deploymentswiftdatamap-parameter-351ee63800016e4d.yaml new file mode 100644 index 00000000..67a55cd8 --- /dev/null +++ b/releasenotes/notes/add-deploymentswiftdatamap-parameter-351ee63800016e4d.yaml @@ -0,0 +1,6 @@ +--- +features: + - Added new DeploymentSwiftDataMap parameter, which is used to set the + deployment_swift_data property on the Server resoures. The parameter is a + map where the keys are the Heat assigned hostnames, and the value is a map + of the container/object name in Swift. diff --git a/releasenotes/notes/add-server-os-collect-config-data-eeea2f57b3a82654.yaml b/releasenotes/notes/add-server-os-collect-config-data-eeea2f57b3a82654.yaml new file mode 100644 index 00000000..cd352ac1 --- /dev/null +++ b/releasenotes/notes/add-server-os-collect-config-data-eeea2f57b3a82654.yaml @@ -0,0 +1,6 @@ +--- +features: + - Adds a new output, ServerOsCollectConfigData, which is the + os-collect-config configuration associated with each server resource. + This can be used to [pre]configure the os-collect-config agents on + deployed-server's. diff --git a/releasenotes/notes/baremetal-cell-hosts-cd5cf5aa8a33643c.yaml b/releasenotes/notes/baremetal-cell-hosts-cd5cf5aa8a33643c.yaml new file mode 100644 index 00000000..98ba86d7 --- /dev/null +++ b/releasenotes/notes/baremetal-cell-hosts-cd5cf5aa8a33643c.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + When ``environments/services/ironic.yaml`` is used, enable periodic task + in nova-scheduler to automatically discover new nodes. Otherwise a user + has to run nova management command on controllers each time. diff --git a/releasenotes/notes/fix-rpm-deploy-artifact-urls-03d5694073ad159d.yaml b/releasenotes/notes/fix-rpm-deploy-artifact-urls-03d5694073ad159d.yaml new file mode 100644 index 00000000..25016e83 --- /dev/null +++ b/releasenotes/notes/fix-rpm-deploy-artifact-urls-03d5694073ad159d.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix support for RPMs to be installed via DeployArtifactURLs. LP#1697102 diff --git a/releasenotes/notes/service_workflow_tasks-4da5830821b7154b.yaml b/releasenotes/notes/service_workflow_tasks-4da5830821b7154b.yaml new file mode 100644 index 00000000..cf99ec5d --- /dev/null +++ b/releasenotes/notes/service_workflow_tasks-4da5830821b7154b.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + It is now possible to trigger Mistral workflows or workflow actions + before a deployment step is applied. This can be defined within the + scope of a service template and is described as a task property + for the Heat OS::Mistral::Workflow resource, for more details also + see the puppet/services/README.rst file.
\ No newline at end of file diff --git a/releasenotes/notes/split-stack-environments-1f817e24b5d90959.yaml b/releasenotes/notes/split-stack-environments-1f817e24b5d90959.yaml new file mode 100644 index 00000000..1bc99371 --- /dev/null +++ b/releasenotes/notes/split-stack-environments-1f817e24b5d90959.yaml @@ -0,0 +1,7 @@ +--- +features: + - Add 2 new example environments to facilitate deploying split-stack, + environments/overcloud-baremetal.j2.yaml and + environments/overcloud-services.yaml. The environments are used to deploy two + separate Heat stacks, one for just the baremetal+network configuration and one + for the service configuration. diff --git a/releasenotes/notes/vipmap-output-4a9ce99930960346.yaml b/releasenotes/notes/vipmap-output-4a9ce99930960346.yaml new file mode 100644 index 00000000..1f49bacd --- /dev/null +++ b/releasenotes/notes/vipmap-output-4a9ce99930960346.yaml @@ -0,0 +1,5 @@ +--- +features: + - Add VipMap output to the top level stack output. VipMap is a mapping from + each network to the VIP address on that network. Also includes the Redis + VIP. diff --git a/services.yaml b/services.yaml index 724727bb..4d3ca8d1 100644 --- a/services.yaml +++ b/services.yaml @@ -116,6 +116,10 @@ outputs: yaql: expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {}) data: {role_data: {get_attr: [ServiceChain, role_data]}} + service_workflow_tasks: + yaql: + expression: $.data.role_data.where($ != null).select($.get('service_workflow_tasks')).where($ != null).reduce($1.mergeWith($2), {}) + data: {role_data: {get_attr: [ServiceChain, role_data]}} step_config: {get_attr: [ServiceChain, role_data, step_config]} upgrade_tasks: yaql: |