diff options
| -rw-r--r-- | environments/network-management-v6.yaml | 25 | ||||
| -rw-r--r-- | environments/network-management.yaml | 3 | ||||
| -rw-r--r-- | environments/neutron-nuage-config.yaml | 2 | ||||
| -rw-r--r-- | environments/puppet-pacemaker.yaml | 2 | ||||
| -rw-r--r-- | network/management_v6.yaml | 69 | ||||
| -rw-r--r-- | network/ports/management_from_pool_v6.yaml | 52 | ||||
| -rw-r--r-- | overcloud-resource-registry-puppet.yaml | 2 | ||||
| -rw-r--r-- | overcloud.yaml | 2 | ||||
| -rw-r--r-- | puppet/ceph-cluster-config.yaml | 7 | ||||
| -rw-r--r-- | puppet/controller.yaml | 11 | ||||
| -rw-r--r-- | puppet/extraconfig/ceph/ceph-external-config.yaml | 7 | ||||
| -rw-r--r-- | puppet/hieradata/controller.yaml | 67 | ||||
| -rw-r--r-- | puppet/hieradata/database.yaml | 9 | ||||
| -rw-r--r-- | puppet/manifests/overcloud_controller.pp | 14 | ||||
| -rw-r--r-- | puppet/manifests/overcloud_controller_pacemaker.pp | 9 | ||||
| -rw-r--r-- | puppet/services/neutron-plugin-nuage.yaml | 26 | ||||
| -rw-r--r-- | puppet/services/pacemaker/neutron-plugin-nuage.yaml | 28 |
17 files changed, 289 insertions, 46 deletions
diff --git a/environments/network-management-v6.yaml b/environments/network-management-v6.yaml new file mode 100644 index 00000000..812e84f3 --- /dev/null +++ b/environments/network-management-v6.yaml @@ -0,0 +1,25 @@ +# Enable the creation of an IPv6 system management network. This +# creates a Neutron network for isolated Overcloud +# system management traffic and configures each role to +# assign a port (related to that role) on that network. +# Note that the basic sample NIC configuration templates +# do not include the management network, see the +# comments in the sample network config templates in +# network/config/ for an example. +resource_registry: + OS::TripleO::Network::Management: ../network/management_v6.yaml + + # Port assignments for the controller role + OS::TripleO::Controller::Ports::ManagementPort: ../network/ports/management_v6.yaml + + # Port assignments for the compute role + OS::TripleO::Compute::Ports::ManagementPort: ../network/ports/management_v6.yaml + + # Port assignments for the ceph storage role + OS::TripleO::CephStorage::Ports::ManagementPort: ../network/ports/management_v6.yaml + + # Port assignments for the swift storage role + OS::TripleO::SwiftStorage::Ports::ManagementPort: ../network/ports/management_v6.yaml + + # Port assignments for the block storage role + OS::TripleO::BlockStorage::Ports::ManagementPort: ../network/ports/management_v6.yaml diff --git a/environments/network-management.yaml b/environments/network-management.yaml index 2f0cff8b..041617be 100644 --- a/environments/network-management.yaml +++ b/environments/network-management.yaml @@ -4,7 +4,8 @@ # assign a port (related to that role) on that network. # Note that the basic sample NIC configuration templates # do not include the management network, see the -# single-nic-vlans-mgmt templates for an example. +# comments in the sample network config templates in +# network/config/ for an example. resource_registry: OS::TripleO::Network::Management: ../network/management.yaml diff --git a/environments/neutron-nuage-config.yaml b/environments/neutron-nuage-config.yaml index bf5036bf..59f6d34e 100644 --- a/environments/neutron-nuage-config.yaml +++ b/environments/neutron-nuage-config.yaml @@ -5,6 +5,8 @@ resource_registry: OS::TripleO::Services::NeutronL3Agent: OS::Heat::None OS::TripleO::Services::NeutronMetadataAgent: OS::Heat::None OS::TripleO::Services::NeutronOvsAgent: OS::Heat::None + # Override the NeutronCorePlugin to use Nuage + OS::TripleO::Services::NeutronCorePlugin: OS::TripleO::Services::NeutronCorePluginNuage parameter_defaults: NeutronNuageOSControllerIp: '0.0.0.0' diff --git a/environments/puppet-pacemaker.yaml b/environments/puppet-pacemaker.yaml index 52a94d80..70998223 100644 --- a/environments/puppet-pacemaker.yaml +++ b/environments/puppet-pacemaker.yaml @@ -26,7 +26,7 @@ resource_registry: OS::TripleO::Services::NeutronCorePlugin: ../puppet/services/pacemaker/neutron-plugin-ml2.yaml # Neutron Core Plugin Vendors (these typically override NeutronCorePlugin) OS::TripleO::Services::NeutronCorePluginPlumgrid: ../puppet/services/pacemaker/neutron-plugin-plumgrid.yaml - + OS::TripleO::Services::NeutronCorePluginNuage: ../puppet/services/pacemaker/neutron-plugin-nuage.yaml OS::TripleO::Services::NeutronOvsAgent: ../puppet/services/pacemaker/neutron-ovs-agent.yaml OS::TripleO::Services::RabbitMQ: ../puppet/services/pacemaker/rabbitmq.yaml OS::TripleO::Services::HAproxy: ../puppet/services/pacemaker/haproxy.yaml diff --git a/network/management_v6.yaml b/network/management_v6.yaml new file mode 100644 index 00000000..a5e70667 --- /dev/null +++ b/network/management_v6.yaml @@ -0,0 +1,69 @@ +heat_template_version: 2015-04-30 + +description: > + Management network. System administration, SSH, DNS, NTP, etc. This network + would usually be the default gateway for the non-controller nodes. + +parameters: + # the defaults here work for static IP assignment (IPAM) only + ManagementNetCidr: + default: 'fd00:fd00:fd00:6000::/64' + description: Cidr for the management network. + type: string + ManagementNetValueSpecs: + default: {'provider:physical_network': 'management', 'provider:network_type': 'flat'} + description: Value specs for the management network. + type: json + ManagementNetAdminStateUp: + default: false + description: This admin state of of the network. + type: boolean + ManagementNetShared: + default: false + description: Whether this network is shared across all tenants. + type: boolean + ManagementNetName: + default: management + description: The name of the management network. + type: string + ManagementSubnetName: + default: management_subnet + description: The name of the management subnet in Neutron. + type: string + ManagementAllocationPools: + default: [{'start': 'fd00:fd00:fd00:6000::10', 'end': 'fd00:fd00:fd00:6000:ffff:ffff:ffff:fffe'}] + description: Ip allocation pool range for the management network. + type: json + IPv6AddressMode: + default: dhcpv6-stateful + description: Neutron subnet IPv6 address mode + type: string + IPv6RAMode: + default: dhcpv6-stateful + description: Neutron subnet IPv6 router advertisement mode + type: string + +resources: + ManagementNetwork: + type: OS::Neutron::Net + properties: + admin_state_up: {get_param: ManagementNetAdminStateUp} + name: {get_param: ManagementNetName} + shared: {get_param: ManagementNetShared} + value_specs: {get_param: ManagementNetValueSpecs} + + ManagementSubnet: + type: OS::Neutron::Subnet + properties: + ip_version: 6 + ipv6_address_mode: {get_param: IPv6AddressMode} + ipv6_ra_mode: {get_param: IPv6RAMode} + cidr: {get_param: ManagementNetCidr} + name: {get_param: ManagementSubnetName} + network: {get_resource: ManagementNetwork} + allocation_pools: {get_param: ManagementAllocationPools} + +outputs: + OS::stack_id: + description: Neutron management network + value: {get_resource: ManagementNetwork} diff --git a/network/ports/management_from_pool_v6.yaml b/network/ports/management_from_pool_v6.yaml new file mode 100644 index 00000000..d9ac6046 --- /dev/null +++ b/network/ports/management_from_pool_v6.yaml @@ -0,0 +1,52 @@ +heat_template_version: 2015-10-15 + +description: > + Returns an IP from a network mapped list of IPs. This version is for IPv6 + addresses. The ip_address_uri output will have brackets for use in URLs. + +parameters: + ManagementNetName: + description: Name of the management network + default: management + type: string + PortName: + description: Name of the port + default: '' + type: string + ControlPlaneIP: # Here for compatability with noop.yaml + description: IP address on the control plane + default: '' + type: string + IPPool: + default: {} + description: A network mapped list of IPs + type: json + NodeIndex: + default: 0 + description: Index of the IP to get from Pool + type: number + ManagementNetCidr: + default: 'fd00:fd00:fd00:6000::/64' + description: Cidr for the management network. + type: string + +outputs: + ip_address: + description: management network IP + value: {get_param: [IPPool, {get_param: ManagementNetName}, {get_param: NodeIndex}]} + ip_address_uri: + description: management network IP (for compatibility with management_v6.yaml) + value: + list_join: + - '' + - - '[' + - {get_param: [IPPool, {get_param: ManagementNetName}, {get_param: NodeIndex}]} + - ']' + ip_subnet: + description: IP/Subnet CIDR for the management network IP + value: + list_join: + - '' + - - {get_param: [IPPool, {get_param: ManagementNetName}, {get_param: NodeIndex}]} + - '/' + - {str_split: ['/', {get_attr: [ManagementPort, subnets, 0, cidr]}, 1]} diff --git a/overcloud-resource-registry-puppet.yaml b/overcloud-resource-registry-puppet.yaml index 4fab2466..594a77a0 100644 --- a/overcloud-resource-registry-puppet.yaml +++ b/overcloud-resource-registry-puppet.yaml @@ -139,7 +139,7 @@ resource_registry: OS::TripleO::Services::NeutronCorePlugin: puppet/services/neutron-plugin-ml2.yaml # Neutron Core Plugin Vendors (these typically override NeutronCorePlugin) OS::TripleO::Services::NeutronCorePluginPlumgrid: puppet/services/neutron-plugin-plumgrid.yaml - + OS::TripleO::Services::NeutronCorePluginNuage: puppet/services/neutron-plugin-nuage.yaml OS::TripleO::Services::NeutronOvsAgent: puppet/services/neutron-ovs-agent.yaml OS::TripleO::Services::RabbitMQ: puppet/services/rabbitmq.yaml OS::TripleO::Services::HAproxy: puppet/services/haproxy.yaml diff --git a/overcloud.yaml b/overcloud.yaml index d8955b9e..60424885 100644 --- a/overcloud.yaml +++ b/overcloud.yaml @@ -411,7 +411,7 @@ parameters: description: Template string to be used to generate instance names type: string ManageFirewall: - default: false + default: true description: Whether to manage IPtables rules. type: boolean PurgeFirewallRules: diff --git a/puppet/ceph-cluster-config.yaml b/puppet/ceph-cluster-config.yaml index 245710f2..6beb751f 100644 --- a/puppet/ceph-cluster-config.yaml +++ b/puppet/ceph-cluster-config.yaml @@ -33,6 +33,9 @@ parameters: CinderRbdPoolName: default: volumes type: string + CinderBackupRbdPoolName: + default: backups + type: string GlanceRbdPoolName: default: images type: string @@ -97,7 +100,7 @@ resources: secret: 'CLIENT_KEY', mode: '0644', cap_mon: 'allow r', - cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL' + cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL' } }" params: @@ -106,6 +109,7 @@ resources: ADMIN_KEY: {get_param: ceph_admin_key} NOVA_POOL: {get_param: NovaRbdPoolName} CINDER_POOL: {get_param: CinderRbdPoolName} + CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName} GLANCE_POOL: {get_param: GlanceRbdPoolName} GNOCCHI_POOL: {get_param: GnocchiRbdPoolName} nova::compute::rbd::libvirt_images_rbd_pool: {get_param: NovaRbdPoolName} @@ -130,6 +134,7 @@ resources: tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName} ceph_pools: - {get_param: CinderRbdPoolName} + - {get_param: CinderBackupRbdPoolName} - {get_param: NovaRbdPoolName} - {get_param: GlanceRbdPoolName} - {get_param: GnocchiRbdPoolName} diff --git a/puppet/controller.yaml b/puppet/controller.yaml index 70989ccf..516a9ee8 100644 --- a/puppet/controller.yaml +++ b/puppet/controller.yaml @@ -694,6 +694,15 @@ resources: - '@' - {get_param: [EndpointMap, MysqlInternal, host]} - '/gnocchi' + aodh_dsn: + list_join: + - '' + - - {get_param: [EndpointMap, MysqlInternal, protocol]} + - '://aodh:' + - {get_param: AodhPassword} + - '@' + - {get_param: [EndpointMap, MysqlInternal, host]} + - '/aodh' gnocchi_internal_url: {get_param: [EndpointMap, GnocchiInternal, uri]} gnocchi_public_url: { get_param: [ EndpointMap, GnocchiPublic, uri ] } gnocchi_admin_url: { get_param: [ EndpointMap, GnocchiAdmin, uri ] } @@ -962,6 +971,7 @@ resources: snmpd_readonly_user_password: {get_input: snmpd_readonly_user_password} # Aodh + aodh_mysql_conn_string: {get_input: aodh_dsn} aodh::rabbit_userid: {get_input: rabbit_username} aodh::rabbit_password: {get_input: rabbit_password} aodh::rabbit_use_ssl: {get_input: rabbit_client_use_ssl} @@ -974,6 +984,7 @@ resources: aodh::api::keystone_password: {get_input: aodh_password} aodh::api::keystone_auth_uri: {get_input: keystone_auth_uri} aodh::api::keystone_identity_uri: {get_input: keystone_identity_uri} + aodh::auth::auth_url: {get_input: keystone_auth_uri} aodh::auth::auth_password: {get_input: aodh_password} aodh::db::mysql::password: {get_input: aodh_password} # for a migration path from ceilometer-alarm to aodh, we use the same database & coordination diff --git a/puppet/extraconfig/ceph/ceph-external-config.yaml b/puppet/extraconfig/ceph/ceph-external-config.yaml index 7d4dc15b..7c05a5b9 100644 --- a/puppet/extraconfig/ceph/ceph-external-config.yaml +++ b/puppet/extraconfig/ceph/ceph-external-config.yaml @@ -35,6 +35,9 @@ parameters: CinderRbdPoolName: default: volumes type: string + CinderBackupRbdPoolName: + default: backups + type: string GlanceRbdPoolName: default: images type: string @@ -71,7 +74,7 @@ resources: secret: 'CLIENT_KEY', mode: '0644', cap_mon: 'allow r', - cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL' + cap_osd: 'allow class-read object_prefix rbd_children, allow rwx pool=CINDER_POOL, allow rwx pool=CINDERBACKUP_POOL, allow rwx pool=NOVA_POOL, allow rwx pool=GLANCE_POOL, allow rwx pool=GNOCCHI_POOL' } }" params: @@ -79,6 +82,7 @@ resources: CLIENT_KEY: {get_param: ceph_client_key} NOVA_POOL: {get_param: NovaRbdPoolName} CINDER_POOL: {get_param: CinderRbdPoolName} + CINDERBACKUP_POOL: {get_param: CinderBackupRbdPoolName} GLANCE_POOL: {get_param: GlanceRbdPoolName} GNOCCHI_POOL: {get_param: GnocchiRbdPoolName} ceph::profile::params::ms_bind_ipv6: {get_param: CephIPv6} @@ -104,6 +108,7 @@ resources: tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName} ceph_pools: - {get_param: CinderRbdPoolName} + - {get_param: CinderBackupRbdPoolName} - {get_param: NovaRbdPoolName} - {get_param: GlanceRbdPoolName} - {get_param: GnocchiRbdPoolName} diff --git a/puppet/hieradata/controller.yaml b/puppet/hieradata/controller.yaml index e69656cf..7db2b5de 100644 --- a/puppet/hieradata/controller.yaml +++ b/puppet/hieradata/controller.yaml @@ -186,13 +186,13 @@ controller_classes: [] # firewall tripleo::firewall::firewall_rules: '101 mongodb_config': - port: 27019 + dport: 27019 '102 mongodb_sharding': - port: 27018 + dport: 27018 '103 mongod': - port: 27017 + dport: 27017 '104 mysql galera': - port: + dport: - 873 - 3306 - 4444 @@ -200,37 +200,37 @@ tripleo::firewall::firewall_rules: - 4568 - 9200 '105 ntp': - port: 123 + dport: 123 proto: udp '106 vrrp': proto: vrrp '107 haproxy stats': - port: 1993 + dport: 1993 '108 redis': - port: + dport: - 6379 - 26379 '109 rabbitmq': - port: + dport: - 5672 - 35672 '110 ceph': - port: + dport: - 6789 - '6800-6810' '111 keystone': - port: + dport: - 5000 - 13000 - 35357 - 13357 '112 glance': - port: + dport: - 9292 - 9191 - 13292 '113 nova': - port: + dport: - 6080 - 13080 - 8773 @@ -239,43 +239,43 @@ tripleo::firewall::firewall_rules: - 13774 - 8775 '114 neutron server': - port: + dport: - 9696 - 13696 '115 neutron dhcp input': proto: 'udp' - port: 67 + dport: 67 '116 neutron dhcp output': proto: 'udp' chain: 'OUTPUT' - port: 68 + dport: 68 '118 neutron vxlan networks': proto: 'udp' - port: 4789 + dport: 4789 '119 cinder': - port: + dport: - 8776 - 13776 '120 iscsi initiator': - port: 3260 + dport: 3260 '121 memcached': - port: 11211 + dport: 11211 '122 swift proxy': - port: + dport: - 8080 - 13808 '123 swift storage': - port: + dport: - 873 - 6000 - 6001 - 6002 '124 ceilometer': - port: + dport: - 8777 - 13777 '125 heat': - port: + dport: - 8000 - 13800 - 8003 @@ -283,17 +283,30 @@ tripleo::firewall::firewall_rules: - 8004 - 13004 '126 horizon': - port: + dport: - 80 - 443 '127 snmp': - port: 161 + dport: 161 proto: 'udp' '128 aodh': - port: + dport: - 8042 - 13042 '129 gnocchi-api': - port: + dport: - 8041 - 13041 + '130 pacemaker tcp': + proto: 'tcp' + dport: + - 2224 + - 3121 + - 21064 + '131 pacemaker udp': + proto: 'udp' + dport: 5405 + '132 sahara': + dport: + - 8386 + - 13386 diff --git a/puppet/hieradata/database.yaml b/puppet/hieradata/database.yaml index 4eb199c8..f2c95de6 100644 --- a/puppet/hieradata/database.yaml +++ b/puppet/hieradata/database.yaml @@ -69,6 +69,15 @@ gnocchi::db::mysql::allowed_hosts: - '%' - "%{hiera('mysql_bind_host')}" +# Aodh +aodh::db::mysql::user: aodh +aodh::db::mysql::host: "%{hiera('mysql_virtual_ip')}" +aodh::db::mysql::dbname: aodh +aodh::db::mysql::allowed_hosts: + - '%' + - "%{hiera('mysql_bind_host')}" + + sahara::db::mysql::user: sahara sahara::db::mysql::host: "%{hiera('mysql_virtual_ip')}" sahara::db::mysql::dbname: sahara diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp index ae2495aa..29c8d1b1 100644 --- a/puppet/manifests/overcloud_controller.pp +++ b/puppet/manifests/overcloud_controller.pp @@ -84,8 +84,8 @@ if hiera('step') >= 2 { } if downcase(hiera('ceilometer_backend')) == 'mysql' { include ::ceilometer::db::mysql - include ::aodh::db::mysql } + include ::aodh::db::mysql $enable_ceph = hiera('ceph_storage_count', 0) > 0 or hiera('enable_ceph_storage', false) @@ -201,12 +201,10 @@ if hiera('step') >= 4 { } - # If the value of core plugin is set to 'nuage' or'opencontrail' - # include nuage or opencontrail core plugins + # If the value of core plugin is set to 'opencontrail' + # include opencontrail core plugins # else use the default value of 'ml2' - if hiera('neutron::core_plugin') == 'neutron.plugins.nuage.plugin.NuagePlugin' { - include ::neutron::plugins::nuage - } elsif hiera('neutron::core_plugin') == 'neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2' { + if hiera('neutron::core_plugin') == 'neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2' { include ::neutron::plugins::opencontrail } else { @@ -279,11 +277,9 @@ if hiera('step') >= 4 { # Aodh class { '::aodh' : - database_connection => $ceilometer_database_connection, + database_connection => hiera('aodh_mysql_conn_string'), } include ::aodh::db::sync - # To manage the upgrade: - Exec['ceilometer-dbsync'] -> Exec['aodh-db-sync'] include ::aodh::auth include ::aodh::api include ::aodh::wsgi::apache diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp index 77189bd9..ff065695 100644 --- a/puppet/manifests/overcloud_controller_pacemaker.pp +++ b/puppet/manifests/overcloud_controller_pacemaker.pp @@ -269,6 +269,10 @@ if hiera('step') >= 2 { require => Exec['galera-ready'], } } + + class { '::aodh::db::mysql': + require => Exec['galera-ready'], + } } # Ceph @@ -399,9 +403,6 @@ MYSQL_HOST=localhost\n", } - if hiera('neutron::core_plugin') == 'neutron.plugins.nuage.plugin.NuagePlugin' { - include ::neutron::plugins::nuage - } if hiera('neutron::core_plugin') == 'neutron_plugin_contrail.plugins.opencontrail.contrail_plugin.NeutronPluginContrailCoreV2' { include ::neutron::plugins::opencontrail } @@ -519,7 +520,7 @@ MYSQL_HOST=localhost\n", # Aodh class { '::aodh' : - database_connection => $ceilometer_database_connection, + database_connection => hiera('aodh_mysql_conn_string'), } include ::aodh::config include ::aodh::auth diff --git a/puppet/services/neutron-plugin-nuage.yaml b/puppet/services/neutron-plugin-nuage.yaml new file mode 100644 index 00000000..db87f504 --- /dev/null +++ b/puppet/services/neutron-plugin-nuage.yaml @@ -0,0 +1,26 @@ +heat_template_version: 2016-04-08 + +description: > + OpenStack Neutron Nuage plugin + +parameters: + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + +resources: + + NeutronBase: + type: ./neutron-base.yaml + +outputs: + role_data: + description: Role data for the Neutron Nuage plugin + value: + config_settings: + map_merge: + - get_attr: [NeutronBase, role_data, config_settings] + step_config: | + include tripleo::profile::base::neutron::plugins::nuage diff --git a/puppet/services/pacemaker/neutron-plugin-nuage.yaml b/puppet/services/pacemaker/neutron-plugin-nuage.yaml new file mode 100644 index 00000000..704d922a --- /dev/null +++ b/puppet/services/pacemaker/neutron-plugin-nuage.yaml @@ -0,0 +1,28 @@ +heat_template_version: 2016-04-08 + +description: > + OpenStack Neutron Nuage Plugin with Pacemaker configured with Puppet + +parameters: + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + +resources: + + NeutronPluginNuageBase: + type: ../neutron-plugin-nuage.yaml + properties: + EndpointMap: {get_param: EndpointMap} + +outputs: + role_data: + description: Role data for the Neutron Nuage plugin. + value: + config_settings: + map_merge: + - get_attr: [NeutronPluginNuageBase, role_data, config_settings] + step_config: | + include ::tripleo::profile::pacemaker::neutron::plugins::nuage |