diff options
Diffstat (limited to 'ci/environments')
-rw-r--r-- | ci/environments/multinode-3nodes.yaml | 3 | ||||
-rw-r--r-- | ci/environments/multinode-containers.yaml | 4 | ||||
-rw-r--r-- | ci/environments/multinode-core.yaml | 1 | ||||
-rw-r--r-- | ci/environments/multinode.yaml | 3 | ||||
-rw-r--r-- | ci/environments/multinode_major_upgrade.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario001-multinode-containers.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario001-multinode.yaml | 2 | ||||
-rw-r--r-- | ci/environments/scenario002-multinode-containers.yaml | 4 | ||||
-rw-r--r-- | ci/environments/scenario002-multinode.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario003-multinode-containers.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario003-multinode.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario004-multinode-containers.yaml | 7 | ||||
-rw-r--r-- | ci/environments/scenario004-multinode.yaml | 3 | ||||
-rw-r--r-- | ci/environments/scenario006-multinode-containers.yaml | 61 | ||||
-rw-r--r-- | ci/environments/scenario007-multinode.yaml | 75 |
15 files changed, 176 insertions, 2 deletions
diff --git a/ci/environments/multinode-3nodes.yaml b/ci/environments/multinode-3nodes.yaml index ef51a779..e040b015 100644 --- a/ci/environments/multinode-3nodes.yaml +++ b/ci/environments/multinode-3nodes.yaml @@ -55,8 +55,10 @@ - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::MySQLClient - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid - name: Controller CountDefault: 1 @@ -79,3 +81,4 @@ - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid diff --git a/ci/environments/multinode-containers.yaml b/ci/environments/multinode-containers.yaml index 10a2c8c2..7768c4f0 100644 --- a/ci/environments/multinode-containers.yaml +++ b/ci/environments/multinode-containers.yaml @@ -16,6 +16,7 @@ resource_registry: # Some infra instances don't pass the ping test but are otherwise working. # Since the OVB jobs also test this functionality we can shut it off here. OS::TripleO::AllNodes::Validation: ../common/all-nodes-validation-disabled.yaml + OS::TripleO::Services::NovaMigrationTarget: OS::Heat::None parameter_defaults: ControllerServices: @@ -59,7 +60,9 @@ parameter_defaults: - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -71,3 +74,4 @@ parameter_defaults: ceph::profile::params::osd_max_object_namespace_len: 64 SwiftCeilometerPipelineEnabled: False Debug: True + NotificationDriver: 'noop' diff --git a/ci/environments/multinode-core.yaml b/ci/environments/multinode-core.yaml index b5316f1b..0dd59e96 100644 --- a/ci/environments/multinode-core.yaml +++ b/ci/environments/multinode-core.yaml @@ -21,6 +21,7 @@ parameters: Debug: type: string default: '' + description: Set to True to enable debugging on all services. resources: diff --git a/ci/environments/multinode.yaml b/ci/environments/multinode.yaml index 1752a2f6..2b25e58e 100644 --- a/ci/environments/multinode.yaml +++ b/ci/environments/multinode.yaml @@ -57,7 +57,9 @@ parameter_defaults: - OS::TripleO::Services::NovaLibvirt - OS::TripleO::Services::Horizon - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -69,3 +71,4 @@ parameter_defaults: ceph::profile::params::osd_max_object_namespace_len: 64 SwiftCeilometerPipelineEnabled: False Debug: True + NotificationDriver: 'noop' diff --git a/ci/environments/multinode_major_upgrade.yaml b/ci/environments/multinode_major_upgrade.yaml index 609e06ff..d8f71414 100644 --- a/ci/environments/multinode_major_upgrade.yaml +++ b/ci/environments/multinode_major_upgrade.yaml @@ -54,9 +54,11 @@ parameter_defaults: - OS::TripleO::Services::NovaScheduler - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::Pacemaker - OS::TripleO::Services::Horizon - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -65,3 +67,4 @@ parameter_defaults: heat::rpc_response_timeout: 600 SwiftCeilometerPipelineEnabled: False Debug: True + NotificationDriver: 'noop' diff --git a/ci/environments/scenario001-multinode-containers.yaml b/ci/environments/scenario001-multinode-containers.yaml index 89339d10..73dc5b14 100644 --- a/ci/environments/scenario001-multinode-containers.yaml +++ b/ci/environments/scenario001-multinode-containers.yaml @@ -26,6 +26,7 @@ resource_registry: # Some infra instances don't pass the ping test but are otherwise working. # Since the OVB jobs also test this functionality we can shut it off here. OS::TripleO::AllNodes::Validation: ../common/all-nodes-validation-disabled.yaml + OS::TripleO::Services::NovaMigrationTarget: OS::Heat::None parameter_defaults: ControllerServices: @@ -62,6 +63,7 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::MongoDb - OS::TripleO::Services::Redis - OS::TripleO::Services::AodhApi @@ -89,6 +91,7 @@ parameter_defaults: - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::FluentdClient - OS::TripleO::Services::SensuClient + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu diff --git a/ci/environments/scenario001-multinode.yaml b/ci/environments/scenario001-multinode.yaml index 8abd079f..54eef744 100644 --- a/ci/environments/scenario001-multinode.yaml +++ b/ci/environments/scenario001-multinode.yaml @@ -60,6 +60,7 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::MongoDb - OS::TripleO::Services::Redis - OS::TripleO::Services::AodhApi @@ -87,6 +88,7 @@ parameter_defaults: - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::FluentdClient - OS::TripleO::Services::SensuClient + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu diff --git a/ci/environments/scenario002-multinode-containers.yaml b/ci/environments/scenario002-multinode-containers.yaml index 07088633..d300f773 100644 --- a/ci/environments/scenario002-multinode-containers.yaml +++ b/ci/environments/scenario002-multinode-containers.yaml @@ -19,6 +19,7 @@ resource_registry: # Some infra instances don't pass the ping test but are otherwise working. # Since the OVB jobs also test this functionality we can shut it off here. OS::TripleO::AllNodes::Validation: ../common/all-nodes-validation-disabled.yaml + OS::TripleO::Services::NovaMigrationTarget: OS::Heat::None parameter_defaults: ControllerServices: @@ -53,6 +54,7 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::CinderApi - OS::TripleO::Services::CinderBackup - OS::TripleO::Services::CinderScheduler @@ -67,8 +69,10 @@ parameter_defaults: - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu Debug: true SwiftCeilometerPipelineEnabled: false + NotificationDriver: 'noop' diff --git a/ci/environments/scenario002-multinode.yaml b/ci/environments/scenario002-multinode.yaml index 79b74af1..cdbcbfd6 100644 --- a/ci/environments/scenario002-multinode.yaml +++ b/ci/environments/scenario002-multinode.yaml @@ -52,6 +52,7 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::CinderApi - OS::TripleO::Services::CinderBackup - OS::TripleO::Services::CinderScheduler @@ -66,8 +67,10 @@ parameter_defaults: - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu Debug: true SwiftCeilometerPipelineEnabled: false + NotificationDriver: 'noop' diff --git a/ci/environments/scenario003-multinode-containers.yaml b/ci/environments/scenario003-multinode-containers.yaml index 8e1b3a05..e3789ea8 100644 --- a/ci/environments/scenario003-multinode-containers.yaml +++ b/ci/environments/scenario003-multinode-containers.yaml @@ -20,6 +20,7 @@ resource_registry: # Some infra instances don't pass the ping test but are otherwise working. # Since the OVB jobs also test this functionality we can shut it off here. OS::TripleO::AllNodes::Validation: ../common/all-nodes-validation-disabled.yaml + OS::TripleO::Services::NovaMigrationTarget: OS::Heat::None parameter_defaults: ControllerServices: @@ -62,6 +63,7 @@ parameter_defaults: - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -70,3 +72,4 @@ parameter_defaults: GlanceBackend: 'file' KeystoneTokenProvider: 'fernet' SwiftCeilometerPipelineEnabled: false + NotificationDriver: 'noop' diff --git a/ci/environments/scenario003-multinode.yaml b/ci/environments/scenario003-multinode.yaml index 2cce1680..5e797b40 100644 --- a/ci/environments/scenario003-multinode.yaml +++ b/ci/environments/scenario003-multinode.yaml @@ -50,6 +50,7 @@ parameter_defaults: - OS::TripleO::Services::Snmp - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::NovaLibvirt - OS::TripleO::Services::SaharaApi - OS::TripleO::Services::SaharaEngine @@ -59,6 +60,7 @@ parameter_defaults: - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -67,3 +69,4 @@ parameter_defaults: GlanceBackend: 'file' KeystoneTokenProvider: 'fernet' SwiftCeilometerPipelineEnabled: false + NotificationDriver: 'noop' diff --git a/ci/environments/scenario004-multinode-containers.yaml b/ci/environments/scenario004-multinode-containers.yaml index ba530162..6d795f97 100644 --- a/ci/environments/scenario004-multinode-containers.yaml +++ b/ci/environments/scenario004-multinode-containers.yaml @@ -16,8 +16,7 @@ resource_registry: OS::TripleO::Services::SwiftRingBuilder: OS::Heat::None OS::TripleO::Services::ManilaApi: ../../docker/services/manila-api.yaml OS::TripleO::Services::ManilaScheduler: ../../docker/services/manila-scheduler.yaml - # NOTE: being containerized here: https://review.openstack.org/#/c/471527/ - OS::TripleO::Services::ManilaShare: ../../puppet/services/manila-share.yaml + OS::TripleO::Services::ManilaShare: ../../docker/services/manila-share.yaml OS::TripleO::Services::ManilaBackendCephFs: ../../puppet/services/manila-backend-cephfs.yaml # TODO: containerize NeutronBgpVpnApi OS::TripleO::Services::NeutronBgpVpnApi: ../../puppet/services/neutron-bgpvpn-api.yaml @@ -30,6 +29,7 @@ resource_registry: # Some infra instances don't pass the ping test but are otherwise working. # Since the OVB jobs also test this functionality we can shut it off here. OS::TripleO::AllNodes::Validation: ../common/all-nodes-validation-disabled.yaml + OS::TripleO::Services::NovaMigrationTarget: OS::Heat::None parameter_defaults: @@ -74,9 +74,11 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -96,3 +98,4 @@ parameter_defaults: SwiftCeilometerPipelineEnabled: false NeutronServicePlugins: 'router, networking_bgpvpn.neutron.services.plugin.BGPVPNPlugin' BgpvpnServiceProvider: 'BGPVPN:Dummy:networking_bgpvpn.neutron.services.service_drivers.driver_api.BGPVPNDriver:default' + NotificationDriver: 'noop' diff --git a/ci/environments/scenario004-multinode.yaml b/ci/environments/scenario004-multinode.yaml index 8bf805df..bd30347a 100644 --- a/ci/environments/scenario004-multinode.yaml +++ b/ci/environments/scenario004-multinode.yaml @@ -74,9 +74,11 @@ parameter_defaults: - OS::TripleO::Services::Timezone - OS::TripleO::Services::NovaCompute - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::NovaMigrationTarget - OS::TripleO::Services::TripleoPackages - OS::TripleO::Services::TripleoFirewall - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Iscsid ControllerExtraConfig: nova::compute::libvirt::services::libvirt_virt_type: qemu nova::compute::libvirt::libvirt_virt_type: qemu @@ -98,3 +100,4 @@ parameter_defaults: NeutronServicePlugins: 'router, networking_bgpvpn.neutron.services.plugin.BGPVPNPlugin, networking_l2gw.services.l2gateway.plugin.L2GatewayPlugin' BgpvpnServiceProvider: 'BGPVPN:Dummy:networking_bgpvpn.neutron.services.service_drivers.driver_api.BGPVPNDriver:default' L2gwServiceProvider: ['L2GW:l2gw:networking_l2gw.services.l2gateway.service_drivers.L2gwDriver:default'] + NotificationDriver: 'noop' diff --git a/ci/environments/scenario006-multinode-containers.yaml b/ci/environments/scenario006-multinode-containers.yaml new file mode 100644 index 00000000..4715e339 --- /dev/null +++ b/ci/environments/scenario006-multinode-containers.yaml @@ -0,0 +1,61 @@ +resource_registry: + OS::TripleO::Controller::Net::SoftwareConfig: ../common/net-config-multinode.yaml + OS::TripleO::Compute::Net::SoftwareConfig: ../common/net-config-multinode.yaml + OS::TripleO::Services::NovaIronic: ../docker/services/nova-ironic.yaml + OS::TripleO::Services::IronicApi: ../docker/services/ironic-api.yaml + OS::TripleO::Services::IronicConductor: ../docker/services/ironic-conductor.yaml + OS::TripleO::Services::IronicPxe: ../docker/services/ironic-pxe.yaml + OS::TripleO::Services::Docker: OS::Heat::None + +parameter_defaults: + ControllerServices: + - OS::TripleO::Services::Docker + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::Keystone + - OS::TripleO::Services::GlanceApi + - OS::TripleO::Services::IronicApi + - OS::TripleO::Services::IronicConductor + - OS::TripleO::Services::IronicPxe + - OS::TripleO::Services::MySQL + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronDhcpAgent + - OS::TripleO::Services::NeutronL3Agent + - OS::TripleO::Services::NeutronMetadataAgent + - OS::TripleO::Services::NeutronServer + - OS::TripleO::Services::NeutronCorePlugin + - OS::TripleO::Services::NeutronOvsAgent + - OS::TripleO::Services::RabbitMQ + - OS::TripleO::Services::HAproxy + - OS::TripleO::Services::Keepalived + - OS::TripleO::Services::Memcached + - OS::TripleO::Services::Pacemaker + - OS::TripleO::Services::NovaConductor + - OS::TripleO::Services::NovaApi + - OS::TripleO::Services::NovaIronic + - OS::TripleO::Services::NovaPlacement + - OS::TripleO::Services::NovaMetadata + - OS::TripleO::Services::NovaScheduler + - OS::TripleO::Services::Ntp + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Sshd + - OS::TripleO::Services::Securetty + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::MongoDb + - OS::TripleO::Services::Redis + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::TripleoFirewall + + Debug: true + BannerText: | + ****************************************************************** + * This system is for the use of authorized users only. Usage of * + * this system may be monitored and recorded by system personnel. * + * Anyone using this system expressly consents to such monitoring * + * and is advised that if such monitoring reveals possible * + * evidence of criminal activity, system personnel may provide * + * the evidence from such monitoring to law enforcement officials.* + ****************************************************************** + # we don't deploy Swift so we switch to file backend. + GlanceBackend: 'file' + IronicCleaningDiskErase: 'metadata' + NotificationDriver: 'noop' diff --git a/ci/environments/scenario007-multinode.yaml b/ci/environments/scenario007-multinode.yaml new file mode 100644 index 00000000..6db00ef1 --- /dev/null +++ b/ci/environments/scenario007-multinode.yaml @@ -0,0 +1,75 @@ +resource_registry: + OS::TripleO::Controller::Net::SoftwareConfig: ../common/net-config-multinode.yaml + OS::TripleO::Compute::Net::SoftwareConfig: ../common/net-config-multinode.yaml + OS::TripleO::Services::RabbitMQ: ../../puppet/services/pacemaker/rabbitmq.yaml + OS::TripleO::Services::HAproxy: ../../puppet/services/pacemaker/haproxy.yaml + OS::TripleO::Services::Pacemaker: ../../puppet/services/pacemaker.yaml + OS::TripleO::Services::MySQL: ../../puppet/services/pacemaker/database/mysql.yaml + OS::TripleO::Services::Keepalived: OS::Heat::None + OS::TripleO::Tasks::ControllerPreConfig: ../../extraconfig/tasks/pre_puppet_pacemaker.yaml + OS::TripleO::Tasks::ControllerPostConfig: ../../extraconfig/tasks/post_puppet_pacemaker.yaml + OS::TripleO::Tasks::ControllerPostPuppetRestart: ../../extraconfig/tasks/post_puppet_pacemaker_restart.yaml + # Disable neutron services not required for OVN and enable services required for OVN. + OS::TripleO::Services::NeutronL3Agent: OS::Heat::None + OS::TripleO::Services::NeutronOvsAgent: OS::Heat::None + OS::TripleO::Services::NeutronMetadataAgent: OS::Heat::None + OS::TripleO::Services::NeutronDhcpAgent: OS::Heat::None + OS::TripleO::Services::ComputeNeutronOvsAgent: OS::Heat::None + OS::TripleO::Services::NeutronCorePlugin: OS::TripleO::Services::NeutronCorePluginML2OVN + OS::TripleO::Services::ComputeNeutronCorePlugin: ../../puppet/services/neutron-compute-plugin-ovn.yaml + OS::TripleO::Services::OVNDBs: ../../puppet/services/ovn-dbs.yaml + +parameter_defaults: + ControllerServices: + - OS::TripleO::Services::Docker + - OS::TripleO::Services::Kernel + - OS::TripleO::Services::Keystone + - OS::TripleO::Services::GlanceApi + - OS::TripleO::Services::HeatApi + - OS::TripleO::Services::HeatApiCfn + - OS::TripleO::Services::HeatApiCloudwatch + - OS::TripleO::Services::HeatEngine + - OS::TripleO::Services::MySQL + - OS::TripleO::Services::MySQLClient + - OS::TripleO::Services::NeutronServer + - OS::TripleO::Services::NeutronCorePlugin + - OS::TripleO::Services::OVNDBs + - OS::TripleO::Services::ComputeNeutronCorePlugin + - OS::TripleO::Services::RabbitMQ + - OS::TripleO::Services::HAproxy + - OS::TripleO::Services::Keepalived + - OS::TripleO::Services::Memcached + - OS::TripleO::Services::Pacemaker + - OS::TripleO::Services::NovaConductor + - OS::TripleO::Services::NovaApi + - OS::TripleO::Services::NovaPlacement + - OS::TripleO::Services::NovaMetadata + - OS::TripleO::Services::NovaScheduler + - OS::TripleO::Services::Ntp + - OS::TripleO::Services::Snmp + - OS::TripleO::Services::Timezone + - OS::TripleO::Services::NovaCompute + - OS::TripleO::Services::NovaLibvirt + - OS::TripleO::Services::TripleoPackages + - OS::TripleO::Services::TripleoFirewall + - OS::TripleO::Services::Sshd + ControllerExtraConfig: + nova::compute::libvirt::services::libvirt_virt_type: qemu + nova::compute::libvirt::libvirt_virt_type: qemu + # For OVN. + NeutronMechanismDrivers: ovn + OVNVifType: ovs + OVNNeutronSyncMode: log + OVNQosDriver: ovn-qos + OVNTunnelEncapType: geneve + NeutronEnableDHCPAgent: false + NeutronTypeDrivers: 'geneve,vlan,flat,vxlan' + NeutronNetworkType: 'geneve' + NeutronServicePlugins: 'qos,networking_ovn.l3.l3_ovn.OVNL3RouterPlugin' + NeutronVniRanges: ['1:65536', ] + OVNBridgeMappings: 'datacentre:br-ex' + Debug: true + # we don't deploy Swift so we switch to file backend. + GlanceBackend: 'file' + KeystoneTokenProvider: 'fernet' + SwiftCeilometerPipelineEnabled: false |