diff options
Diffstat (limited to 'manifests/profile')
| -rw-r--r-- | manifests/profile/base/database/mysql.pp | 3 | ||||
| -rw-r--r-- | manifests/profile/base/keystone.pp | 3 | ||||
| -rw-r--r-- | manifests/profile/base/neutron/server.pp | 9 | ||||
| -rw-r--r-- | manifests/profile/base/tacker.pp | 86 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/ceph/rbdmirror.pp | 2 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/cinder/backup.pp | 22 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/cinder/volume.pp | 22 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/database/mysql.pp | 20 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/database/redis.pp | 36 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/haproxy.pp | 79 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/manila.pp | 22 | ||||
| -rw-r--r-- | manifests/profile/pacemaker/rabbitmq.pp | 50 |
12 files changed, 298 insertions, 56 deletions
diff --git a/manifests/profile/base/database/mysql.pp b/manifests/profile/base/database/mysql.pp index 5154464..6f93903 100644 --- a/manifests/profile/base/database/mysql.pp +++ b/manifests/profile/base/database/mysql.pp @@ -185,6 +185,9 @@ class tripleo::profile::base::database::mysql ( if hiera('sahara_api_enabled', false) { include ::sahara::db::mysql } + if hiera('tacker_enabled', false) { + include ::tacker::db::mysql + } if hiera('trove_api_enabled', false) { include ::trove::db::mysql } diff --git a/manifests/profile/base/keystone.pp b/manifests/profile/base/keystone.pp index a3a39e9..e84e22f 100644 --- a/manifests/profile/base/keystone.pp +++ b/manifests/profile/base/keystone.pp @@ -248,6 +248,9 @@ class tripleo::profile::base::keystone ( if hiera('swift_proxy_enabled', false) { include ::swift::keystone::auth } + if hiera('tacker_enabled', false) { + include ::tacker::keystone::auth + } if hiera('trove_api_enabled', false) { include ::trove::keystone::auth } diff --git a/manifests/profile/base/neutron/server.pp b/manifests/profile/base/neutron/server.pp index 4667ae2..fcd804d 100644 --- a/manifests/profile/base/neutron/server.pp +++ b/manifests/profile/base/neutron/server.pp @@ -73,7 +73,7 @@ class tripleo::profile::base::neutron::server ( # We start neutron-server on the bootstrap node first, because # it will try to populate tables and we need to make sure this happens # before it starts on other nodes - if $step >= 4 and $sync_db { + if $step >= 4 and $sync_db or $step >= 5 and !$sync_db { include ::neutron::server::notifications # We need to override the hiera value neutron::server::sync_db which is set # to true @@ -82,11 +82,4 @@ class tripleo::profile::base::neutron::server ( l3_ha => $l3_ha, } } - if $step >= 5 and !$sync_db { - include ::neutron::server::notifications - class { '::neutron::server': - sync_db => $sync_db, - l3_ha => $l3_ha, - } - } } diff --git a/manifests/profile/base/tacker.pp b/manifests/profile/base/tacker.pp new file mode 100644 index 0000000..e9f6b77 --- /dev/null +++ b/manifests/profile/base/tacker.pp @@ -0,0 +1,86 @@ +# Copyright 2016 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# == Class: tripleo::profile::base::tacker +# +# Tacker server profile for tripleo +# +# === Parameters +# +# [*bootstrap_node*] +# (Optional) The hostname of the node responsible for bootstrapping tasks +# Defaults to hiera('bootstrap_nodeid') +# +# [*messaging_driver*] +# Driver for messaging service. +# Defaults to hiera('messaging_service_name', 'rabbit') +# +# [*messaging_hosts*] +# list of the messaging host fqdns +# Defaults to hiera('rabbitmq_node_names') +# +# [*messaging_password*] +# Password for messaging nova queue +# Defaults to hiera('nova::rabbit_password') +# +# [*messaging_port*] +# IP port for messaging service +# Defaults to hiera('nova::rabbit_port', 5672) +# +# [*messaging_username*] +# Username for messaging nova queue +# Defaults to hiera('nova::rabbit_userid', 'guest') +# +# [*messaging_use_ssl*] +# Flag indicating ssl usage. +# Defaults to hiera('nova::rabbit_use_ssl', '0') +# +# [*step*] +# (Optional) The current step of the deployment +# Defaults to hiera('step') + +class tripleo::profile::base::tacker ( + $bootstrap_node = hiera('bootstrap_nodeid', undef), + $messaging_driver = hiera('messaging_service_name', 'rabbit'), + $messaging_hosts = any2array(hiera('rabbitmq_node_names', undef)), + $messaging_password = hiera('tacker::rabbit_password'), + $messaging_port = hiera('tacker::rabbit_port', '5672'), + $messaging_username = hiera('tacker::rabbit_userid', 'guest'), + $messaging_use_ssl = hiera('tacker::rabbit_use_ssl', '0'), + $step = hiera('step'), +) { + if $::hostname == downcase($bootstrap_node) { + $sync_db = true + } else { + $sync_db = false + } + + if $step >= 4 or ($step >= 3 and $sync_db){ + $messaging_use_ssl_real = sprintf('%s', bool2num(str2bool($messaging_use_ssl))) + class { '::tacker': + sync_db => $sync_db, + default_transport_url => os_transport_url({ + 'transport' => $messaging_driver, + 'hosts' => $messaging_hosts, + 'port' => sprintf('%s', $messaging_port), + 'username' => $messaging_username, + 'password' => $messaging_password, + 'ssl' => $messaging_use_ssl_real, + }), + } + + include ::tacker::server + include ::tacker::db + } +} diff --git a/manifests/profile/pacemaker/ceph/rbdmirror.pp b/manifests/profile/pacemaker/ceph/rbdmirror.pp index 1fab0db..ea445a1 100644 --- a/manifests/profile/pacemaker/ceph/rbdmirror.pp +++ b/manifests/profile/pacemaker/ceph/rbdmirror.pp @@ -37,7 +37,7 @@ # Defaults to hiera('step') # class tripleo::profile::pacemaker::ceph::rbdmirror ( - $bootstrap_node = hiera('ceph_rbdmirror_bootstrap_short_node_name'), + $bootstrap_node = hiera('ceph_rbdmirror_short_bootstrap_node_name'), $client_name = 'openstack', $stack_action = hiera('stack_action'), $step = hiera('step'), diff --git a/manifests/profile/pacemaker/cinder/backup.pp b/manifests/profile/pacemaker/cinder/backup.pp index 4e33a34..ff0d8c9 100644 --- a/manifests/profile/pacemaker/cinder/backup.pp +++ b/manifests/profile/pacemaker/cinder/backup.pp @@ -27,9 +27,14 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::cinder::backup ( $bootstrap_node = hiera('cinder_backup_short_bootstrap_node_name'), $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), ) { Service <| tag == 'cinder::backup' |> { @@ -47,6 +52,15 @@ class tripleo::profile::pacemaker::cinder::backup ( include ::tripleo::profile::base::cinder::backup + if $step >= 2 { + pacemaker::property { 'cinder-backup-role-node-property': + property => 'cinder-backup-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + } + if $step >= 3 and $pacemaker_master and hiera('stack_action') == 'UPDATE' { Cinder_config<||> ~> @@ -55,7 +69,13 @@ class tripleo::profile::pacemaker::cinder::backup ( if $step >= 5 and $pacemaker_master { pacemaker::resource::service { $::cinder::params::backup_service : - op_params => 'start timeout=200s stop timeout=200s', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['cinder-backup-role eq true'], + } } } diff --git a/manifests/profile/pacemaker/cinder/volume.pp b/manifests/profile/pacemaker/cinder/volume.pp index b03a1f4..0d6a598 100644 --- a/manifests/profile/pacemaker/cinder/volume.pp +++ b/manifests/profile/pacemaker/cinder/volume.pp @@ -27,9 +27,14 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::cinder::volume ( $bootstrap_node = hiera('cinder_volume_short_bootstrap_node_name'), $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), ) { Service <| tag == 'cinder::volume' |> { hasrestart => true, @@ -46,6 +51,15 @@ class tripleo::profile::pacemaker::cinder::volume ( include ::tripleo::profile::base::cinder::volume + if $step >= 2 { + pacemaker::property { 'cinder-volume-role-node-property': + property => 'cinder-volume-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + } + if $step >= 3 and $pacemaker_master and hiera('stack_action') == 'UPDATE' { Cinder_api_paste_ini<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::cinder::params::volume_service}"] Cinder_config<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::cinder::params::volume_service}"] @@ -54,7 +68,13 @@ class tripleo::profile::pacemaker::cinder::volume ( if $step >= 5 and $pacemaker_master { pacemaker::resource::service { $::cinder::params::volume_service : - op_params => 'start timeout=200s stop timeout=200s', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['cinder-volume-role eq true'], + } } } diff --git a/manifests/profile/pacemaker/database/mysql.pp b/manifests/profile/pacemaker/database/mysql.pp index 3506cb1..6a83f10 100644 --- a/manifests/profile/pacemaker/database/mysql.pp +++ b/manifests/profile/pacemaker/database/mysql.pp @@ -36,11 +36,16 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::database::mysql ( $bootstrap_node = hiera('mysql_short_bootstrap_node_name'), $bind_address = $::hostname, $gmcast_listen_addr = hiera('mysql_bind_host'), $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), ) { if $::hostname == downcase($bootstrap_node) { $pacemaker_master = true @@ -118,6 +123,12 @@ class tripleo::profile::pacemaker::database::mysql ( } if $step >= 2 { + pacemaker::property { 'galera-role-node-property': + property => 'galera-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } if $pacemaker_master { pacemaker::resource::ocf { 'galera' : ocf_agent_name => 'heartbeat:galera', @@ -125,7 +136,14 @@ class tripleo::profile::pacemaker::database::mysql ( master_params => '', meta_params => "master-max=${galera_nodes_count} ordered=true", resource_params => "additional_parameters='--open-files-limit=16384' enable_creation=true wsrep_cluster_address='gcomm://${galera_nodes}'", - require => Class['::mysql::server'], + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['galera-role eq true'], + }, + require => [Class['::mysql::server'], + Pacemaker::Property['galera-role-node-property']], before => Exec['galera-ready'], } exec { 'galera-ready' : diff --git a/manifests/profile/pacemaker/database/redis.pp b/manifests/profile/pacemaker/database/redis.pp index 7490fa0..3ef6815 100644 --- a/manifests/profile/pacemaker/database/redis.pp +++ b/manifests/profile/pacemaker/database/redis.pp @@ -36,11 +36,16 @@ # for when redis is managed by pacemaker. Defaults to hiera('redis_file_limit') # or 10240 (default in redis systemd limits) # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::database::redis ( $bootstrap_node = hiera('redis_short_bootstrap_node_name'), $enable_load_balancer = hiera('enable_load_balancer', true), $step = hiera('step'), $redis_file_limit = hiera('redis_file_limit', 10240), + $pcs_tries = hiera('pcs_tries', 20), ) { if $::hostname == downcase($bootstrap_node) { $pacemaker_master = true @@ -71,14 +76,29 @@ class tripleo::profile::pacemaker::database::redis ( } } - if $step >= 2 and $pacemaker_master { - pacemaker::resource::ocf { 'redis': - ocf_agent_name => 'heartbeat:redis', - master_params => '', - meta_params => 'notify=true ordered=true interleave=true', - resource_params => 'wait_last_known_master=true', - op_params => 'start timeout=200s stop timeout=200s', - require => Class['::redis'], + if $step >= 2 { + pacemaker::property { 'redis-role-node-property': + property => 'redis-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + if $pacemaker_master { + pacemaker::resource::ocf { 'redis': + ocf_agent_name => 'heartbeat:redis', + master_params => '', + meta_params => 'notify=true ordered=true interleave=true', + resource_params => 'wait_last_known_master=true', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['redis-role eq true'], + }, + require => [Class['::redis'], + Pacemaker::Property['redis-role-node-property']], + } } } } diff --git a/manifests/profile/pacemaker/haproxy.pp b/manifests/profile/pacemaker/haproxy.pp index b326761..f006f78 100644 --- a/manifests/profile/pacemaker/haproxy.pp +++ b/manifests/profile/pacemaker/haproxy.pp @@ -31,10 +31,15 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::haproxy ( $bootstrap_node = hiera('haproxy_short_bootstrap_node_name'), $enable_load_balancer = hiera('enable_load_balancer', true), $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), ) { include ::tripleo::profile::base::haproxy @@ -50,56 +55,90 @@ class tripleo::profile::pacemaker::haproxy ( } } - if $step >= 2 and $pacemaker_master and $enable_load_balancer { + if $step >= 2 and $enable_load_balancer { + pacemaker::property { 'haproxy-role-node-property': + property => 'haproxy-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + if $pacemaker_master { + $haproxy_location_rule = { + resource_discovery => 'exclusive', + score => 0, + expression => ['haproxy-role eq true'], + } # FIXME: we should not have to access tripleo::haproxy class # parameters here to configure pacemaker VIPs. The configuration # of pacemaker VIPs could move into puppet-tripleo or we should # make use of less specific hiera parameters here for the settings. pacemaker::resource::service { 'haproxy': - op_params => 'start timeout=200s stop timeout=200s', - clone_params => true, + op_params => 'start timeout=200s stop timeout=200s', + clone_params => true, + location_rule => $haproxy_location_rule, + tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $control_vip = hiera('controller_virtual_ip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_control_vip': - vip_name => 'control', - ip_address => $control_vip, + vip_name => 'control', + ip_address => $control_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $public_vip = hiera('public_virtual_ip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_public_vip': - ensure => $public_vip and $public_vip != $control_vip, - vip_name => 'public', - ip_address => $public_vip, + ensure => $public_vip and $public_vip != $control_vip, + vip_name => 'public', + ip_address => $public_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $redis_vip = hiera('redis_vip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_redis_vip': - ensure => $redis_vip and $redis_vip != $control_vip, - vip_name => 'redis', - ip_address => $redis_vip, + ensure => $redis_vip and $redis_vip != $control_vip, + vip_name => 'redis', + ip_address => $redis_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $internal_api_vip = hiera('internal_api_virtual_ip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_internal_api_vip': - ensure => $internal_api_vip and $internal_api_vip != $control_vip, - vip_name => 'internal_api', - ip_address => $internal_api_vip, + ensure => $internal_api_vip and $internal_api_vip != $control_vip, + vip_name => 'internal_api', + ip_address => $internal_api_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $storage_vip = hiera('storage_virtual_ip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_storage_vip': - ensure => $storage_vip and $storage_vip != $control_vip, - vip_name => 'storage', - ip_address => $storage_vip, + ensure => $storage_vip and $storage_vip != $control_vip, + vip_name => 'storage', + ip_address => $storage_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } $storage_mgmt_vip = hiera('storage_mgmt_virtual_ip') tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_storage_mgmt_vip': - ensure => $storage_mgmt_vip and $storage_mgmt_vip != $control_vip, - vip_name => 'storage_mgmt', - ip_address => $storage_mgmt_vip, + ensure => $storage_mgmt_vip and $storage_mgmt_vip != $control_vip, + vip_name => 'storage_mgmt', + ip_address => $storage_mgmt_vip, + location_rule => $haproxy_location_rule, + pcs_tries => $pcs_tries, + require => Pacemaker::Property['haproxy-role-node-property'], } + } } } diff --git a/manifests/profile/pacemaker/manila.pp b/manifests/profile/pacemaker/manila.pp index 547a86f..7bcf8d6 100644 --- a/manifests/profile/pacemaker/manila.pp +++ b/manifests/profile/pacemaker/manila.pp @@ -45,6 +45,10 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::manila ( $backend_generic_enabled = hiera('manila_backend_generic_enabled', false), $backend_netapp_enabled = hiera('manila_backend_netapp_enabled', false), @@ -52,6 +56,7 @@ class tripleo::profile::pacemaker::manila ( $ceph_mds_enabled = hiera('ceph_mds_enabled', false), $bootstrap_node = hiera('manila_share_short_bootstrap_node_name'), $step = hiera('step'), + $pcs_tries = hiera('pcs_tries', 20), ) { if $::hostname == downcase($bootstrap_node) { $pacemaker_master = true @@ -70,6 +75,15 @@ class tripleo::profile::pacemaker::manila ( include ::tripleo::profile::base::manila::share + if $step >= 2 { + pacemaker::property { 'manila-share-role-node-property': + property => 'manila-share-role', + value => true, + tries => $pcs_tries, + node => $::hostname, + } + } + if $step >= 4 { # manila generic: if $backend_generic_enabled { @@ -185,7 +199,13 @@ allow command \"auth get\", allow command \"auth get-or-create\"', # only manila-share is pacemaker managed, and in a/p pacemaker::resource::service { $::manila::params::share_service : - op_params => 'start timeout=200s stop timeout=200s', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['manila-share-role eq true'], + }, } } diff --git a/manifests/profile/pacemaker/rabbitmq.pp b/manifests/profile/pacemaker/rabbitmq.pp index 85ebe34..f4b679a 100644 --- a/manifests/profile/pacemaker/rabbitmq.pp +++ b/manifests/profile/pacemaker/rabbitmq.pp @@ -41,11 +41,16 @@ # for more details. # Defaults to hiera('step') # +# [*pcs_tries*] +# (Optional) The number of times pcs commands should be retried. +# Defaults to hiera('pcs_tries', 20) +# class tripleo::profile::pacemaker::rabbitmq ( $bootstrap_node = hiera('rabbitmq_short_bootstrap_node_name'), $erlang_cookie = hiera('rabbitmq::erlang_cookie'), $user_ha_queues = hiera('rabbitmq::nr_ha_queues', 0), $rabbit_nodes = hiera('rabbitmq_node_names'), + $pcs_tries = hiera('pcs_tries', 20), $step = hiera('step'), ) { if $::hostname == downcase($bootstrap_node) { @@ -72,22 +77,37 @@ class tripleo::profile::pacemaker::rabbitmq ( } } - if $step >= 2 and $pacemaker_master { - include ::stdlib - # The default nr of ha queues is ceiling(N/2) - if $user_ha_queues == 0 { - $nr_rabbit_nodes = size($rabbit_nodes) - $nr_ha_queues = $nr_rabbit_nodes / 2 + ($nr_rabbit_nodes % 2) - } else { - $nr_ha_queues = $user_ha_queues + if $step >= 2 { + pacemaker::property { 'rabbitmq-role-node-property': + property => 'rabbitmq-role', + value => true, + tries => $pcs_tries, + node => $::hostname, } - pacemaker::resource::ocf { 'rabbitmq': - ocf_agent_name => 'heartbeat:rabbitmq-cluster', - resource_params => "set_policy='ha-all ^(?!amq\\.).* {\"ha-mode\":\"exactly\",\"ha-params\":${nr_ha_queues}}'", - clone_params => 'ordered=true interleave=true', - meta_params => 'notify=true', - op_params => 'start timeout=200s stop timeout=200s', - require => Class['::rabbitmq'], + if $pacemaker_master { + include ::stdlib + # The default nr of ha queues is ceiling(N/2) + if $user_ha_queues == 0 { + $nr_rabbit_nodes = size($rabbit_nodes) + $nr_ha_queues = $nr_rabbit_nodes / 2 + ($nr_rabbit_nodes % 2) + } else { + $nr_ha_queues = $user_ha_queues + } + pacemaker::resource::ocf { 'rabbitmq': + ocf_agent_name => 'heartbeat:rabbitmq-cluster', + resource_params => "set_policy='ha-all ^(?!amq\\.).* {\"ha-mode\":\"exactly\",\"ha-params\":${nr_ha_queues}}'", + clone_params => 'ordered=true interleave=true', + meta_params => 'notify=true', + op_params => 'start timeout=200s stop timeout=200s', + tries => $pcs_tries, + location_rule => { + resource_discovery => 'exclusive', + score => 0, + expression => ['rabbitmq-role eq true'], + }, + require => [Class['::rabbitmq'], + Pacemaker::Property['rabbitmq-role-node-property']], + } } } } |