summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xbuild.sh1
-rwxr-xr-xdeploy/adapters/ansible/kubernetes/ansible-kubernetes.yml6
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh9
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml14
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml83
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg48
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf49
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml11
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml11
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml16
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml4
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml1
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml1
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml1
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j234
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml45
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml3
-rwxr-xr-xdeploy/compass_conf/flavor/kubernetes.conf2
-rwxr-xr-xdeploy/compass_conf/package_installer/ansible-kubernetes.conf2
-rwxr-xr-xdeploy/compass_conf/role/kubernetes_ansible.conf7
-rw-r--r--deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl2
21 files changed, 345 insertions, 5 deletions
diff --git a/build.sh b/build.sh
index 0aea6eb6..462f76a6 100755
--- a/build.sh
+++ b/build.sh
@@ -142,3 +142,4 @@ process_param $*
prepare_env
download_packages
build_tar
+
diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
index eb80066e..bfdc8958 100755
--- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
+++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
@@ -25,6 +25,12 @@
roles:
- install-k8s-dependence
+- hosts: ha
+ remote_user: root
+ max_fail_percentage: 0
+ roles:
+ - ha
+
- hosts: localhost
remote_user: root
max_fail_percentage: 0
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh
new file mode 100644
index 00000000..62e79b3b
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+count=`ss -tnl | grep 6443 | wc -l`
+
+if [ $count = 0 ]; then
+ exit 1
+else
+ exit 0
+fi
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml
new file mode 100644
index 00000000..03ed82ec
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: restart haproxy
+ service: name=haproxy state=restarted enabled=yes
+
+- name: restart keepalived
+ service: name=keepalived state=restarted enabled=yes
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml
new file mode 100644
index 00000000..c7e58376
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml
@@ -0,0 +1,83 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- include_vars: "{{ ansible_os_family }}.yml"
+
+- name: install keepalived haproxy
+ action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+ with_items: "{{ packages | union(packages_noarch) }}"
+
+- name: generate ha service list
+ lineinfile: dest=/opt/service create=yes line= '{{ item }}'
+ with_items: "{{ services | union(services_noarch) }}"
+
+- name: install pexpect
+ pip: name=pexpect state=present extra_args='--pre'
+
+- name: activate ip_nonlocal_bind
+ sysctl: name=net.ipv4.ip_nonlocal_bind value=1
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_intvl
+ sysctl: name=net.ipv4.tcp_keepalive_intvl value=1
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_probes
+ sysctl: name=net.ipv4.tcp_keepalive_probes value=5
+ state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_time
+ sysctl: name=net.ipv4.tcp_keepalive_time value=5
+ state=present reload=yes
+
+- name: update haproxy cfg
+ template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg
+ notify: restart haproxy
+
+- name: set haproxy enable flag
+ lineinfile: dest=/etc/default/haproxy state=present
+ regexp="ENABLED=*"
+ line="ENABLED=1"
+ notify: restart haproxy
+ when: ansible_os_family == "Debian"
+
+- name: set haproxy log
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="local0.* /var/log/haproxy.log"
+ line="local0.* /var/log/haproxy.log"
+
+- name: set rsyslog udp module
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="^#$ModLoad imudp"
+ line="$ModLoad imudp"
+
+- name: set rsyslog udp port
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="^#$UDPServerRun 514"
+ line="$UDPServerRun 514"
+
+- name: set keepalived start param
+ lineinfile: dest=/etc/default/keepalived state=present
+ regexp="^DAEMON_ARGS=*"
+ line="DAEMON_ARGS=\"-D -d -S 1\""
+ when: ansible_os_family == "Debian"
+
+- name: set keepalived log
+ lineinfile: dest=/etc/rsyslog.conf state=present
+ regexp="local1.* /var/log/keepalived.log"
+ line="local1.* /var/log/keepalived.log"
+
+- name: update keepalived info
+ template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf
+ notify: restart keepalived
+
+- name: restart rsyslog
+ shell: service rsyslog restart
+
+- meta: flush_handlers
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg
new file mode 100644
index 00000000..5cd240c0
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg
@@ -0,0 +1,48 @@
+
+global
+ #chroot /var/run/haproxy
+ daemon
+ user haproxy
+ group haproxy
+ maxconn 4000
+ pidfile /var/run/haproxy/haproxy.pid
+ #log 127.0.0.1 local0
+ tune.bufsize 1000000
+ stats socket /var/run/haproxy.sock
+ stats timeout 2m
+
+defaults
+ log global
+ maxconn 8000
+ option redispatch
+ option dontlognull
+ option splice-auto
+ timeout http-request 10s
+ timeout queue 1m
+ timeout connect 10s
+ timeout client 50s
+ timeout server 50s
+ timeout check 10s
+ retries 3
+
+listen kubernetes-apiserver-https
+ bind {{ public_vip.ip }}:8383
+ option ssl-hello-chk
+ mode tcp
+ option tcpka
+ option tcplog
+ timeout client 3h
+ timeout server 3h
+ balance roundrobin
+{% for host,ip in haproxy_hosts.items() %}
+ server {{ host }} {{ ip }}:6443 weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen stats
+ mode http
+ bind 0.0.0.0:9999
+ stats enable
+ stats refresh 30s
+ stats uri /
+ stats realm Global\ statistics
+ stats auth admin:admin
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf
new file mode 100644
index 00000000..c649bed5
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf
@@ -0,0 +1,49 @@
+global_defs {
+ router_id {{ inventory_hostname }}
+}
+
+vrrp_sync_group VG1 {
+ group {
+ internal_vip
+ public_vip
+ }
+}
+
+vrrp_instance internal_vip {
+ interface {{ sys_intf_mappings.mgmt.interface }}
+ virtual_router_id {{ vrouter_id_internal }}
+ state BACKUP
+ nopreempt
+ advert_int 1
+ priority {{ 50 + (host_index[inventory_hostname] * 50) }}
+
+ authentication {
+ auth_type PASS
+ auth_pass 1234
+ }
+
+
+ virtual_ipaddress {
+ {{ internal_vip.ip }}/{{ internal_vip.netmask }} dev {{ sys_intf_mappings.mgmt.interface }}
+ }
+}
+
+vrrp_instance public_vip {
+ interface {{ sys_intf_mappings.external.interface }}
+ virtual_router_id {{ vrouter_id_public }}
+ state BACKUP
+ nopreempt
+ advert_int 1
+ priority {{ 50 + (host_index[inventory_hostname] * 50) }}
+
+ authentication {
+ auth_type PASS
+ auth_pass 4321
+ }
+
+ virtual_ipaddress {
+ {{ network_cfg.public_vip.ip }}/{{ network_cfg.public_vip.netmask }} dev {{ sys_intf_mappings.external.interface }}
+ }
+
+}
+
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml
new file mode 100644
index 00000000..b9f46bdf
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+services: []
+packages: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml
new file mode 100644
index 00000000..b9f46bdf
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+services: []
+packages: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml
new file mode 100644
index 00000000..77735d1e
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml
@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+packages_noarch:
+ - keepalived
+ - haproxy
+
+services_noarch:
+ - keepalived
+ - haproxy
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
index 6487e4ef..e683a3fe 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
@@ -9,6 +9,10 @@
---
- include_vars: "{{ ansible_os_family }}.yml"
+- name: Install yum epel-release
+ command: yum -y install epel-release
+ when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7'
+
- name: Install yum packages
yum:
pkg: "{{ item }}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
index e016b855..8ced18b4 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
@@ -2,6 +2,7 @@
packages:
- ubuntu-cloud-keyring
- python-dev
+ - python-pip
- openvswitch-switch
- openvswitch-switch-dpdk
- python-memcache
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
index 3ec18e7f..b7e1d3dc 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
@@ -1,6 +1,7 @@
---
packages:
- python-devel
+ - python-pip
- gcc
- redhat-lsb-core
- python-crypto
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
index 713b6b5f..7158325a 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
@@ -8,7 +8,6 @@
##############################################################################
---
packages_noarch:
- - python-pip
- ntp
services_noarch: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2
new file mode 100644
index 00000000..d998d4cb
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2
@@ -0,0 +1,34 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = kubernetes
+DNS.2 = kubernetes.default
+DNS.3 = kubernetes.default.svc
+DNS.4 = kubernetes.default.svc.{{ dns_domain }}
+DNS.5 = localhost
+{% for host in groups['kube-master'] %}
+DNS.{{ 5 + loop.index }} = {{ host }}
+{% endfor %}
+{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+{% set idx = groups['kube-master'] | length | int + 5 + 1 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['kube-master'] %}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx = groups['kube-master'] | length | int * 2 + 1 %}
+IP.{{ idx }} = {{ kube_apiserver_ip }}
+IP.{{ idx + 1 }} = 127.0.0.1
+{% if supplementary_addresses_in_ssl_keys is defined %}
+{% set is = idx + 1 %}
+{% for addr in supplementary_addresses_in_ssl_keys %}
+IP.{{ is + loop.index }} = {{ addr }}
+{% endfor %}
+{% endif %}
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
index 2763e53e..8a78c68d 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
@@ -96,6 +96,51 @@
regexp: '^helm_enabled:'
line: 'helm_enabled: {{ helm_flag }}'
+- name: enable external lb | set lb domain_nam
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+ regexp: '^## apiserver_loadbalancer_domain_name:'
+ line: 'apiserver_loadbalancer_domain_name: {{ apiserver_loadbalancer_domain_name }}'
+
+- name: enable external lb |
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+ regexp: '^#loadbalancer_apiserver:'
+ line: 'loadbalancer_apiserver:'
+
+- name: enable external lb | set vip address
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+ regexp: '^# address: 1.2.3.4'
+ line: ' address: {{ vipaddress }}'
+
+- name: enable external lb | set vip port
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+ regexp: '^# port: 1234'
+ line: ' port: {{ exlb_port }}'
+
+- name: enable internal lb
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+ regexp: '^#loadbalancer_apiserver_localhost: true'
+ line: 'loadbalancer_apiserver_localhost: true'
+
+- name: add vip to ssl keys
+ lineinfile:
+ dest: /opt/kargo_k8s/inventory/group_vars/k8s-cluster.yml
+ line: 'supplementary_addresses_in_ssl_keys: [{{ vipaddress }}]'
+
+- name: rm openssl file
+ file:
+ path: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2
+ state: absent
+
+- name: copy openssl.conf.j2
+ copy:
+ src: openssl.conf.j2
+ dest: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2
+
- name: copy overrided variables
copy:
src: "{{ item }}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
index 2d396d06..b73056e5 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
@@ -1,2 +1,5 @@
---
helm_flag: true
+apiserver_loadbalancer_domain_name: "{{ public_vip.ip }}"
+vipaddress: "{{ public_vip.ip }}"
+exlb_port: 8383
diff --git a/deploy/compass_conf/flavor/kubernetes.conf b/deploy/compass_conf/flavor/kubernetes.conf
index 35c43155..71acadff 100755
--- a/deploy/compass_conf/flavor/kubernetes.conf
+++ b/deploy/compass_conf/flavor/kubernetes.conf
@@ -4,7 +4,7 @@ FLAVORS = [{
'display_name': 'ansible-kubernetes',
'template': 'ansible-kubernetes.tmpl',
'roles': [
- 'kube_master', 'etcd', 'kube_node'
+ 'kube_master', 'etcd', 'kube_node', 'ha'
],
}]
diff --git a/deploy/compass_conf/package_installer/ansible-kubernetes.conf b/deploy/compass_conf/package_installer/ansible-kubernetes.conf
index 32590c82..820691b7 100755
--- a/deploy/compass_conf/package_installer/ansible-kubernetes.conf
+++ b/deploy/compass_conf/package_installer/ansible-kubernetes.conf
@@ -7,7 +7,7 @@ SETTINGS = {
'playbook_file': 'site.yml',
'inventory_file': 'inventory.py',
'inventory_json_file': 'inventory.json',
- 'inventory_group': ['kube_master', 'etcd', 'kube_node'],
+ 'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha'],
'group_variable': 'all',
'etc_hosts_path': 'roles/pre-k8s/templates/hosts',
'runner_dirs': ['roles','kubernetes/roles']
diff --git a/deploy/compass_conf/role/kubernetes_ansible.conf b/deploy/compass_conf/role/kubernetes_ansible.conf
index ae096f47..c27779ad 100755
--- a/deploy/compass_conf/role/kubernetes_ansible.conf
+++ b/deploy/compass_conf/role/kubernetes_ansible.conf
@@ -11,5 +11,10 @@ ROLES = [{
'role': 'kube_node',
'display_name': 'kube node',
'description': 'kube Node'
-}
+}, {
+ 'role': 'ha',
+ 'display_name': 'ha',
+ 'description': 'ha'
+}
+
]
diff --git a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
index 440bf7d7..ff4d587c 100644
--- a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
+++ b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
@@ -82,7 +82,7 @@ dashboard_host: "{{ internal_ip }}"
haproxy_hosts:
#for $item in $has
#set $hostname=$item["hostname"]
- $hostname: $ip_settings[$hostname]["mgmt"]["ip"]
+ $hostname: $ip_settings[$hostname]["external"]["ip"]
#end for
host_index: