diff options
42 files changed, 485 insertions, 53 deletions
@@ -102,7 +102,7 @@ function build_tar() sudo rm -rf compass_dists mkdir -p compass_dists sudo cp -f *.tar *.iso compass_dists - sudo cp $COMPASS_PATH/build/build.yaml compass_dists + sudo cp $COMPASS_PATH/build/build*.yaml compass_dists sudo cp -rf $COMPASS_PATH/util/docker-compose ./ sudo tar -zcf compass.tar.gz docker-compose compass_dists sudo mv compass.tar.gz $TAR_DIR/$TAR_NAME @@ -142,3 +142,4 @@ process_param $* prepare_env download_packages build_tar + diff --git a/build/build-aarch64.yaml b/build/build-aarch64.yaml index 47bd0eb1..e197f33b 100644 --- a/build/build-aarch64.yaml +++ b/build/build-aarch64.yaml @@ -13,29 +13,25 @@ packages: - name: compass-deck description: "RESTful API and DB Handlers for Compass" get_method: docker - url: $COMPASS_DOCKER_REPO/compass-deck:latest + url: linaro/compass-deck:latest - - name: compass-tasks-k8s + # XXX: only supports k8s now + - name: compass-tasks description: "compass task container for kubernets deployment via kargo" get_method: docker - url: $COMPASS_DOCKER_REPO/compass-tasks-k8s:latest + url: linaro/compass-tasks-k8s:latest - name: compass-cobbler description: "cobbler container for compass" get_method: docker - url: $COMPASS_DOCKER_REPO/compass-cobbler:latest + url: linaro/compass-cobbler:latest - name: compass-db description: "datebase container for compass" get_method: docker - url: $COMPASS_DOCKER_REPO/compass-db:latest + url: linaro/compass-db:latest - name: compass-mq description: "message queue container for compass" get_method: docker - url: $COMPASS_DOCKER_REPO/compass-mq:latest - - - name: compass-docker-compose - description: "containerized compass compose ansible" - get_method: git - url: $COMPASS_GIT_REPO/compass-docker-compose + url: linaro/compass-mq:latest diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml index eb80066e..bfdc8958 100755 --- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml +++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml @@ -25,6 +25,12 @@ roles: - install-k8s-dependence +- hosts: ha + remote_user: root + max_fail_percentage: 0 + roles: + - ha + - hosts: localhost remote_user: root max_fail_percentage: 0 diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh new file mode 100644 index 00000000..62e79b3b --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +count=`ss -tnl | grep 6443 | wc -l` + +if [ $count = 0 ]; then + exit 1 +else + exit 0 +fi diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml new file mode 100644 index 00000000..03ed82ec --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- name: restart haproxy + service: name=haproxy state=restarted enabled=yes + +- name: restart keepalived + service: name=keepalived state=restarted enabled=yes diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml new file mode 100644 index 00000000..c7e58376 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml @@ -0,0 +1,83 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- include_vars: "{{ ansible_os_family }}.yml" + +- name: install keepalived haproxy + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: "{{ packages | union(packages_noarch) }}" + +- name: generate ha service list + lineinfile: dest=/opt/service create=yes line= '{{ item }}' + with_items: "{{ services | union(services_noarch) }}" + +- name: install pexpect + pip: name=pexpect state=present extra_args='--pre' + +- name: activate ip_nonlocal_bind + sysctl: name=net.ipv4.ip_nonlocal_bind value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_intvl + sysctl: name=net.ipv4.tcp_keepalive_intvl value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_probes + sysctl: name=net.ipv4.tcp_keepalive_probes value=5 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_time + sysctl: name=net.ipv4.tcp_keepalive_time value=5 + state=present reload=yes + +- name: update haproxy cfg + template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg + notify: restart haproxy + +- name: set haproxy enable flag + lineinfile: dest=/etc/default/haproxy state=present + regexp="ENABLED=*" + line="ENABLED=1" + notify: restart haproxy + when: ansible_os_family == "Debian" + +- name: set haproxy log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local0.* /var/log/haproxy.log" + line="local0.* /var/log/haproxy.log" + +- name: set rsyslog udp module + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$ModLoad imudp" + line="$ModLoad imudp" + +- name: set rsyslog udp port + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$UDPServerRun 514" + line="$UDPServerRun 514" + +- name: set keepalived start param + lineinfile: dest=/etc/default/keepalived state=present + regexp="^DAEMON_ARGS=*" + line="DAEMON_ARGS=\"-D -d -S 1\"" + when: ansible_os_family == "Debian" + +- name: set keepalived log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local1.* /var/log/keepalived.log" + line="local1.* /var/log/keepalived.log" + +- name: update keepalived info + template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf + notify: restart keepalived + +- name: restart rsyslog + shell: service rsyslog restart + +- meta: flush_handlers diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg new file mode 100644 index 00000000..5cd240c0 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg @@ -0,0 +1,48 @@ + +global + #chroot /var/run/haproxy + daemon + user haproxy + group haproxy + maxconn 4000 + pidfile /var/run/haproxy/haproxy.pid + #log 127.0.0.1 local0 + tune.bufsize 1000000 + stats socket /var/run/haproxy.sock + stats timeout 2m + +defaults + log global + maxconn 8000 + option redispatch + option dontlognull + option splice-auto + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 50s + timeout server 50s + timeout check 10s + retries 3 + +listen kubernetes-apiserver-https + bind {{ public_vip.ip }}:8383 + option ssl-hello-chk + mode tcp + option tcpka + option tcplog + timeout client 3h + timeout server 3h + balance roundrobin +{% for host,ip in haproxy_hosts.items() %} + server {{ host }} {{ ip }}:6443 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen stats + mode http + bind 0.0.0.0:9999 + stats enable + stats refresh 30s + stats uri / + stats realm Global\ statistics + stats auth admin:admin diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf new file mode 100644 index 00000000..c649bed5 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf @@ -0,0 +1,49 @@ +global_defs { + router_id {{ inventory_hostname }} +} + +vrrp_sync_group VG1 { + group { + internal_vip + public_vip + } +} + +vrrp_instance internal_vip { + interface {{ sys_intf_mappings.mgmt.interface }} + virtual_router_id {{ vrouter_id_internal }} + state BACKUP + nopreempt + advert_int 1 + priority {{ 50 + (host_index[inventory_hostname] * 50) }} + + authentication { + auth_type PASS + auth_pass 1234 + } + + + virtual_ipaddress { + {{ internal_vip.ip }}/{{ internal_vip.netmask }} dev {{ sys_intf_mappings.mgmt.interface }} + } +} + +vrrp_instance public_vip { + interface {{ sys_intf_mappings.external.interface }} + virtual_router_id {{ vrouter_id_public }} + state BACKUP + nopreempt + advert_int 1 + priority {{ 50 + (host_index[inventory_hostname] * 50) }} + + authentication { + auth_type PASS + auth_pass 4321 + } + + virtual_ipaddress { + {{ network_cfg.public_vip.ip }}/{{ network_cfg.public_vip.netmask }} dev {{ sys_intf_mappings.external.interface }} + } + +} + diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml new file mode 100644 index 00000000..b9f46bdf --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +services: [] +packages: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml new file mode 100644 index 00000000..b9f46bdf --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +services: [] +packages: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml new file mode 100644 index 00000000..77735d1e --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +packages_noarch: + - keepalived + - haproxy + +services_noarch: + - keepalived + - haproxy diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml index 6487e4ef..e683a3fe 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml @@ -9,6 +9,10 @@ --- - include_vars: "{{ ansible_os_family }}.yml" +- name: Install yum epel-release + command: yum -y install epel-release + when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7' + - name: Install yum packages yum: pkg: "{{ item }}" diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml index e016b855..8ced18b4 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml @@ -2,6 +2,7 @@ packages: - ubuntu-cloud-keyring - python-dev + - python-pip - openvswitch-switch - openvswitch-switch-dpdk - python-memcache diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml index 3ec18e7f..b7e1d3dc 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml @@ -1,6 +1,7 @@ --- packages: - python-devel + - python-pip - gcc - redhat-lsb-core - python-crypto diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml index 713b6b5f..7158325a 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml @@ -8,7 +8,6 @@ ############################################################################## --- packages_noarch: - - python-pip - ntp services_noarch: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 new file mode 100644 index 00000000..d998d4cb --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 @@ -0,0 +1,34 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = kubernetes +DNS.2 = kubernetes.default +DNS.3 = kubernetes.default.svc +DNS.4 = kubernetes.default.svc.{{ dns_domain }} +DNS.5 = localhost +{% for host in groups['kube-master'] %} +DNS.{{ 5 + loop.index }} = {{ host }} +{% endfor %} +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +{% set idx = groups['kube-master'] | length | int + 5 + 1 %} +DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} +{% endif %} +{% for host in groups['kube-master'] %} +IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +{% endfor %} +{% set idx = groups['kube-master'] | length | int * 2 + 1 %} +IP.{{ idx }} = {{ kube_apiserver_ip }} +IP.{{ idx + 1 }} = 127.0.0.1 +{% if supplementary_addresses_in_ssl_keys is defined %} +{% set is = idx + 1 %} +{% for addr in supplementary_addresses_in_ssl_keys %} +IP.{{ is + loop.index }} = {{ addr }} +{% endfor %} +{% endif %} diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml index 2763e53e..af52ad04 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml @@ -67,7 +67,7 @@ - name: copy inventoriy.json file copy: - src: /var/ansible/run/kubernetes-opnfv2/inventories/inventory.json + src: "{{ run_dir }}/inventories/inventory.json" dest: /tmp/inventory.json tags: - ansible @@ -96,6 +96,51 @@ regexp: '^helm_enabled:' line: 'helm_enabled: {{ helm_flag }}' +- name: enable external lb | set lb domain_nam + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^## apiserver_loadbalancer_domain_name:' + line: 'apiserver_loadbalancer_domain_name: {{ apiserver_loadbalancer_domain_name }}' + +- name: enable external lb | + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^#loadbalancer_apiserver:' + line: 'loadbalancer_apiserver:' + +- name: enable external lb | set vip address + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^# address: 1.2.3.4' + line: ' address: {{ vipaddress }}' + +- name: enable external lb | set vip port + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^# port: 1234' + line: ' port: {{ exlb_port }}' + +- name: enable internal lb + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^#loadbalancer_apiserver_localhost: true' + line: 'loadbalancer_apiserver_localhost: true' + +- name: add vip to ssl keys + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/k8s-cluster.yml + line: 'supplementary_addresses_in_ssl_keys: [{{ vipaddress }}]' + +- name: rm openssl file + file: + path: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 + state: absent + +- name: copy openssl.conf.j2 + copy: + src: openssl.conf.j2 + dest: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 + - name: copy overrided variables copy: src: "{{ item }}" diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml index 2d396d06..b73056e5 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml @@ -1,2 +1,5 @@ --- helm_flag: true +apiserver_loadbalancer_domain_name: "{{ public_vip.ip }}" +vipaddress: "{{ public_vip.ip }}" +exlb_port: 8383 diff --git a/deploy/adapters/ansible/roles/config-osa/templates/user_variables.yml.j2 b/deploy/adapters/ansible/roles/config-osa/templates/user_variables.yml.j2 index 130b5ad1..03e3a2af 100644 --- a/deploy/adapters/ansible/roles/config-osa/templates/user_variables.yml.j2 +++ b/deploy/adapters/ansible/roles/config-osa/templates/user_variables.yml.j2 @@ -45,6 +45,11 @@ neutron_plugin_type: ml2.ovs neutron_ml2_drivers_type: "local,flat,{{ tenant_net_info['type'] }}" +neutron_plugin_base: + - router + - metering + - trunk + neutron_provider_networks: network_flat_networks: "*" network_types: "{{ tenant_net_info['type'] }}" diff --git a/deploy/adapters/ansible/roles/post-openstack/tasks/main.yml b/deploy/adapters/ansible/roles/post-openstack/tasks/main.yml index 7022a4b5..fb0dc67d 100644 --- a/deploy/adapters/ansible/roles/post-openstack/tasks/main.yml +++ b/deploy/adapters/ansible/roles/post-openstack/tasks/main.yml @@ -40,6 +40,7 @@ --subnet-range "{{ public_net_info.floating_ip_cidr }}" \ "{{ public_net_info.subnet }}" when: + - public_net_info.enable == "True" - inventory_hostname == groups['utility_all'][0] - name: create openstack flavors diff --git a/deploy/adapters/ansible/roles/setup-host/tasks/main.yml b/deploy/adapters/ansible/roles/setup-host/tasks/main.yml index 4eba3d00..0a63f7f5 100644 --- a/deploy/adapters/ansible/roles/setup-host/tasks/main.yml +++ b/deploy/adapters/ansible/roles/setup-host/tasks/main.yml @@ -10,7 +10,7 @@ --- - name: openstack-hosts-setup - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible openstack-hosts-setup.yml \ @@ -25,7 +25,7 @@ when: openstack_hosts_setup_result.stdout.find('Mark openstack-hosts-setup completed') == -1 - name: security-hardening - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible security-hardening.yml \ @@ -40,7 +40,7 @@ when: security_hardening_result.stdout.find('Mark security-hardening completed') == -1 - name: lxc-hosts-setup - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible lxc-hosts-setup.yml \ @@ -55,7 +55,7 @@ when: lxc_hosts_setup_result.stdout.find('Mark lxc-hosts-setup completed') == -1 - name: lxc-containers-create - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible lxc-containers-create.yml \ @@ -66,7 +66,7 @@ register: failed_container - name: destroy the failed_container - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible lxc-containers-destroy.yml \ @@ -77,7 +77,7 @@ ignore_errors: "True" - name: retry to setup failed_container - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible lxc-containers-create.yml --limit {{item}} \ diff --git a/deploy/adapters/ansible/roles/setup-infrastructure/tasks/main.yml b/deploy/adapters/ansible/roles/setup-infrastructure/tasks/main.yml index 7cf5c86f..4e3a926f 100644 --- a/deploy/adapters/ansible/roles/setup-infrastructure/tasks/main.yml +++ b/deploy/adapters/ansible/roles/setup-infrastructure/tasks/main.yml @@ -8,7 +8,7 @@ ############################################################################## --- - name: setup infrastructure - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible setup-infrastructure.yml \ diff --git a/deploy/adapters/ansible/roles/setup-openstack/tasks/main.yml b/deploy/adapters/ansible/roles/setup-openstack/tasks/main.yml index a6ecb82f..c572936d 100644 --- a/deploy/adapters/ansible/roles/setup-openstack/tasks/main.yml +++ b/deploy/adapters/ansible/roles/setup-openstack/tasks/main.yml @@ -8,7 +8,7 @@ ############################################################################## --- - name: setup openstack - shell: "export ANSIBLE_LOG_PATH=/var/ansible/run/openstack_pike-opnfv2/ansible.log; \ + shell: "export ANSIBLE_LOG_PATH={{ run_dir }}/ansible.log; \ export ANSIBLE_SCP_IF_SSH=y; \ cd /opt/openstack-ansible/playbooks; \ openstack-ansible setup-openstack.yml \ diff --git a/deploy/adapters/cobbler/snippets/kickstart_sysctl.conf b/deploy/adapters/cobbler/snippets/kickstart_sysctl.conf index c227ecfa..112f010d 100644 --- a/deploy/adapters/cobbler/snippets/kickstart_sysctl.conf +++ b/deploy/adapters/cobbler/snippets/kickstart_sysctl.conf @@ -59,7 +59,6 @@ net.ipv4.ip_local_port_range = 15000 61000 net.ipv4.tcp_fin_timeout=30 # fast cycling of sockets in time_wait state and re-using them -net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_tw_reuse = 1 # increase the maximum number of requests queued to a listen socket diff --git a/deploy/adapters/cobbler/snippets/preseed_sysctl.conf b/deploy/adapters/cobbler/snippets/preseed_sysctl.conf index c227ecfa..112f010d 100644 --- a/deploy/adapters/cobbler/snippets/preseed_sysctl.conf +++ b/deploy/adapters/cobbler/snippets/preseed_sysctl.conf @@ -59,7 +59,6 @@ net.ipv4.ip_local_port_range = 15000 61000 net.ipv4.tcp_fin_timeout=30 # fast cycling of sockets in time_wait state and re-using them -net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_tw_reuse = 1 # increase the maximum number of requests queued to a listen socket diff --git a/deploy/adapters/cobbler/snippets/sysctl.xml b/deploy/adapters/cobbler/snippets/sysctl.xml index fe13bf7f..42f95976 100644 --- a/deploy/adapters/cobbler/snippets/sysctl.xml +++ b/deploy/adapters/cobbler/snippets/sysctl.xml @@ -62,7 +62,6 @@ net.ipv4.ip_local_port_range = 15000 61000 net.ipv4.tcp_fin_timeout=30 # fast cycling of sockets in time_wait state and re-using them -net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_tw_reuse = 1 # increase the maximum number of requests queued to a listen socket diff --git a/deploy/client.py b/deploy/client.py index 910aa7f8..891e632c 100644 --- a/deploy/client.py +++ b/deploy/client.py @@ -399,17 +399,20 @@ class CompassClient(object): except: raise RuntimeError('subnet %s format is invalid' % subnet) - if CONF.expansion == "false": + subnet_exist = False + for subnet_in_db in subnets_in_db: + if subnet == subnet_in_db['subnet']: + subnet_mapping[subnet] = subnet_in_db['id'] + subnet_exist = True + break + + if not subnet_exist: status, resp = self.client.add_subnet(subnet) LOG.info('add subnet %s status %s response %s', subnet, status, resp) if not self.is_ok(status): raise RuntimeError('failed to add subnet %s' % subnet) subnet_mapping[resp['subnet']] = resp['id'] - else: - for subnet_in_db in subnets_in_db: - if subnet == subnet_in_db['subnet']: - subnet_mapping[subnet] = subnet_in_db['id'] self.subnet_mapping = subnet_mapping @@ -475,8 +478,8 @@ class CompassClient(object): if host['hostname'] in hostnames: self.host_mapping[host['hostname']] = host['id'] - if CONF.expansion == "false": - assert(len(self.host_mapping) == len(machines)) + # if CONF.expansion == "false": + # assert(len(self.host_mapping) == len(machines)) def set_cluster_os_config(self, cluster_id): """set cluster os config.""" diff --git a/deploy/compass_conf/flavor/kubernetes.conf b/deploy/compass_conf/flavor/kubernetes.conf index 35c43155..71acadff 100755 --- a/deploy/compass_conf/flavor/kubernetes.conf +++ b/deploy/compass_conf/flavor/kubernetes.conf @@ -4,7 +4,7 @@ FLAVORS = [{ 'display_name': 'ansible-kubernetes', 'template': 'ansible-kubernetes.tmpl', 'roles': [ - 'kube_master', 'etcd', 'kube_node' + 'kube_master', 'etcd', 'kube_node', 'ha' ], }] diff --git a/deploy/compass_conf/package_installer/ansible-kubernetes.conf b/deploy/compass_conf/package_installer/ansible-kubernetes.conf index 32590c82..820691b7 100755 --- a/deploy/compass_conf/package_installer/ansible-kubernetes.conf +++ b/deploy/compass_conf/package_installer/ansible-kubernetes.conf @@ -7,7 +7,7 @@ SETTINGS = { 'playbook_file': 'site.yml', 'inventory_file': 'inventory.py', 'inventory_json_file': 'inventory.json', - 'inventory_group': ['kube_master', 'etcd', 'kube_node'], + 'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha'], 'group_variable': 'all', 'etc_hosts_path': 'roles/pre-k8s/templates/hosts', 'runner_dirs': ['roles','kubernetes/roles'] diff --git a/deploy/compass_conf/role/kubernetes_ansible.conf b/deploy/compass_conf/role/kubernetes_ansible.conf index ae096f47..c27779ad 100755 --- a/deploy/compass_conf/role/kubernetes_ansible.conf +++ b/deploy/compass_conf/role/kubernetes_ansible.conf @@ -11,5 +11,10 @@ ROLES = [{ 'role': 'kube_node', 'display_name': 'kube node', 'description': 'kube Node' -} +}, { + 'role': 'ha', + 'display_name': 'ha', + 'description': 'ha' +} + ] diff --git a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl index 440bf7d7..f132365a 100644 --- a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl +++ b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl @@ -23,6 +23,8 @@ #set kube_masters = $getVar('kube_master', []) #set kube_nodes = $getVar('kube_node', []) +run_dir: $getVar('run_dir', '') + enable_secgroup: $getVar('enable_secgroup', True) enable_fwaas: $getVar('enable_fwaas', True) enable_vpnaas: $getVar('enable_vpnaas', True) @@ -82,7 +84,7 @@ dashboard_host: "{{ internal_ip }}" haproxy_hosts: #for $item in $has #set $hostname=$item["hostname"] - $hostname: $ip_settings[$hostname]["mgmt"]["ip"] + $hostname: $ip_settings[$hostname]["external"]["ip"] #end for host_index: diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh index 7689f41c..cf215f3b 100755 --- a/deploy/compass_vm.sh +++ b/deploy/compass_vm.sh @@ -10,6 +10,26 @@ compass_vm_dir=$WORK_DIR/vm/compass rsa_file=$compass_vm_dir/boot.rsa ssh_args="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i $rsa_file" + +function check_container_alive() { + docker exec -it compass-deck bash -c "exit" 1>/dev/null 2>&1 + local deck_state=$? + docker exec -it compass-tasks bash -c "exit" 1>/dev/null 2>&1 + local tasks_state=$? + docker exec -it compass-cobbler bash -c "exit" 1>/dev/null 2>&1 + local cobbler_state=$? + docker exec -it compass-db bash -c "exit" 1>/dev/null 2>&1 + local db_state=$? + docker exec -it compass-mq bash -c "exit" 1>/dev/null 2>&1 + local mq_state=$? + + if [ $((deck_state||tasks_state||cobbler_state||db_state||mq-state)) == 0 ]; then + echo "true" + else + echo "false" + fi +} + function tear_down_compass() { sudo virsh destroy compass > /dev/null 2>&1 sudo virsh undefine compass > /dev/null 2>&1 diff --git a/deploy/conf/base.conf b/deploy/conf/base.conf index c2bf5291..5395405d 100644 --- a/deploy/conf/base.conf +++ b/deploy/conf/base.conf @@ -8,7 +8,6 @@ export EXT_NAT_GW=${EXT_NAT_GW:-192.16.1.1} export EXT_NAT_IP_START=${EXT_NAT_IP_START:-192.16.1.3} export EXT_NAT_IP_END=${EXT_NAT_IP_END:-192.16.1.254} export EXTERNAL_NIC=${EXTERNAL_NIC:-eth0} -export CLUSTER_NAME="opnfv2" export DOMAIN="ods.com" export PARTITIONS="/=30%,/home=5%,/tmp=5%,/var=60%" export SUBNETS="10.1.0.0/24,172.16.2.0/24,172.16.3.0/24,172.16.4.0/24" diff --git a/deploy/conf/hardware_environment/huawei-pod1/k8-nosdn-nofeature-ha.yml b/deploy/conf/hardware_environment/huawei-pod1/k8-nosdn-nofeature-ha.yml index 995d0107..2cedcf4d 100644 --- a/deploy/conf/hardware_environment/huawei-pod1/k8-nosdn-nofeature-ha.yml +++ b/deploy/conf/hardware_environment/huawei-pod1/k8-nosdn-nofeature-ha.yml @@ -25,6 +25,7 @@ hosts: roles: - kube_master - etcd + - ha - name: host2 mac: 'D8:49:0B:DA:5A:B7' @@ -35,6 +36,7 @@ hosts: roles: - kube_master - etcd + - ha - name: host3 mac: '78:D7:52:A0:B1:99' @@ -45,6 +47,7 @@ hosts: roles: - kube_master - etcd + - ha - name: host4 mac: 'D8:49:0B:DA:5B:5D' diff --git a/deploy/conf/vm_environment/k8-nosdn-nofeature-ha.yml b/deploy/conf/vm_environment/k8-nosdn-nofeature-ha.yml index 003f41be..42262057 100644 --- a/deploy/conf/vm_environment/k8-nosdn-nofeature-ha.yml +++ b/deploy/conf/vm_environment/k8-nosdn-nofeature-ha.yml @@ -16,16 +16,19 @@ hosts: roles: - kube_master - etcd + - ha - name: host2 roles: - kube_master - etcd + - ha - name: host3 roles: - kube_master - etcd + - ha - name: host4 roles: diff --git a/deploy/config_parse.py b/deploy/config_parse.py index 3d8dedc5..1575ca37 100644 --- a/deploy/config_parse.py +++ b/deploy/config_parse.py @@ -104,6 +104,7 @@ def export_dha_file(dha, dha_file, ofile): plugin_list.append(plugin_str) env.update({'plugins': ','.join(plugin_list)}) + env.update({'CLUSTER_NAME': dha.get('NAME', "opnfv")}) env.update({'TYPE': dha.get('TYPE', "virtual")}) env.update({'FLAVOR': dha.get('FLAVOR', "cluster")}) env.update({'HOSTNAMES': hostnames(dha, ',')}) diff --git a/deploy/launch.sh b/deploy/launch.sh index 6cbad1fa..98d9e4d6 100755 --- a/deploy/launch.sh +++ b/deploy/launch.sh @@ -55,7 +55,8 @@ if [[ "$EXPANSION" == "false" ]]; then export machines - if [[ "$DEPLOY_COMPASS" == "true" ]]; then + CONTAINER_ALIVE=$(check_container_alive) + if [[ "$DEPLOY_COMPASS" == "true" && "$CONTAINER_ALIVE" == "false" ]]; then if ! prepare_env;then echo "prepare_env failed" exit 1 @@ -71,7 +72,7 @@ if [[ "$EXPANSION" == "false" ]]; then log_error "launch_compass failed" exit 1 fi - else + elif [[ "$DEPLOY_COMPASS" == "true" && "$CONTAINER_ALIVE" == "true" ]]; then refresh_compass_core fi else diff --git a/deploy/status_callback.py b/deploy/status_callback.py index f0615f3f..6169b87f 100644 --- a/deploy/status_callback.py +++ b/deploy/status_callback.py @@ -8,7 +8,7 @@ ############################################################################## import httplib -import json +import simplejson as json import sys # noqa:F401 from ansible.plugins.callback import CallbackBase diff --git a/util/docker-compose/group_vars/all b/util/docker-compose/group_vars/all index 1dafe814..920589ce 100755 --- a/util/docker-compose/group_vars/all +++ b/util/docker-compose/group_vars/all @@ -19,7 +19,7 @@ install_ip_range: 10.1.0.100 10.1.0.250 pip_pkg_name: pip-openstack.tar.gz feature_pkg_name: packages.tar.gz -distros: +distros_x86_64: ubuntu-16.04.3: breed: "ubuntu" kickstart: "default16.seed" @@ -30,6 +30,8 @@ distros: version: "ubuntu" iso_name: ubuntu-16.04.3-server-amd64 ppa_name: xenial-pike-ppa + cobbler_name: ubuntu-16.04.3-server + cobbler_arch: x86_64 CentOS-7: breed: "redhat" @@ -38,4 +40,29 @@ distros: version: "centos" iso_name: CentOS-7-x86_64-Minimal-1708 ppa_name: centos7-pike-ppa + cobbler_name: CentOS-7-Minimal-1708 + cobbler_arch: x86_64 +distros_aarch64: + ubuntu-16.04: + breed: "ubuntu" + kickstart: "default16.seed" + kopts: "netcfg/choose_interface=auto console=ttyS0 console=ttyAMA0 console=tty0 net.ifnames=0" + kopts_post: "console=ttyS0 console=ttyAMA0 console=tty0 net.ifnames=0" + extra: "--apt-dists=xenial --apt-components=main" + aliase: "xenial" + version: "ubuntu" + iso_name: ubuntu-16.04-server-arm64 + ppa_name: xenial-pike-ppa + cobbler_name: ubuntu-16.04-server + cobbler_arch: arm + + CentOS-7: + breed: "redhat" + kickstart: "default-aarch64.ks" + aliase: "centos7" + version: "centos" + iso_name: CentOS-7-aarch64-Minimal-1611 + ppa_name: centos7-pike-ppa + cobbler_name: CentOS-7-Minimal-1611 + cobbler_arch: arm diff --git a/util/docker-compose/roles/cobbler/tasks/main.yml b/util/docker-compose/roles/cobbler/tasks/main.yml index e3f13088..0a30e0c5 100755 --- a/util/docker-compose/roles/cobbler/tasks/main.yml +++ b/util/docker-compose/roles/cobbler/tasks/main.yml @@ -44,23 +44,22 @@ docker cp "{{ compass_dists_dir }}/{{ item.value.ppa_name }}.tar.gz" compass-cobbler:/tmp docker exec compass-cobbler bash -c \ "mkdir -p /var/lib/cobbler/repo_mirror/{{ item.value.ppa_name }}" - with_dict: "{{ distros }}" + with_dict: "{{ vars['distros_' + ansible_architecture] }}" run_once: "True" ignore_errors: "True" - name: cobbler import iso shell: | - name=`basename {{ item.value.iso_name }} | \ - sed -e 's/.iso//g' -e 's/-amd64//g' -e 's/-x86_64//g'` + name="{{ item.value.cobbler_name }}" docker exec compass-cobbler bash -c \ "service cobblerd restart; sleep 3; cobbler import --name $name \ --path /mnt/{{ item.value.iso_name }} \ - --arch x86_64 \ + --arch {{ item.value.cobbler_arch }} \ --kickstart /var/lib/cobbler/kickstarts/{{ item.value.kickstart }} \ --breed {{ item.value.breed }}" - with_dict: "{{ distros }}" + with_dict: "{{ vars['distros_' + ansible_architecture] }}" run_once: "True" register: result2 until: result2.rc == 0 @@ -73,14 +72,14 @@ "cobbler repo add --name {{ item.value.ppa_name }} \ --mirror /var/lib/cobbler/repo_mirror/{{ item.value.ppa_name }} \ --mirror-locally True \ - --arch=x86_64 {{ item.value.extra | default()}}" - with_dict: "{{ distros }}" + --arch={{ item.value.cobbler_arch }} \ + {{ item.value.extra | default() }}" + with_dict: "{{ vars['distros_' + ansible_architecture] }}" run_once: "True" -- name: edit profiles +- name: edit profiles for x86_64 shell: | - name=`basename {{ item.value.iso_name }} | \ - sed -e 's/.iso//g' -e 's/-amd64//g' -e 's/-x86_64//g'`-x86_64 + name="{{ item.value.cobbler_name }}-x86_64" docker exec compass-cobbler bash -c \ "cobbler profile edit --name $name \ --name-servers '' \ @@ -88,8 +87,38 @@ --ksmeta \"tree=http://{{ host_ip }}/cobbler/ks_mirror/$name\" \ --kopts \"{{ item.value.kopts | default('') }}\" \ --kopts-post \"{{ item.value.kopts_post | default('') }}\"" - with_dict: "{{ distros }}" + with_dict: "{{ distros_x86_64 }}" run_once: "True" + when: ansible_architecture == "x86_64" + +- name: edit profile for aarch64 + shell: | + # Cobbler naming on Arm is a mess + # Imported name: + # - CentOS-7-Minimal-1611 + # - ubuntu-16.04-server + # Profile name: + # - CentOS-7-Minimal-1611-arm-arm + # - ubuntu-16.04-server-arm-arm64-arm + # Kickstart meta url name + # - http://x.x.x.x/cobbler/ks_mirror/CentOS-7-Minimal-1611-arm + # - http://x.x.x.x/cobbler/ks_mirror/ubuntu-16.04-server-arm-arm64 + name="{{ item.value.cobbler_name }}-arm" + profiles=`docker exec compass-cobbler bash -c \ + "cobbler profile list | grep $name"` + for profile_name in $profiles; do + docker exec compass-cobbler bash -c \ + "cobbler profile edit \ + --name $profile_name \ + --name-servers '' \ + --repos {{ item.value.ppa_name }} \ + --ksmeta \"tree=http://{{ host_ip }}/cobbler/ks_mirror/${profile_name%????}\" \ + --kopts \"{{ item.value.kopts | default('') }}\" \ + --kopts-post \"{{ item.value.kopts_post | default('') }}\"" + done + with_dict: "{{ distros_aarch64 }}" + run_once: "True" + when: ansible_architecture == "aarch64" - name: cobbler sync shell: | diff --git a/util/docker-compose/roles/cobbler/templates/dhcp.template.j2 b/util/docker-compose/roles/cobbler/templates/dhcp.template.j2 index 0665f2fe..8b1f2a61 100755 --- a/util/docker-compose/roles/cobbler/templates/dhcp.template.j2 +++ b/util/docker-compose/roles/cobbler/templates/dhcp.template.j2 @@ -42,6 +42,8 @@ subnet {{ install_subnet }} netmask {{ install_netmask }} { filename "grub/grub-x86.efi"; } else if option pxe-system-type = 00:07 { filename "grub/grub-x86_64.efi"; + } else if option pxe-system-type = 00:0B { + filename "grub/grub-aarch64.efi"; } else { filename "pxelinux.0"; } diff --git a/util/docker-compose/roles/compass/tasks/main.yml b/util/docker-compose/roles/compass/tasks/main.yml index 9ee7902e..c7dba96b 100755 --- a/util/docker-compose/roles/compass/tasks/main.yml +++ b/util/docker-compose/roles/compass/tasks/main.yml @@ -1,5 +1,8 @@ --- -- include_vars: "{{ compass_dists_dir }}/build.yaml" +- include_vars: "{{ item }}" + with_first_found: + - "{{ compass_dists_dir }}/build-{{ ansible_architecture }}.yaml" + - "{{ compass_dists_dir }}/build.yaml" - set_fact: image_name: "{{ item.name | regex_replace('-', '_') }}" |