diff options
Diffstat (limited to 'xci/infra/bifrost')
| -rw-r--r-- | xci/infra/bifrost/README.md | 53 | ||||
| -rw-r--r-- | xci/infra/bifrost/infra-provision.sh | 25 | ||||
| -rw-r--r-- | xci/infra/bifrost/playbooks/bootstrap-bifrost.yml | 42 | ||||
| -rw-r--r-- | xci/infra/bifrost/playbooks/opnfv-virtual.yaml | 123 | ||||
| -rwxr-xr-x | xci/infra/bifrost/scripts/bifrost-provision.sh | 158 | ||||
| -rwxr-xr-x | xci/infra/bifrost/scripts/destroy-env.sh | 78 |
6 files changed, 479 insertions, 0 deletions
diff --git a/xci/infra/bifrost/README.md b/xci/infra/bifrost/README.md new file mode 100644 index 00000000..60d07244 --- /dev/null +++ b/xci/infra/bifrost/README.md @@ -0,0 +1,53 @@ +===================== +How to deploy bifrost +===================== +The scripts and playbooks defined on this repo, need to be combined with proper `Bifrost <http://git.openstack.org/cgit/openstack/bifrost>`_ code. + +Please follow that steps: + +1. Clone bifrost:: + + sudo git clone https://git.openstack.org/openstack/bifrost /opt/bifrost + +2. Clone releng:: + + sudo git clone https://gerrit.opnfv.org/gerrit/releng-xci /opt/releng-xci + +3. Clone infracloud:: + + sudo git clone https://git.openstack.org/openstack-infra/puppet-infracloud /opt/puppet-infracloud + +4. Combine releng scripts and playbooks with bifrost:: + + sudo cp -R /opt/releng-xci/bifrost/* /opt/bifrost/ + +5. Copy /opt/puppet-infracloud/templates/bifrost/create_bridge.py.erb to /opt/puppet-infracloud/files/elements/infra-cloud-bridge/static/opt/create_bridge.py, + and replace tag <%= @bridge_name -%> with br_opnfv + +6. If you are on a RHEL/CentOS box, ensure that selinux is disabled + +7. Run destroy script if you need to cleanup previous environment:: + + cd /opt/bifrost + sudo ./scripts/destroy-env.sh + +8. Run deployment script to spin up 3 vms with bifrost: xcimaster, controller and compute:: + + cd /opt/bifrost + sudo ./scripts/test-bifrost-deployment.sh + +It is likely that the script will show some errors due to timeout. Please ignore the errors, and wait until the vms are completely bootstrapped. To verify it you can check with ironic:: + + cd /opt/bifrost + source env-vars + ironic node-list + +And wait until all the vms are in **active** Provisioning State. + +9. Check the IPs assigned to each of the VMS. You can check it by looking at inventory: + + cat /tmp/baremetal.csv + +10. You can enter into the vms with devuser login/pass: + + ssh devuser@192.168.122.2 diff --git a/xci/infra/bifrost/infra-provision.sh b/xci/infra/bifrost/infra-provision.sh new file mode 100644 index 00000000..f847e66d --- /dev/null +++ b/xci/infra/bifrost/infra-provision.sh @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------- +# Start provisioning VM nodes +#------------------------------------------------------------------------------- +# This playbook +# - removes directories that were created by the previous xci run +# - clones opnfv/releng-xci and openstack/bifrost repositories +# - combines opnfv/releng-xci and openstack/bifrost scripts/playbooks +# - destroys VMs, removes ironic db, leases, logs +# - creates and provisions VMs for the chosen flavor +#------------------------------------------------------------------------------- +BIFROST_ROOT_DIR="$(dirname $(realpath ${BASH_SOURCE[0]}))" + +echo "Info: Starting provisining VM nodes using openstack/bifrost" +echo "-------------------------------------------------------------------------" +# We are using sudo so we need to make sure that env_reset is not present +sudo sed -i "s/^Defaults.*env_reset/#&/" /etc/sudoers +cd $BIFROST_ROOT_DIR +sudo -E bash ./scripts/destroy-env.sh +cd $BIFROST_ROOT_DIR/playbooks/ +ansible-playbook ${XCI_ANSIBLE_PARAMS} -i "localhost," bootstrap-bifrost.yml +cd ${XCI_CACHE}/repos/bifrost +bash ./scripts/bifrost-provision.sh +echo "-----------------------------------------------------------------------" +echo "Info: VM nodes are provisioned!" +echo "-----------------------------------------------------------------------" diff --git a/xci/infra/bifrost/playbooks/bootstrap-bifrost.yml b/xci/infra/bifrost/playbooks/bootstrap-bifrost.yml new file mode 100644 index 00000000..23eb976c --- /dev/null +++ b/xci/infra/bifrost/playbooks/bootstrap-bifrost.yml @@ -0,0 +1,42 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2017 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +- hosts: localhost + connection: local + gather_facts: true + vars_files: + - "{{ XCI_PATH }}/xci/var/opnfv.yml" + pre_tasks: + - name: Load distribution variables + include_vars: + file: "{{ XCI_PATH }}/xci/var/{{ ansible_os_family }}.yml" + roles: + - role: clone-repository + project: "opnfv/bifrost" + repo: "{{ OPENSTACK_BIFROST_GIT_URL }}" + dest: "{{ XCI_CACHE }}/repos/bifrost" + version: "{{ OPENSTACK_BIFROST_VERSION }}" + + tasks: + - name: Load distribution variables + include_vars: + file: "{{ XCI_PATH }}/xci/var/{{ ansible_os_family }}.yml" + - name: Synchronize local development bifrost repository to XCI paths + # command module is much faster than the copy module + synchronize: + src: "{{ OPENSTACK_BIFROST_DEV_PATH }}" + dest: "{{ XCI_CACHE }}/repos/bifrost" + recursive: yes + delete: yes + when: + - OPENSTACK_BIFROST_DEV_PATH != "" + - name: combine opnfv/releng-xci and openstack/bifrost scripts/playbooks + copy: + src: "{{ XCI_PATH}}/xci/infra/bifrost/" + dest: "{{ XCI_CACHE }}/repos/bifrost" diff --git a/xci/infra/bifrost/playbooks/opnfv-virtual.yaml b/xci/infra/bifrost/playbooks/opnfv-virtual.yaml new file mode 100644 index 00000000..f44adb4d --- /dev/null +++ b/xci/infra/bifrost/playbooks/opnfv-virtual.yaml @@ -0,0 +1,123 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 RedHat and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- hosts: localhost + connection: local + name: "Host and Ironic bootstrapping" + become: yes + gather_facts: yes + pre_tasks: + - name: Remove pre-existing leases file + file: path=/var/lib/misc/dnsmasq.leases state=absent + - name: "Override the ipv4_gateway setting" + set_fact: + ipv4_gateway: "192.168.122.1" + - name: Prepare the XCI pre-built images + block: + - name: Create the PXE directory + file: + path: /httpboot + state: directory + - name: Download the {{ xci_distro }} image checksum file + get_url: + dest: /httpboot/deployment_image.qcow2.sha256.txt + force: no + url: http://artifacts.opnfv.org/releng/xci/images/{{ xci_distro }}.qcow2.sha256.txt + timeout: 3000 + - name: Extract checksum + shell: awk '{print $1}' /httpboot/deployment_image.qcow2.sha256.txt + register: _image_checksum + - fail: + msg: "Failed to get image checksum" + when: _image_checksum == '' + - set_fact: + image_checksum: "{{ _image_checksum.stdout }}" + - name: Download the {{ xci_distro }} image file + get_url: + url: http://artifacts.opnfv.org/releng/xci/images/{{ xci_distro }}.qcow2 + checksum: "sha256:{{ image_checksum }}" + timeout: 3000 + dest: /httpboot/deployment_image.qcow2 + force: no + - name: Set correct mode for {{ xci_distro }}.qcow2 file + file: + path: /httpboot/deployment_image.qcow2 + mode: '0755' + owner: 'root' + group: 'root' + when: use_prebuilt_images | bool == true + + roles: + - role: bifrost-prep-for-install + when: skip_install is not defined + - role: bifrost-keystone-install + - role: bifrost-ironic-install + cleaning: false + testing: true + # NOTE(TheJulia): While the next step creates a ramdisk, some elements + # do not support ramdisk-image-create as they invoke steps to cleanup + # the ramdisk which causes ramdisk-image-create to believe it failed. + - role: bifrost-create-dib-image + dib_imagename: "{{ http_boot_folder }}/ipa" + build_ramdisk: false + dib_os_element: "{{ ipa_dib_os_element|default('debian') }}" + dib_os_release: "jessie" + dib_elements: "ironic-agent {{ ipa_extra_dib_elements | default('') }}" + dib_notmpfs: true + when: + - create_ipa_image | bool == true + - not use_prebuilt_images | bool == false + - role: bifrost-create-dib-image + dib_imagetype: "qcow2" + dib_imagename: "{{deploy_image}}" + dib_env_vars: + DIB_PYTHON_VERSION: 2 + dib_os_element: "{{ lookup('env','DIB_OS_ELEMENT') }}" + dib_os_release: "{{ lookup('env', 'DIB_OS_RELEASE') }}" + extra_dib_elements: "{{ lookup('env', 'EXTRA_DIB_ELEMENTS') | default('') }}" + dib_elements: "vm enable-serial-console simple-init devuser openssh-server growroot pip-and-virtualenv {{ extra_dib_elements }}" + dib_packages: "{{ lookup('env', 'DIB_OS_PACKAGES') }}" + dib_notmpfs: true + when: + - create_image_via_dib | bool == true + - transform_boot_image | bool == false + - use_prebuilt_images | bool == false + - role: bifrost-keystone-client-config + user: "{{ ansible_env.SUDO_USER }}" + clouds: + bifrost: + config_username: "{{ ironic.keystone.default_username }}" + config_password: "{{ ironic.keystone.default_password }}" + config_project_name: "baremetal" + config_region_name: "{{ keystone.bootstrap.region_name }}" + config_auth_url: "{{ keystone.bootstrap.public_url }}" + environment: + http_proxy: "{{ lookup('env','http_proxy') }}" + https_proxy: "{{ lookup('env','https_proxy') }}" + +- hosts: baremetal + name: "Enrollment and Deployment" + vars: + multinode_testing: "{{ inventory_dhcp | bool == true }}" + become: no + connection: local + gather_facts: yes + roles: + - role: ironic-enroll-dynamic + - { role: ironic-inspect-node, when: inspect_nodes | default('false') | bool == true } + - role: bifrost-configdrives-dynamic + - role: bifrost-deploy-nodes-dynamic + +- hosts: baremetal + name: "Deploy machines." + become: no + connection: local + serial: 1 + roles: + - role: bifrost-prepare-for-test-dynamic diff --git a/xci/infra/bifrost/scripts/bifrost-provision.sh b/xci/infra/bifrost/scripts/bifrost-provision.sh new file mode 100755 index 00000000..939acb58 --- /dev/null +++ b/xci/infra/bifrost/scripts/bifrost-provision.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -eu +set -o pipefail + +export PYTHONUNBUFFERED=1 +SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)" +BIFROST_HOME=$SCRIPT_HOME/.. +ENABLE_VENV="false" +PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600} +# This is normally exported by XCI env but we should initialize it here +# in case we run this script on its own for debug purposes +XCI_ANSIBLE_PARAMS=${XCI_ANSIBLE_PARAMS:-} +# Ironic SHAs +BIFROST_IRONIC_INSPECTOR_VERSION=${BIFROST_IRONIC_INSPECTOR_VERSION:-master} +BIFROST_IRONIC_INSPECTOR_CLIENT_VERSION=${BIFROST_IRONIC_INSPECTOR_CLIENT_VERSION:-master} +BIFROST_IRONIC_CLIENT_VERSION=${BIFROST_IRONIC_CLIENT_VERSION:-master} +BIFROST_IRONIC_VERSION=${BIFROST_IRONIC_VERSION:-master} + +# set UPPER_CONSTRAINTS_FILE since it is needed in order to limit libvirt-python to 4.0.0 +export UPPER_CONSTRAINTS_FILE=https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt + +# Ensure the right inventory files is used based on branch +CURRENT_BIFROST_BRANCH=$(git rev-parse --abbrev-ref HEAD) +if [ $CURRENT_BIFROST_BRANCH = "master" ]; then + BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} + INVENTORY_FILE_FORMAT="baremetal_json_file" +else + BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.csv'} + INVENTORY_FILE_FORMAT="baremetal_csv_file" +fi +export BIFROST_INVENTORY_SOURCE=$BAREMETAL_DATA_FILE + +# Default settings for VMs +export TEST_VM_NUM_NODES=${TEST_VM_NUM_NODES:-3} +export TEST_VM_NODE_NAMES=${TEST_VM_NODE_NAMES:-"opnfv controller00 compute00"} +export VM_DOMAIN_TYPE=${VM_DOMAIN_TYPE:-kvm} +export VM_CPU=${VM_CPU:-4} +export VM_DISK=${VM_DISK:-100} +export VM_MEMORY_SIZE=${VM_MEMORY_SIZE:-8192} +export VM_DISK_CACHE=${VM_DISK_CACHE:-unsafe} + +# Settings for bifrost +TEST_PLAYBOOK="opnfv-virtual.yaml" +USE_INSPECTOR=true +USE_CIRROS=false +TESTING_USER=root +DOWNLOAD_IPA=true +CREATE_IPA_IMAGE=false +INSPECT_NODES=true +INVENTORY_DHCP=false +INVENTORY_DHCP_STATIC_IP=false +WRITE_INTERFACES_FILE=true + +# Settings for console access +export DIB_DEV_USER_PWDLESS_SUDO=yes +export DIB_DEV_USER_PASSWORD=devuser + +# Additional dib elements +export EXTRA_DIB_ELEMENTS=${EXTRA_DIB_ELEMENTS:-"openssh-server"} + +# dib configuration +case ${XCI_DISTRO,,} in + # These should ideally match the CI jobs + ubuntu) + export DIB_OS_RELEASE="${DIB_OS_RELEASE:-xenial}" + export DIB_OS_ELEMENT="${DIB_OS_ELEMENT:-ubuntu-minimal}" + export DIB_OS_PACKAGES="${DIB_OS_PACKAGES:-vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl,iptables}" + ;; + centos) + export DIB_OS_RELEASE="${DIB_OS_RELEASE:-7}" + export DIB_OS_ELEMENT="${DIB_OS_ELEMENT:-centos-minimal}" + export DIB_OS_PACKAGES="${DIB_OS_PACKAGES:-vim,less,bridge-utils,iputils,rsyslog,curl,iptables}" + ;; + opensuse) + export DIB_OS_RELEASE="${DIB_OS_RELEASE:-42.3}" + export DIB_OS_ELEMENT="${DIB_OS_ELEMENT:-opensuse-minimal}" + export DIB_OS_PACKAGES="${DIB_OS_PACKAGES:-vim,less,bridge-utils,iputils,rsyslog,curl,iptables}" + ;; +esac + +# Copy the OS images if found +if [[ -e ${XCI_PATH}/deployment_image.qcow2 ]]; then + sudo mkdir -p /httpboot + sudo mv ${XCI_PATH}/deployment_image.qcow2* /httpboot/ +fi + +# Install missing dependencies +$(which pip) install -q --user --upgrade -r "$(dirname $0)/../requirements.txt" + +# Change working directory +cd $BIFROST_HOME/playbooks + +# NOTE(hwoarang): Disable selinux as we are hitting issues with it from time to +# time. Remove this when Centos7 is a proper gate on bifrost so we know that +# selinux works as expected. +if [[ -e /etc/centos-release ]]; then + echo "*************************************" + echo "WARNING: Disabling selinux on CentOS7" + echo "*************************************" + sudo setenforce 0 +fi + +# Create the VMS +ansible-playbook ${XCI_ANSIBLE_PARAMS} \ + -i inventory/localhost \ + test-bifrost-create-vm.yaml \ + -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \ + -e test_vm_cpu='host-model' \ + -e test_vm_memory_size=${VM_MEMORY_SIZE} \ + -e enable_venv=${ENABLE_VENV} \ + -e test_vm_domain_type=${VM_DOMAIN_TYPE} \ + -e ${INVENTORY_FILE_FORMAT}=${BAREMETAL_DATA_FILE} + +# Execute the installation and VM startup test +ansible-playbook ${XCI_ANSIBLE_PARAMS} \ + -i inventory/bifrost_inventory.py \ + ${TEST_PLAYBOOK} \ + -e use_cirros=${USE_CIRROS} \ + -e testing_user=${TESTING_USER} \ + -e test_vm_num_nodes=${TEST_VM_NUM_NODES} \ + -e test_vm_cpu='host-model' \ + -e inventory_dhcp=${INVENTORY_DHCP} \ + -e inventory_dhcp_static_ip=${INVENTORY_DHCP_STATIC_IP} \ + -e enable_venv=${ENABLE_VENV} \ + -e enable_inspector=${USE_INSPECTOR} \ + -e inspect_nodes=${INSPECT_NODES} \ + -e download_ipa=${DOWNLOAD_IPA} \ + -e create_ipa_image=${CREATE_IPA_IMAGE} \ + -e write_interfaces_file=${WRITE_INTERFACES_FILE} \ + -e ipv4_gateway=192.168.122.1 \ + -e wait_timeout=${PROVISION_WAIT_TIMEOUT} \ + -e enable_keystone=false \ + -e ironicinspector_source_install=true \ + -e ironicinspector_git_branch=${BIFROST_IRONIC_INSPECTOR_VERSION} \ + -e ironicinspectorclient_source_install=true \ + -e ironicinspectorclient_git_branch=${BIFROST_IRONIC_INSPECTOR_CLIENT_VERSION} \ + -e ironicclient_source_install=true \ + -e ironicclient_git_branch=${BIFROST_IRONIC_CLIENT_VERSION} \ + -e ironic_git_branch=${BIFROST_IRONIC_VERSION} \ + -e use_prebuilt_images=${BIFROST_USE_PREBUILT_IMAGES} \ + -e xci_distro=${XCI_DISTRO} +EXITCODE=$? + +if [ $EXITCODE != 0 ]; then + echo "************************************" + echo "Provisioning failed. See logs folder" + echo "************************************" +fi + +exit $EXITCODE diff --git a/xci/infra/bifrost/scripts/destroy-env.sh b/xci/infra/bifrost/scripts/destroy-env.sh new file mode 100755 index 00000000..9d53dc15 --- /dev/null +++ b/xci/infra/bifrost/scripts/destroy-env.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 RedHat and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# We need to execute everything as root +if [[ $(whoami) != "root" ]]; then + echo "Error: This script must be run as root!" + exit 1 +fi + +# Start fresh +rm -rf /opt/stack +# HOME is normally set by sudo -H +rm -rf ${HOME}/.config/openstack +rm -rf ${HOME}/.ansible + +# bifrost installs everything on venv so we need to look there if virtualbmc is not installed on the host. +if which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && source /opt/stack/bifrost/bin/activate; }; then + # Delete all libvirt VMs and hosts from vbmc (look for a port number) + for vm in $(vbmc list | awk '/[0-9]/{{ print $2 }}'); do + if which virsh &>/dev/null; then + virsh destroy $vm || true + virsh undefine $vm || true + fi + vbmc delete $vm + done + which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && deactivate; } +fi + +# Destroy all XCI VMs if the previous operation failed +[[ -n ${XCI_FLAVOR} ]] && \ + for vm in ${TEST_VM_NODE_NAMES}; do + if which virsh &>/dev/null; then + virsh destroy $vm || true + virsh undefine $vm || true + fi + done + +service ironic-conductor stop || true + +echo "removing inventory files created by previous builds" +rm -rf /tmp/baremetal.* + +echo "removing ironic database" +if $(which mysql &> /dev/null); then + mysql_ironic_user=$(sudo grep "connection" /etc/ironic/ironic.conf | cut -d : -f 2 ) + msyql_ironic_password=$(sudo grep "connection" /etc/ironic/ironic.conf | cut -d : -f 3) + mysql -u${mysql_ironic_user#*//} -p${msyql_ironic_password%%@*} --execute "drop database ironic;" +fi +echo "removing leases" +[[ -e /var/lib/misc/dnsmasq/dnsmasq.leases ]] && > /var/lib/misc/dnsmasq/dnsmasq.leases +echo "removing logs" +rm -rf /var/log/libvirt/baremetal_logs/* + +# clean up dib images by default +CLEAN_DIB_IMAGES=${CLEAN_DIB_IMAGES:-true} + +if [ $CLEAN_DIB_IMAGES = "true" ]; then + rm -rf /httpboot /tftpboot + mkdir /httpboot /tftpboot + chmod -R 755 /httpboot /tftpboot +fi + +# remove VM disk images +rm -rf /var/lib/libvirt/images/*.qcow2 + +echo "restarting services" +service dnsmasq restart || true +service libvirtd restart +service ironic-api restart || true +service ironic-conductor start || true +service ironic-inspector restart || true |