summaryrefslogtreecommitdiffstats
path: root/xci/files/xci-lib.sh
blob: 2f360e44e507b7361f735894df805dd8d2e8b540 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
# SPDX-license-identifier: Apache-2.0
##############################################################################
# Copyright (c) 2018 SUSE LINUX GmbH.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################

# Avoid double sourcing the file
[[ -n ${XCI_LIB_SOURCED:-} ]] && return 0 || export XCI_LIB_SOURCED=1

function usage() {
    echo "
Usage: $(basename ${0}) [-i <idf>] [-p <pdf>]

    -h: This message
    -i: Installer Descriptor File (IDF). (Default ${XCI_PATH}/xci/var/idf.yml)
    -p: Pod Descriptor File (PDF). (Default ${XCI_PATH}/xci/var/pdf.yml)
    "
    exit 0
}

function parse_cmdline_opts() {
    IDF=${XCI_PATH}/xci/var/idf.yml
    PDF=${XCI_PATH}/xci/var/pdf.yml

    while getopts ":hi:p:" o; do
        case "${o}" in
            i) IDF="${OPTARG}" ;;
            p) PDF="${OPTARG}" ;;
            h) usage ;;
            *) echo "ERROR: Invalid option '-${OPTARG}'"; usage ;;
        esac
    done

    # Do all the exports
    export PDF=$(realpath ${PDF})
    export IDF=$(realpath ${IDF})
}

function bootstrap_xci_env() {
    # Declare our virtualenv
    export XCI_VENV=${XCI_PATH}/venv/
    # source user vars
    source $XCI_PATH/xci/config/user-vars
    # source pinned versions
    source $XCI_PATH/xci/config/pinned-versions
    # source flavor configuration
    source "$XCI_PATH/xci/config/${XCI_FLAVOR}-vars"
    # source installer configuration
    source "$XCI_PATH/xci/installer/${INSTALLER_TYPE}/env" &>/dev/null || true
    # source xci configuration
    source $XCI_PATH/xci/config/env-vars
}

function install_ansible() {
    set -eu

    # Use the upper-constraints file from the pinned requirements repository.
    local uc="https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_REQUIREMENTS_VERSION}/upper-constraints.txt"
    local osa_uc="https://raw.githubusercontent.com/openstack/openstack-ansible/${OPENSTACK_OSA_VERSION}/global-requirement-pins.txt"
    local install_map

    declare -A PKG_MAP

    # workaround: for latest bindep to work, it needs to use en_US local
    export LANG="C"

    CHECK_CMD_PKGS=(
        gcc
        libffi
        libopenssl
        lsb-release
        make
        net-tools
        python-devel
        python
        python-pyyaml
        venv
        wget
        curl
    )

    source /etc/os-release || source /usr/lib/os-release
    case ${ID,,} in
      *suse*)
        OS_FAMILY="Suse"
        INSTALLER_CMD="sudo -H -E zypper -q install -y --no-recommends"
        CHECK_CMD="zypper search --match-exact --installed"
        PKG_MAP=(
            [gcc]=gcc
            [libffi]=libffi-devel
            [libopenssl]=libopenssl-devel
            [lsb-release]=lsb-release
            [make]=make
            [net-tools]=net-tools
            [pip]=python-pip
            [python]=python
            [python-devel]=python-devel
            [python-pyyaml]=python-PyYAML
            [venv]=python-virtualenv
            [wget]=wget
            [curl]=curl
        )
        EXTRA_PKG_DEPS=( python-xml )
        sudo zypper -n ref
        # NOTE (cinerama): we can't install python without removing this package
        # if it exists
        if $(${CHECK_CMD} patterns-openSUSE-minimal_base-conflicts &> /dev/null); then
            sudo -H zypper remove -y patterns-openSUSE-minimal_base-conflicts
        fi
        ;;

        ubuntu|debian)
        OS_FAMILY="Debian"
        export DEBIAN_FRONTEND=noninteractive
        INSTALLER_CMD="sudo -H -E apt-get -y -q=3 install"
        CHECK_CMD="dpkg -l"
        PKG_MAP=(
            [gcc]=gcc
            [libffi]=libffi-dev
            [libopenssl]=libssl-dev
            [lsb-release]=lsb-release
            [make]=make
            [net-tools]=net-tools
            [pip]=python-pip
            [python]=python-minimal
            [python-devel]=libpython-dev
            [python-pyyaml]=python-yaml
            [venv]=python-virtualenv
            [wget]=wget
            [curl]=curl
        )
        EXTRA_PKG_DEPS=( apt-utils )
        sudo apt-get update
        ;;

        rhel|fedora|centos)
        OS_FAMILY="RedHat"
        PKG_MANAGER=$(which dnf || which yum)
        INSTALLER_CMD="sudo -H -E ${PKG_MANAGER} -q -y install"
        CHECK_CMD="rpm -q"
        PKG_MAP=(
            [gcc]=gcc
            [libffi]=libffi-devel
            [libopenssl]=openssl-devel
            [lsb-release]=redhat-lsb
            [make]=make
            [net-tools]=net-tools
            [pip]=python2-pip
            [python]=python
            [python-devel]=python-devel
            [python-pyyaml]=PyYAML
            [venv]=python-virtualenv
            [wget]=wget
            [curl]=curl
        )
        sudo $PKG_MANAGER updateinfo
        EXTRA_PKG_DEPS=( deltarpm )
        ;;

        *) echo "ERROR: Supported package manager not found.  Supported: apt, dnf, yum, zypper"; exit 1;;
    esac

    # Build instllation map
    for pkgmap in ${CHECK_CMD_PKGS[@]}; do
        install_map+=(${PKG_MAP[$pkgmap]} )
    done

    install_map+=(${EXTRA_PKG_DEPS[@]} )

    ${INSTALLER_CMD} ${install_map[@]}

    # Note(cinerama): If pip is linked to pip3, the rest of the install
    # won't work. Remove the alternatives. This is due to ansible's
    # python 2.x requirement.
    if [[ $(readlink -f /etc/alternatives/pip) =~ "pip3" ]]; then
        sudo -H update-alternatives --remove pip $(readlink -f /etc/alternatives/pip)
    fi

    # We need to prepare our virtualenv now
    virtualenv --quiet --no-site-packages ${XCI_VENV}
    set +u
    source ${XCI_VENV}/bin/activate
    set -u

    # We are inside the virtualenv now so we should be good to use pip and python from it.
    pip -q install --upgrade pip==9.0.3 # We need a version which supports the '-c' parameter
    pip -q install --upgrade -c $uc -c $osa_uc ara virtualenv pip setuptools shade ansible==$XCI_ANSIBLE_PIP_VERSION ansible-lint==3.4.21

    ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
    export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks"
}

ansible_lint() {
    set -eu
    local playbooks_dir=(xci/playbooks xci/installer/osa/playbooks xci/installer/kubespray/playbooks)
    # Extract role from scenario information
    local testing_role=$(sed -n "/^- scenario: ${DEPLOY_SCENARIO}/,/^$/p" ${XCI_PATH}/xci/opnfv-scenario-requirements.yml | grep role | rev | cut -d '/' -f -1 | rev)

    # clear XCI_CACHE
    rm -rf ${XCI_CACHE}/repos/openstack-ansible-tests

    # Clone OSA rules too
    git clone --quiet --depth 1 https://github.com/openstack/openstack-ansible-tests.git \
        ${XCI_CACHE}/repos/openstack-ansible-tests

    # Because of https://github.com/willthames/ansible-lint/issues/306, ansible-lint does not understand
    # import and includes yet so we need to trick it with a fake playbook so we can test our roles. We
    # only test the role for the scenario we are testing
    echo "Building testing playbook for role: ${testing_role}"
    cat > ${XCI_PATH}/xci/playbooks/test-playbook.yml << EOF
        - name: Testing playbook
          hosts: localhost
          roles:
            - ${testing_role}
EOF

    # Only check our own playbooks
    for dir in ${playbooks_dir[@]}; do
        for play in $(ls ${XCI_PATH}/${dir}/*.yml); do
            echo -en "Checking '${play}' playbook..."
            ansible-lint --nocolor -R -r \
                ${XCI_CACHE}/repos/openstack-ansible-tests/ansible-lint ${play}
            echo -en "[OK]\n"
        done
    done

    # Remove testing playbook
    rm ${XCI_PATH}/xci/playbooks/test-playbook.yml
}

collect_xci_logs() {
    echo "----------------------------------"
    echo "Info: Collecting XCI logs"
    echo "----------------------------------"

    # Create the ARA log directory and store the sqlite source database
    mkdir -p ${LOG_PATH}/ara/ ${LOG_PATH}/opnfv/ara

    rsync -q -a "${HOME}/.ara/ansible.sqlite" "${LOG_PATH}/ara/"
    rsync -q -a root@${OPNFV_HOST_IP}:releng-xci/${LOG_PATH#$XCI_PATH/}/ ${LOG_PATH}/opnfv/ &> /dev/null || true
    rsync -q -a root@${OPNFV_HOST_IP}:.ara/ansible.sqlite ${LOG_PATH}/opnfv/ara/ &> /dev/null || true

    sudo -H -E bash -c 'chown ${SUDO_UID}:${SUDO_GID} -R ${LOG_PATH}/'
}

submit_bug_report() {
    cd ${XCI_PATH}
    echo ""
    echo "-------------------------------------------------------------------------"
    echo "Oh nooooo! The XCI deployment failed miserably :-("
    echo ""
    echo "If you need help, please choose one of the following options"
    echo "* #opnfv-pharos @ freenode network"
    echo "* opnfv-tech-discuss mailing list (https://lists.opnfv.org/mailman/listinfo/opnfv-tech-discuss)"
    echo "  - Please prefix the subject with [XCI]"
    echo "* https://jira.opnfv.org (Release Engineering project)"
    echo ""
    echo "Do not forget to submit the following information on your bug report:"
    echo ""
    git diff --quiet && echo "releng-xci tree status: clean" || echo "releng-xci tree status: local modifications"
    echo "opnfv/releng-xci version: $(git rev-parse HEAD)"
    echo "openstack/bifrost version: $OPENSTACK_BIFROST_VERSION"
    echo "openstack/openstack-ansible version: $OPENSTACK_OSA_VERSION"
    echo "xci flavor: $XCI_FLAVOR"
    echo "xci installer: $INSTALLER_TYPE"
    echo "xci scenario: $DEPLOY_SCENARIO"
    echo "Environment variables:"
    env | grep --color=never '\(OPNFV\|XCI\|INSTALLER_TYPE\|OPENSTACK\|SCENARIO\|ANSIBLE\)'
    echo "-------------------------------------------------------------------------"
}

log_xci_information() {
    local scenario_version scenario_sha

    cd ${XCI_SCENARIOS_CACHE}/${DEPLOY_SCENARIO}
    scenario_sha=$(git rev-parse HEAD)
    scenario_version=$(git describe --exact 2>/dev/null || echo "master")
    cd -
    echo "Info: Starting XCI Deployment"
    echo "Info: Deployment parameters"
    echo "-------------------------------------------------------------------------"
    echo "OPNFV scenario: $DEPLOY_SCENARIO"
    echo "Scenario version: ${scenario_version} (sha: ${scenario_sha})"
    echo "xci flavor: $XCI_FLAVOR"
    echo "xci installer: $INSTALLER_TYPE"
    echo "infra deployment: $INFRA_DEPLOYMENT"
    echo "opnfv/releng-xci version: $(git rev-parse HEAD)"
    [[ "$INFRA_DEPLOYMENT" == "bifrost" ]] && echo "openstack/bifrost version: $OPENSTACK_BIFROST_VERSION"
    [[ "$INSTALLER_TYPE" == "osa" ]] && echo "openstack/openstack-ansible version: $OPENSTACK_OSA_VERSION"
    [[ "$INSTALLER_TYPE" == "kubespray" ]] && echo "kubespray version: $KUBESPRAY_VERSION"
    echo "-------------------------------------------------------------------------"
}

exit_trap() {
    submit_bug_report
    collect_xci_logs
}

# vim: set ts=4 sw=4 expandtab: