From f86ea2412bd80aef741cfc1249bfe3e9a004df92 Mon Sep 17 00:00:00 2001 From: Martin Klozik Date: Wed, 5 Dec 2018 09:04:12 +0100 Subject: CI: Daily job Trigger OPNFV and ONAP installation by AUTO daily job. JIRA: AUTO-79 Change-Id: Ide29fa85155e7a73c3f397890a4bbdf28c39a9dd Signed-off-by: Martin Klozik --- ci/build-auto.sh | 158 +++++++++++++++++++++++++++++++++++++------------ ci/deploy-onap-fuel.sh | 9 ++- ci/plot-results.sh | 101 +++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 43 deletions(-) create mode 100755 ci/plot-results.sh diff --git a/ci/build-auto.sh b/ci/build-auto.sh index 96588b9..00b67b1 100755 --- a/ci/build-auto.sh +++ b/ci/build-auto.sh @@ -20,10 +20,12 @@ # Usage: # build-auto.sh job_type -# where job_type is one of "verify", "merge", "daily" +# +# Parameters: +# job_type - is one of "verify", "merge" or "daily" # # Example: -# ./ci/build-auto.sh daily +# ./ci/build-auto.sh verify # # exit codes @@ -31,11 +33,21 @@ EXIT=0 EXIT_UNKNOWN_JOB_TYPE=1 EXIT_LINT_FAILED=2 +EXIT_FUEL_FAILED=10 # # configuration # AUTOENV_DIR="$HOME/autoenv" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +LOG_DIR=$HOME/auto_ci_daily_logs +WORKSPACE=${WORKSPACE:-$PWD} + +# POD and SCENARIO details used during OPNFV deployment performed by daily job +NODE_NAME=${NODE_NAME:-"ericsson-virtual1"} +POD_LAB=$(echo $NODE_NAME | cut -d '-' -f1) +POD_NAME=$(echo $NODE_NAME | cut -d '-' -f2) +DEPLOY_SCENARIO=${DEPLOY_SCENARIO:-"os-nosdn-onap-noha"} # # functions @@ -47,6 +59,42 @@ function execute_auto_lint_check() { fi } +# check and install required packages +function dependencies_check() { + . /etc/os-release + if [ $ID == "ubuntu" ] ; then + echo "Dependencies check" + echo "==================" + # install system packages + for PACKAGE in "virtualenv" "pylint" "yamllint" "gnuplot" ; do + if dpkg -s $PACKAGE &> /dev/null ; then + printf " %-70s %-6s\n" $PACKAGE "OK" + else + printf " %-70s %-6s\n" $PACKAGE "missing" + sudo apt-get install -y $PACKAGE + fi + done + echo + fi +} + +# create virtualenv if needed and enable it +function virtualenv_prepare() { + if [ ! -e $AUTOENV_DIR ] ; then + echo "Create AUTO environment" + echo "=======================" + virtualenv "$AUTOENV_DIR" + echo + fi + + # activate and update virtualenv + echo "Update AUTO environment" + echo "=======================" + source "$AUTOENV_DIR"/bin/activate + pip install -r ./requirements.txt + echo +} + # # main # @@ -55,20 +103,8 @@ echo # enter workspace dir cd $WORKSPACE -# create virtualenv if needed -if [ ! -e $AUTOENV_DIR ] ; then - echo "Create AUTO environment" - echo "=======================" - virtualenv "$AUTOENV_DIR" - echo -fi - -# activate and update virtualenv -echo "Update AUTO environment" -echo "=======================" -source "$AUTOENV_DIR"/bin/activate -pip install -r ./requirements.txt -echo +# check if required packages are installed +dependencies_check # execute job based on passed parameter case $1 in @@ -77,15 +113,9 @@ case $1 in echo "AUTO verify job" echo "===============" - # Example of verify job body. Functions can call - # external scripts, etc. - + virtualenv_prepare execute_auto_lint_check #execute_auto_doc_check - #install_opnfv MCP - #install_onap - #execute_sanity_check - #execute_tests $1 # Everything went well, so report SUCCESS to Jenkins exit $EXIT @@ -95,15 +125,9 @@ case $1 in echo "AUTO merge job" echo "==============" - # Example of merge job body. Functions can call - # external scripts, etc. - + virtualenv_prepare execute_auto_lint_check #execute_auto_doc_check - #install_opnfv MCP - #install_onap - #execute_sanity_check - #execute_tests $1 # propagate result to the Jenkins job exit $EXIT @@ -112,15 +136,73 @@ case $1 in echo "==============" echo "AUTO daily job" echo "==============" + echo + echo "Deployment details:" + echo " LAB: $POD_LAB" + echo " POD: $POD_NAME" + echo " Scenario: $DEPLOY_SCENARIO" + echo " WORKSPACE: $WORKSPACE" + echo - # Example of daily job body. Functions can call - # external scripts, etc. - - #install_opnfv MCP - #install_onap - #execute_sanity_check - #execute_tests $1 - #push_results_and_logs_to_artifactory + # create log dir if needed + if [ ! -e $LOG_DIR ] ; then + echo "Create AUTO LOG DIRECTORY" + echo "=========================" + echo "mkdir $LOG_DIR" + mkdir $LOG_DIR + echo + fi + + echo "Installation of OPNFV and ONAP" + echo "==============================" + # clone fuel and execute installation of ONAP scenario to install + # ONAP on top of OPNFV deployment + [ -e fuel ] && rm -rf fuel + git clone https://gerrit.opnfv.org/gerrit/fuel + cd fuel + # Fuel master branch is currently broken; thus use stable/gambia + # branch with recent master version of ONAP scenario + git checkout stable/gambia + git checkout origin/master mcp/config/states/onap \ + mcp/config/scenario/os-nosdn-onap-ha.yaml \ + mcp/config/scenario/os-nosdn-onap-noha.yaml + # use larger disk size for virtual nodes + sed -i -re 's/(qemu-img resize.*)100G/\1400G/' mcp/scripts/lib_jump_deploy.sh + + LOG_FILE="$LOG_DIR/deploy_${TIMESTAMP}.log" + echo "ci/deploy.sh -l $POD_LAB -p $POD_NAME -s $DEPLOY_SCENARIO |&\ + tee $LOG_FILE" + DEPLOY_START=$(date +%Y%m%d_%H%M%S) + ci/deploy.sh -l $POD_LAB -p $POD_NAME -s $DEPLOY_SCENARIO |&\ + tee $LOG_FILE + + # report failure if fuel failed to install OPNFV or ONAP + [ $? -ne 0 ] && exit $EXIT_FUEL_FAILED + + # process report + DEPLOY_END=$(date +%Y%m%d_%H%M%S) + REPORT_FILE="$LOG_DIR/deploy_report_${TIMESTAMP}.txt" + CSV_SUMMARY="$LOG_DIR/deploy_summary_${TIMESTAMP}.csv" + MARKER="ONAP INSTALLATION REPORT" + # cut report from installation log file + sed -n "/^$MARKER/,/^END OF $MARKER/p;/^END OF $MARKER/q" \ + $LOG_FILE > $REPORT_FILE + PODS_TOTAL=$(grep "PODs Total" $REPORT_FILE | sed -e 's/[^0-9]//g') + PODS_FAILED=$(grep "PODs Failed" $REPORT_FILE | sed -e 's/[^0-9]//g') + TC_SUM=$(grep "tests total" $REPORT_FILE | tail -n1 |\ + sed -e 's/[^0-9,]//g') + + echo "Start Time,End Time,Total PODs,Failed PODs,Total Tests,Passed"\ + "Tests,Failed Tests" >> $CSV_SUMMARY + echo "$DEPLOY_START,$DEPLOY_END,$PODS_TOTAL,$PODS_FAILED,$TC_SUM"\ + >> $CSV_SUMMARY + + # plot graphs from result summaries and print txt versions if possible + cd $WORKSPACE + ci/plot-results.sh + for GRAPH in $(ls -1 graph*txt 2> /dev/null) ; do + cat $GRAPH + done # propagate result to the Jenkins job exit $EXIT diff --git a/ci/deploy-onap-fuel.sh b/ci/deploy-onap-fuel.sh index 2e8c3ad..70e1287 100755 --- a/ci/deploy-onap-fuel.sh +++ b/ci/deploy-onap-fuel.sh @@ -101,7 +101,7 @@ echo "Storage for instances: CMP_STORAGE_TOTAL=$CMP_STORAGE_TOTAL" echo "Number of VMs: VM_COUNT=$VM_COUNT" # Calculate VM parameters; there will be up to 1 VM per Compute node # to maximize resources available for VMs -PER=90 # % of compute resources will be consumed by VMs +PER=85 # % of compute resources will be consumed by VMs VM_DISK_MAX=100 # GB - max VM disk size VM_MEM_MAX=81920 # MB - max VM RAM size VM_CPUS_MAX=56 # max count of VM CPUs @@ -180,16 +180,15 @@ while [ $VM_ITER -le $VM_COUNT ] ; do --nic net-id=onap_private_network --security-group onap_security_group \ --key-name onap_key ${VM_NAME[$VM_ITER]} \ --availability-zone ${HOST_ZONE}:${HOST_NAME[$HOST_ITER]} - sleep 5 # wait for VM init before floating IP can be assigned + sleep 10 # wait for VM init before floating IP can be assigned openstack server add floating ip ${VM_NAME[$VM_ITER]} ${VM_IP[$VM_ITER]} + echo "Waiting for ${VM_NAME[$VM_ITER]} to start up for 1m at $(date)" + sleep 1m VM_ITER=$(($VM_ITER+1)) HOST_ITER=$(($HOST_ITER+1)) [ $HOST_ITER -ge $HOST_COUNT ] && HOST_ITER=0 done -echo "Waiting for VMs to start up for 2m at $(date)" -sleep 2m - openstack server list -c ID -c Name -c Status -c Networks -c Host --long # check that SSH to all VMs is working diff --git a/ci/plot-results.sh b/ci/plot-results.sh new file mode 100755 index 0000000..22ab1d6 --- /dev/null +++ b/ci/plot-results.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# +# Copyright 2017-2018 Intel Corporation., Tieto +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script for graphical representation of AUTO result summaries +# +# Usage: +# ./create_graph [directory] +# +# where: +# "directory" is an optional directory name, where summary of auto +# installation report is stored +# Default value: "$HOME/auto_ci_daily_logs" + +NUMBER_OF_RESULTS=50 # max number of recent results to be compared in graph +DIR="$HOME/auto_ci_daily_logs" + +function clean_data() { + rm -rf summary.csv + rm -rf graph*plot + rm -rf graph*txt + rm -rf graph*png +} + +function prepare_data() { + FIRST=1 + CSV_LIST=$(ls -1 ${DIR}/deploy_summary*csv | tail -n ${NUMBER_OF_RESULTS}) + for result_file in $CSV_LIST ; do + tmp_dir=`dirname $result_file` + TIMESTAMP=`basename $tmp_dir | cut -d'_' -f2-` + if [ $FIRST -eq 1 ] ; then + head -n1 $result_file > summary.csv + FIRST=0 + fi + tail -n+2 ${result_file} >> summary.csv + done +} + +function plot_data() { + echo "Created graphs:" + for TYPE in png txt; do + for GRAPH in "graph_pods" "graph_tcs" ; do + OUTPUT="$GRAPH.plot" + GRAPH_NAME="${GRAPH}.${TYPE}" + cat > $OUTPUT <<- EOM +set datafile separator "," +set xdata time +set timefmt "%Y%m%d_%H%M%S" +set format x "%m-%d" +set xlabel "date" +set format y "%8.0f" +EOM + if [ "$TYPE" == "png" ] ; then + echo 'set term png size 1024,768' >> $OUTPUT + else + echo 'set term dumb 100,30' >> $OUTPUT + fi + + if [ "$GRAPH" == "graph_pods" ] ; then + echo 'set ylabel "PODs"' >> $OUTPUT + echo 'set yrange [0:]' >> $OUTPUT + echo "set title \"ONAP K8S PODs\"" >> $OUTPUT + COL1=3 + COL2=4 + else + echo 'set ylabel "testcases"' >> $OUTPUT + echo 'set yrange [0:]' >> $OUTPUT + echo "set title \"ONAP Health TestCases\"" >> $OUTPUT + COL1=5 + COL2=6 + fi + + iter=0 + echo "set output \"$GRAPH_NAME\"" >> $OUTPUT + echo -n "plot " >> $OUTPUT + echo $"'summary.csv' using 1:$COL1 with linespoints title columnheader($COL1) \\" >> $OUTPUT + echo $", 'summary.csv' using 1:$COL2 with linespoints title columnheader($COL2) \\" >> $OUTPUT + gnuplot $OUTPUT + echo -e "\t$GRAPH_NAME" + done + done +} + +# +# Main body +# +clean_data +prepare_data +plot_data -- cgit 1.2.3-korg