From 6fb97028e93a7844000204be53ed937a47d9082f Mon Sep 17 00:00:00 2001 From: Victor Morales Date: Fri, 28 Aug 2020 16:35:12 -0400 Subject: Improve kuberef lint script The OPNFV security audit releng's job prevent to retrieve binaries from external sources which results on depending to the binaries provided by the OS package managers. This change pretends to cover different scenarios where the Jenkins builder is unknown. Signed-off-by: Victor Morales Change-Id: I37d9b15d3ac9de0b06a96e07eed90f7cc717f444 --- jjb/kuberef/Vagrantfile | 70 ++++++++++++++++++++++++++++++++++ jjb/kuberef/kuberef-run-linting.sh | 78 +++++++++++++++++++++++++++++++++++--- jjb/kuberef/tox.ini | 28 ++++++++++++++ 3 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 jjb/kuberef/Vagrantfile create mode 100644 jjb/kuberef/tox.ini diff --git a/jjb/kuberef/Vagrantfile b/jjb/kuberef/Vagrantfile new file mode 100644 index 000000000..61132f57b --- /dev/null +++ b/jjb/kuberef/Vagrantfile @@ -0,0 +1,70 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +$no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost" +# NOTE: This range is based on vagrant-libvirt network definition CIDR 192.168.121.0/24 +(1..254).each do |i| + $no_proxy += ",192.168.121.#{i}" +end +$no_proxy += ",10.0.2.15" + +distros = { +"centos_7" => "generic/centos7", +"ubuntu_xenial" => "generic/ubuntu1604", +"ubuntu_bionic" => "generic/ubuntu1804", +"ubuntu_focal" => "generic/ubuntu2004", +"opensuse" => "opensuse/Tumbleweed.x86_64" +} + +Vagrant.configure("2") do |config| + config.vm.provider :libvirt + config.vm.provider :virtualbox + + config.vm.synced_folder './', '/vagrant', type: "rsync", + rsync__args: ["--verbose", "--archive", "--delete", "-z"] + distros.each do |key,box| + config.vm.define key do |node| + node.vm.box = box + node.vm.box_check_update = false + end + end + + config.vm.provision 'shell', privileged: false, inline: <<-SHELL + set -o errexit + + cd /vagrant + ./kuberef-run-linting.sh + SHELL + + [:virtualbox, :libvirt].each do |provider| + config.vm.provider provider do |p| + p.cpus = 1 + p.memory = ENV['MEMORY'] || 512 + end + end + + config.vm.provider "virtualbox" do |v| + v.gui = false + end + + config.vm.provider :libvirt do |v| + v.random_hostname = true + v.management_network_address = "192.168.121.0/24" + end + + if ENV['http_proxy'] != nil and ENV['https_proxy'] != nil + if Vagrant.has_plugin?('vagrant-proxyconf') + config.proxy.http = ENV['http_proxy'] || ENV['HTTP_PROXY'] || "" + config.proxy.https = ENV['https_proxy'] || ENV['HTTPS_PROXY'] || "" + config.proxy.no_proxy = $no_proxy + config.proxy.enabled = { docker: false, git: false } + end + end +end diff --git a/jjb/kuberef/kuberef-run-linting.sh b/jjb/kuberef/kuberef-run-linting.sh index 810f93321..084eac91c 100755 --- a/jjb/kuberef/kuberef-run-linting.sh +++ b/jjb/kuberef/kuberef-run-linting.sh @@ -12,10 +12,51 @@ set -o nounset set -o pipefail set -o xtrace +# _vercmp() - Function that compares two versions +function _vercmp { + local v1=$1 + local op=$2 + local v2=$3 + local result + + # sort the two numbers with sort's "-V" argument. Based on if v2 + # swapped places with v1, we can determine ordering. + result=$(echo -e "$v1\n$v2" | sort -V | head -1) + + case $op in + "==") + [ "$v1" = "$v2" ] + return + ;; + ">") + [ "$v1" != "$v2" ] && [ "$result" = "$v2" ] + return + ;; + "<") + [ "$v1" != "$v2" ] && [ "$result" = "$v1" ] + return + ;; + ">=") + [ "$result" = "$v2" ] + return + ;; + "<=") + [ "$result" = "$v1" ] + return + ;; + *) + die $LINENO "unrecognised op: $op" + ;; + esac +} + echo "Requirements validation" # shellcheck disable=SC1091 source /etc/os-release || source /usr/lib/os-release +min_shellcheck_version=0.4.0 +min_tox_version=3.5 + pkgs="" if ! command -v shellcheck; then case ${ID,,} in @@ -27,9 +68,19 @@ if ! command -v shellcheck; then ;; esac fi - if ! command -v pip; then - pkgs+=" python-pip" + case ${ID,,} in + *suse*|rhel|centos|fedora) + pkgs+=" python3-pip python3-setuptools" + ;; + ubuntu|debian) + if _vercmp "${VERSION_ID}" '<=' "18.04"; then + pkgs+=" python-pip python-setuptools" + else + pkgs+=" python3-pip python3-setuptools" + fi + ;; + esac fi if [ -n "$pkgs" ]; then @@ -37,11 +88,13 @@ if [ -n "$pkgs" ]; then case ${ID,,} in *suse*) sudo zypper install --gpg-auto-import-keys refresh - sudo -H -E zypper install -y --no-recommends "$pkgs" + # shellcheck disable=SC2086 + sudo -H -E zypper install -y --no-recommends $pkgs ;; ubuntu|debian) sudo apt-get update - sudo -H -E apt-get -y --no-install-recommends install "$pkgs" + # shellcheck disable=SC2086 + sudo -H -E apt-get -y --no-install-recommends install $pkgs ;; rhel|centos|fedora) PKG_MANAGER=$(command -v dnf || command -v yum) @@ -49,9 +102,18 @@ if [ -n "$pkgs" ]; then sudo -H -E "$PKG_MANAGER" -q -y install epel-release fi sudo "$PKG_MANAGER" updateinfo --assumeyes - sudo -H -E "${PKG_MANAGER}" -y install "$pkgs" + # shellcheck disable=SC2086 + sudo -H -E "$PKG_MANAGER" -y install $pkgs ;; esac + if ! command -v pip && command -v pip3 ; then + sudo ln -s "$(command -v pip3)" /usr/bin/pip + fi + sudo "$(command -v pip)" install --upgrade pip +fi + +if ! command -v tox || _vercmp "$(tox --version | awk '{print $1}')" '<' "$min_tox_version"; then + sudo "$(command -v pip)" install tox==$min_tox_version fi echo "Server tools information:" @@ -61,4 +123,8 @@ shellcheck -V echo "Linting process execution" tox -e lint -bash -c 'shopt -s globstar; shellcheck -x **/*.sh' +if _vercmp "$(shellcheck --version | awk 'FNR==2{print $2}')" '<' "$min_shellcheck_version"; then + bash -c 'shopt -s globstar; shellcheck **/*.sh' +else + bash -c 'shopt -s globstar; shellcheck -x **/*.sh' +fi diff --git a/jjb/kuberef/tox.ini b/jjb/kuberef/tox.ini new file mode 100644 index 000000000..328a05df0 --- /dev/null +++ b/jjb/kuberef/tox.ini @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +[tox] +minversion = 3.5 +skipsdist = True +envlist = lint + +[testenv] +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +usedevelop = False +install_command = pip install {opts} {packages} + +[testenv:lint] +deps = + {env:BASHATE_INSTALL_PATH:bashate} +whitelist_externals = bash +commands = bash -c "find {toxinidir} \ + -not -path {toxinidir}/.tox/\* \ +# E006 check for lines longer than 79 columns + -name \*.sh | xargs bashate -v -iE006" -- cgit 1.2.3-korg