diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/__init__.py | 2 | ||||
| -rw-r--r-- | utils/dashboard/process_data.py | 4 | ||||
| -rwxr-xr-x | utils/dashboard/rubbos_collector.py | 22 | ||||
| -rwxr-xr-x | utils/dashboard/uploader.py | 4 | ||||
| -rw-r--r-- | utils/infra_setup/__init__.py | 2 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/common.py | 3 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/consts/parameters.py | 1 | ||||
| -rwxr-xr-x | utils/infra_setup/heat/template.py | 4 | ||||
| -rwxr-xr-x | utils/tools/bottlenecks_image_modify | 172 | ||||
| -rwxr-xr-x | utils/tools/ubuntu-server-clouding-modify.sh | 43 |
10 files changed, 241 insertions, 16 deletions
diff --git a/utils/__init__.py b/utils/__init__.py index 38727287..b124dfa9 100644 --- a/utils/__init__.py +++ b/utils/__init__.py @@ -5,4 +5,4 @@ # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -##############################################################################
\ No newline at end of file +############################################################################## diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py index e71a1e82..6ca6ee1e 100644 --- a/utils/dashboard/process_data.py +++ b/utils/dashboard/process_data.py @@ -8,14 +8,14 @@ ############################################################################## -import subprocess as subp import sys from rubbos_collector import RubbosCollector from uploader import Uploader def printUsage(): - print "Usage: python process_data.py required_params(**) optional_params([])" + print ("Usage: python process_data.py required_params(**)" + " optional_params([])") print " ** -i|--input input_data_dir" print " ** -s|--suite suite_name" print " ** -c|--conf conf_file" diff --git a/utils/dashboard/rubbos_collector.py b/utils/dashboard/rubbos_collector.py index d9f86032..140700e3 100755 --- a/utils/dashboard/rubbos_collector.py +++ b/utils/dashboard/rubbos_collector.py @@ -17,19 +17,27 @@ def exec_shell(cmd): def get_onetime_data(dir_name): - cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name + cmd = ("grep -in 'remote client nodes' %s/index.html|awk " + "'{print $5}'|awk -F '<' '{print $1}'" % dir_name) client_node_num = int(exec_shell(cmd)) - cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name + cmd = ("grep -n 'Number of clients' %s/index.html|awk " + "'{print $5}'|awk -F '<' '{print $1}'" % dir_name) each_client_num = int(exec_shell(cmd)) total_client = (client_node_num + 1) * each_client_num - cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + cmd = ('grep -n "throughput" %s/stat_client*.html |awk -F "<B>"' + ' \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'' + 'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name) throughput = int(exec_shell(cmd)) - cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + cmd = ('grep -n "Total" %s/stat_client*.html |awk -F "<B>"' + ' \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>"' + ' \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name) request = int(exec_shell(cmd)) - cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + cmd = ('grep -n "Total" %s/stat_client*.html |awk -F "<B>"' + ' \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>"' + ' \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name) error_request = int(exec_shell(cmd)) return total_client, throughput, request, error_request @@ -44,8 +52,8 @@ class RubbosCollector(object): cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home result = [] for subdir in exec_shell(cmd).split('\n'): - total_client, throughput, request, error_request = get_onetime_data( - data_home + '/' + subdir) + total_client, throughput, request, error_request = \ + get_onetime_data(data_home + '/' + subdir) result.append({'client': total_client, 'throughput': throughput, 'request': request, diff --git a/utils/dashboard/uploader.py b/utils/dashboard/uploader.py index a4686560..97ffd38c 100755 --- a/utils/dashboard/uploader.py +++ b/utils/dashboard/uploader.py @@ -51,7 +51,8 @@ class Uploader(object): def _test(): - #data = '{"details": [{"client": 200, "throughput": 20}, {"client": 300, "throughput": 20}], "case_name": "rubbos"}' + # data = '{"details": [{"client": 200, "throughput": 20}, + # {"client": 300, "throughput": 20}], "case_name": "rubbos"}' if len(sys.argv) < 2: print ("no argumens input!!") exit(1) @@ -60,5 +61,6 @@ def _test(): data = json.load(stream) Uploader().upload_result(data) + if __name__ == "__main__": _test() diff --git a/utils/infra_setup/__init__.py b/utils/infra_setup/__init__.py index 38727287..b124dfa9 100644 --- a/utils/infra_setup/__init__.py +++ b/utils/infra_setup/__init__.py @@ -5,4 +5,4 @@ # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -##############################################################################
\ No newline at end of file +############################################################################## diff --git a/utils/infra_setup/heat/common.py b/utils/infra_setup/heat/common.py index 28257ac7..ab0dadf5 100755 --- a/utils/infra_setup/heat/common.py +++ b/utils/infra_setup/heat/common.py @@ -97,6 +97,7 @@ def get_heat_api_version(): return api_version return DEFAULT_HEAT_API_VERSION + def get_nova_api_version(): api_version = os.getenv('OS_COMPUTE_API_VERSION') if api_version is not None: @@ -110,4 +111,4 @@ def get_glance_api_version(): if api_version is not None: log.info("GLANCE_API_VERSION is set in env as '%s'", api_version) return api_version - return DEFAULT_GLANCE_API_VERSION
\ No newline at end of file + return DEFAULT_GLANCE_API_VERSION diff --git a/utils/infra_setup/heat/consts/parameters.py b/utils/infra_setup/heat/consts/parameters.py index f275c25b..15aa5b5d 100755 --- a/utils/infra_setup/heat/consts/parameters.py +++ b/utils/infra_setup/heat/consts/parameters.py @@ -7,7 +7,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import files # ------------------------------------------------------ # Directories and file locations diff --git a/utils/infra_setup/heat/template.py b/utils/infra_setup/heat/template.py index 28c20b72..cc652fb4 100755 --- a/utils/infra_setup/heat/template.py +++ b/utils/infra_setup/heat/template.py @@ -114,9 +114,9 @@ class HeatStack(HeatObject): i += 1 if self.uuid is not None: - sys.exit("delete stack failed!!!") + sys.exit("delete stack failed!!!") else: - HeatStack.stacks.remove(self) + HeatStack.stacks.remove(self) @staticmethod def delete_all(): diff --git a/utils/tools/bottlenecks_image_modify b/utils/tools/bottlenecks_image_modify new file mode 100755 index 00000000..f232fe73 --- /dev/null +++ b/utils/tools/bottlenecks_image_modify @@ -0,0 +1,172 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# bottlenecks: this file is copied from yardstick and slightly modified +############################################################################## + +# bottlenecks-img-modify - download and modify a Ubuntu cloud image +# +# The actual customization is done by a script passed with an absolute path as +# the only single argument. The command needs to be invoked as sudo +# +# Example invocation: +# bottlenecks-img-modify /home/opnfv/bottlenecks/utils/tools/ubuntu-server-cloudimg-modify.sh +# +# Warning: the script will create files by default in: +# /tmp/workspace/bottlenecks +# the files will be owned by root! +# +# TODO: image resize is needed if the base image is too small +# + +set -ex + +die() { + echo "error: $1" >&2 + exit 1 +} + +test $# -eq 1 -o $# -eq 2 || die "no image specific script as argument" +test $(id -u) -eq 0 || die "should invoke using sudo" + +cmd=$1 +RELEASE=$2 +test -x $cmd +mountdir="/mnt/bottlenecks" +workspace=${WORKSPACE:-"/tmp/workspace/bottlenecks"} +host=${HOST:-"cloud-images.ubuntu.com"} +release=${RELEASE:-"xenial"} +image_path="${release}/current/${release}-server-cloudimg-amd64-disk1.img" +image_url=${IMAGE_URL:-"https://${host}/${image_path}"} +md5sums_path="${release}/current/MD5SUMS" +md5sums_url=${MD5SUMS_URL:-"https://${host}/${md5sums_path}"} + +imgfile="${workspace}/bottlenecks-image.img" +raw_imgfile="${workspace}/bottlenecks-${release}-server.raw" +filename=$(basename $image_url) + +# download and checksum base image, conditionally if local copy is outdated +download() { + test -d $workspace || mkdir -p $workspace + cd $workspace + rm -f MD5SUMS # always download the checksum file to a detect stale image + wget $md5sums_url + test -e $filename || wget -nc --progress=dot:giga $image_url + grep $filename MD5SUMS | md5sum -c || + if [ $? -ne 0 ]; then + rm $filename + wget -nc --progress=dot:giga $image_url + grep $filename MD5SUMS | md5sum -c + fi + + for i in $(seq 0 9); do + [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i + done + + qemu-img convert $filename $raw_imgfile + cd - +} + +# mount image +setup() { + # qemu-img resize $raw_imgfile +5GB + mkdir -p $mountdir + + loopdevice=$(kpartx -l $raw_imgfile | head -1 | cut -f1 -d ' ') + + kpartx -av $raw_imgfile + # for trouble shooting + sleep 2 + dmsetup ls + fdisk -l /dev/${loopdevice:0:5} || true + mount /dev/mapper/$loopdevice $mountdir + mount -t proc none $mountdir/proc + + cp $cmd $mountdir/$(basename $cmd) +} + +# modify image running a script using in a chrooted environment +modify() { + # resolv.conf does not exist in base image, pass nameserver value from host + nameserver_ip=$(grep -m 1 '^nameserver' \ + /etc/resolv.conf | awk '{ print $2 '}) + + # prevent init scripts from running during install + echo $'#!/bin/sh\nexit 101' >$mountdir/usr/sbin/policy-rc.d + chmod a+x $mountdir/usr/sbin/policy-rc.d + + chroot $mountdir /$(basename $cmd) $nameserver_ip + + rm -rf $mountdir/usr/sbin/policy-rc.d + + umount -f $mountdir/proc + umount $mountdir + + qemu-img convert -c -o compat=0.10 -O qcow2 $raw_imgfile $imgfile + + if dmsetup table | grep $loopdevice; then + dmsetup clear $loopdevice || true + fi +} + +# cleanup (umount) the image +cleanup() { + # designed to be idempotent + mount | grep $mountdir/proc && umount $mountdir/proc + mount | grep $mountdir && umount $mountdir + mount | grep "/mnt/vivid" && umount "/mnt/vivid" + + if [ -f $raw_imgfile ]; then + #kpartx -dv $raw_imgfile sometimes failed, we should checked it agein. + #if [ -z "$(kpartx -l $raw_imgfile | grep 'loop deleted')" ]; then + # kpartx -dv $raw_imgfile + #fi + kpartx -dv $raw_imgfile || true + fi + + rm -f $raw_imgfile + rm -rf $mountdir +} + +exitcode="" +error_trap() +{ + local rc=$? + + set +e + + if [ -z "$exitcode" ]; then + exitcode=$rc + fi + + dmesg -T | tail -50 + + cleanup + + echo "Image build failed with $exitcode" + + exit $exitcode +} + +main() { + cleanup + + trap "error_trap" EXIT SIGTERM + + download + setup + modify + + trap - EXIT SIGTERM + cleanup + + echo "the modified image is found here: $imgfile" +} + +main diff --git a/utils/tools/ubuntu-server-clouding-modify.sh b/utils/tools/ubuntu-server-clouding-modify.sh new file mode 100755 index 00000000..505825a1 --- /dev/null +++ b/utils/tools/ubuntu-server-clouding-modify.sh @@ -0,0 +1,43 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# bottlenecks: this file is copied from yardstick and slightly modified +############################################################################## + +# installs required packages +# must be run from inside the image (either chrooted or running) + +set -ex + +if [ $# -eq 1 ]; then + nameserver_ip=$1 + + # /etc/resolv.conf is a symbolic link to /run, restore at end + rm /etc/resolv.conf + echo "nameserver $nameserver_ip" > /etc/resolv.conf + echo "nameserver 8.8.8.8" >> /etc/resolv.conf + echo "nameserver 8.8.4.4" >> /etc/resolv.conf +fi + +# Add hostname to /etc/hosts. +# Allow console access via pwd +cat <<EOF >/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg +manage_etc_hosts: True +password: RANDOM +chpasswd: { expire: False } +ssh_pwauth: True +EOF + +#install some software +apt-get update +apt-get install -y \ + netperf \ + sysstat + +# restore symlink +ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf |