diff options
20 files changed, 663 insertions, 303 deletions
diff --git a/ci/docker/Dockerfile b/ci/docker/Dockerfile index 217c1361..2c243792 100644 --- a/ci/docker/Dockerfile +++ b/ci/docker/Dockerfile @@ -8,6 +8,7 @@ ############################################################################## FROM ubuntu:14.04 +MAINTAINER MatthewLi <matthew.lijun@huawei.com> LABEL image=opnfv/bottlenecks @@ -20,16 +21,19 @@ ENV RELENG_REPO_DIR ${REPOS_DIR}/releng #new test suite required packages can be added here RUN apt-get update && apt-get install -y \ + libffi-dev \ + libssl-dev \ + libxml2-dev \ + libxslt1-dev \ curl \ wget \ git \ sshpass \ python \ - python-glanceclient \ - python-heatclient \ - python-keystoneclient \ - python-neutronclient \ - python-novaclient + python-dev \ + python-pip \ + python-setuptools && \ + easy_install -U setuptools RUN apt-get -y autoremove && \ apt-get clean @@ -37,7 +41,8 @@ RUN apt-get -y autoremove && \ RUN mkdir -p ${REPOS_DIR} RUN git config --global http.sslVerify false -# TODO fix code clone handler -#RUN git clone https://gerrit.opnfv.org/gerrit/bottlenecks ${BOTTLENECKS_REPO_DIR} -#RUN git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR} +RUN git clone https://gerrit.opnfv.org/gerrit/bottlenecks ${BOTTLENECKS_REPO_DIR} +RUN git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO_DIR} +RUN easy_install pytz +RUN pip install -r ${REPOS_DIR}/bottlenecks/requirements.txt diff --git a/config/bottlenecks_cfg.yaml b/config/bottlenecks_cfg.yaml index 8193651f..c12ec9d6 100755 --- a/config/bottlenecks_cfg.yaml +++ b/config/bottlenecks_cfg.yaml @@ -1 +1 @@ -suite_prefix_config: /home/opnfv/bottlenecks/testsuites/ +suite_prefix_config: /home/opnfv/bottlenecks/testsuites diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e07a0c65 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,79 @@ +############################################################################## +# Copyright (c) 2016 Huawei Tech. and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +appdirs==1.4.0 +Babel==2.2.0 +backport-ipaddress==0.1 +cliff==2.0.0 +cmd2==0.6.8 +coverage==4.1b2 +debtcollector==1.3.0 +ecdsa==0.13 +extras==0.0.3 +fixtures==1.4.0 +flake8==2.5.4 +funcsigs==0.4 +functools32==3.2.3.post2 +futures==3.0.5 +iso8601==0.1.11 +Jinja2==2.8 +jsonpatch==1.13 +jsonpointer==1.10 +jsonschema==2.5.1 +keystoneauth1==2.3.0 +linecache2==1.0.0 +lxml==3.5.0 +MarkupSafe==0.23 +mccabe==0.4.0 +mock==1.3.0 +monotonic==1.0 +msgpack-python==0.4.7 +netaddr==0.7.18 +netifaces==0.10.4 +nose==1.3.7 +openstacksdk==0.8.1 +os-client-config==1.16.0 +oslo.config==3.9.0 +oslo.i18n==3.4.0 +oslo.serialization==2.4.0 +oslo.utils==3.7.0 +paramiko==1.16.0 +pbr==1.8.1 +pep8==1.7.0 +positional==1.0.1 +prettytable==0.7.2 +pycrypto==2.6.1 +pyflakes==1.0.0 +pyparsing==2.1.0 +pyrsistent==0.11.12 +python-cinderclient==1.6.0 +python-glanceclient==2.0.0 +python-heatclient==1.0.0 +python-keystoneclient==2.3.1 +python-mimeparse==1.5.1 +python-neutronclient==4.1.1 +python-novaclient==3.3.0 +python-openstackclient==2.2.0 +python-subunit==1.2.0 +python-swiftclient==3.0.0 +pytz==2015.7 +PyYAML==3.11 +requests==2.9.1 +requestsexceptions==1.1.3 +scp==0.10.2 +simplejson==3.8.2 +six==1.10.0 +stevedore==1.12.0 +testrepository==0.0.20 +testtools==2.0.0 +traceback2==1.4.0 +unicodecsv==0.14.1 +unittest2==1.1.0 +warlock==1.2.0 +wrapt==1.10.6 diff --git a/run_tests.sh b/run_tests.sh index e6e90846..093f7248 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -33,10 +33,10 @@ function check_testcase(){ check_suite="$1" case $check_suite in "-rubbos") - $SUITE_PREFIX=$SUITE_PREFIX_CONFIG/rubbos/testcase_cfg + SUITE_PREFIX=$SUITE_PREFIX_CONFIG/rubbos/testcase_cfg ;; "-vstf") - $SUITE_PREFIX=$SUITE_PREFIX_CONFIG/vstf/testcase_cfg + SUITE_PREFIX=$SUITE_PREFIX_CONFIG/vstf/testcase_cfg ;; esac @@ -47,10 +47,10 @@ function check_testcase(){ all_testcases_insuite=`cat /tmp/all_testcases.yaml | awk -F '/' '{print $NF}' | awk -F '.' '{print $1}'` all_testcases=(${all_testcases_insuite}) - if ["${TEST_CASE}" != ""]; then + if [ "${TEST_CASE}" != "" ]; then testcase_exec=(${TEST_CASE// /}) for i in "${testcase_exec[@]}"; do - if [[ " ${all_test_cases[*]} " != *" $i "* ]]; then + if [[ " ${all_testcases[*]} " != *" $i "* ]]; then error "unknown test case: $i. available test cases are: ${all_test_cases[@]}" fi done @@ -71,7 +71,7 @@ function run_test(){ if [[ -f $test_file ]]; then testcases=($(cat $test_file)) else - error("no rubbos test suite file ") + error "no rubbos test suite file" fi for i in "${testcases[@]}"; do #check if the testcase is legal or not @@ -79,7 +79,7 @@ function run_test(){ #adjust config parameters, different test suite has different methods, take rubbos as an example #run test case, different test suite has different methods file={$BASEDIR}/testsuites/rubbos/testcase_cfg/{$i}.yaml - python ${BOTTLENECK_TOP_DIR}/testsuites/rubbos/run_rubbos.py -c $file + python /home/opnfv/bottlenecks/testsuites/rubbos/run_rubbos.py -c $file done ;; "vstf") @@ -88,7 +88,7 @@ function run_test(){ if [[ -f $test_file ]]; then testcases=($(cat $test_file)) else - error("no vstf test suite file ") + error "no vstf test suite file " fi for i in "${testcases[@]}"; do #check if the testcase is legal or not diff --git a/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_off.pp b/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_off.pp index b1d0b3db..aeb1f12f 100644 --- a/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_off.pp +++ b/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_off.pp @@ -39,10 +39,10 @@ class rubbos_client::rubbos_client_off { backup => false, } - # bench/rubbos.properties.template - file {'${rubbos_home}/bench/rubbos.properties.template': + # Client/rubbos.properties.template + file {'${rubbos_home}/Client/rubbos.properties.template': ensure => absent, - path => "${rubbos_home}/bench/rubbos.properties.template", + path => "${rubbos_home}/Client/rubbos.properties.template", show_diff => false, backup => false, } diff --git a/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_on.pp b/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_on.pp index 9bdf0724..7cda7e14 100644 --- a/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_on.pp +++ b/testsuites/rubbos/puppet_manifests/modules/rubbos_client/manifests/rubbos_client_on.pp @@ -119,16 +119,17 @@ class rubbos_client::rubbos_client_on { backup => false, } - # rubbos.properties.template - file {'${rubbos_home}/bench/rubbos.properties.template': + # Client/rubbos.properties.template + file {'${rubbos_home}/Client/rubbos.properties.template': ensure => file, - path => "${rubbos_home}/bench/rubbos.properties.template", + path => "${rubbos_home}/Client/rubbos.properties.template", source => "puppet:///modules/rubbos_client/rubbos.properties.template", show_diff => false, backup => false, + require => Exec['tar xvzf ${rubbos_home}/Client.tar.gz'], } - # executable scripts + # bench/executable scripts file {'${rubbos_home}/bench/run_emulator.sh': ensure => file, path => "${rubbos_home}/bench/run_emulator.sh", @@ -153,7 +154,6 @@ class rubbos_client::rubbos_client_on { File['${rubbos_home}/Makefile'], File['${rubbos_home}/config.mk'], Exec['tar xvzf ${rubbos_home}/Client.tar.gz']], - #File['${rubbos_home}/Client']], } exec {'ant jar': diff --git a/testsuites/rubbos/run_rubbos.py b/testsuites/rubbos/run_rubbos.py index d487313a..c17f0d3f 100755 --- a/testsuites/rubbos/run_rubbos.py +++ b/testsuites/rubbos/run_rubbos.py @@ -7,13 +7,18 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## + + import os import argparse import time -import heatclient -import keystoneclient -import glanceclient -import novaclient +import logging +import urllib2 +import shutil +from heatclient.client import Client as HeatClient +from keystoneclient.v2_0.client import Client as KeystoneClient +from glanceclient.v2.client import Client as GlanceClient +from novaclient.client import Client as NovaClient #------------------------------------------------------ # parser for configuration files in each test case @@ -21,17 +26,17 @@ import novaclient parser = argparse.ArgumentParser() parser.add_argument("-c", "--conf", help="configuration files for the testcase, in yaml format", - default="rubbos_1-1-1.yaml") + default="/tmp/rubbos_1-1-1.yaml") args = parser.parse_args() #-------------------------------------------------- # logging configuration #-------------------------------------------------- -logger = logging.getLogger('run_rubbos') -logger.setLevel(logging.DEBUG) +logger = logging.getLogger(__name__) + def _get_keystone_client(): - keystone_client = keystoneclient.v2_0.client.Client( + keystone_client = KeystoneClient( auth_url=os.environ.get('OS_AUTH_URL'), username=os.environ.get('OS_USERNAME'), password=os.environ.get('OS_PASSWORD'), @@ -42,116 +47,189 @@ def _get_keystone_client(): def _get_heat_client(): keystone = _get_keystone_client() heat_endpoint = keystone.service_catalog.url_for(service_type='orchestration') - heat_client = heatclient.client.Client('1', endpoint=heat_endpoint, token=keystone.auth_token) - + heat_client = HeatClient('1', endpoint=heat_endpoint, token=keystone.auth_token) return heat_client def _get_glance_client(): keystone = _get_keystone_client() - glance_endpoint = keystone.service_catalog.url_for(service_type='image', - endpoint_type='publicURL') - return glanceclient.v2.client.Client(glance_endpoint, token=keystone.auth_token) + glance_endpoint = keystone.service_catalog.url_for(service_type='image', endpoint_type='publicURL') + return GlanceClient(glance_endpoint, token=keystone.auth_token) def _get_nova_client(): - nova_client = novaclient.client.Client("2", auth_url=os.environ.get('OS_AUTH_URL'), - username=os.environ.get('OS_USERNAME'), - password=os.environ.get('OS_PASSWORD'), - tenant_name=os.environ.get('OS_TENANT_NAME'), - region_name=os.environ.get('OS_REGION_NAME'), - cacert=os.environ.get('OS_CACERT')) + nova_client = NovaClient("2", os.environ.get('OS_USERNAME'), + os.environ.get('OS_PASSWORD'), + os.environ.get('OS_TENANT_NAME'), + os.environ.get('OS_AUTH_URL')) return nova_client -def download_url(url, dest_path): - """ - Download a file to a destination path given a URL - """ - name = url.rsplit('/')[-1] - dest = dest_path + "/" + name +def _download_url(src_url, dest_dir): + ''' Download a file to a destination path given a URL''' + file_name = src_url.rsplit('/')[-1] + dest = dest_dir + "/" + file_name try: - response = urllib2.urlopen(url) + response = urllib2.urlopen(src_url) except (urllib2.HTTPError, urllib2.URLError): - return False + return None with open(dest, 'wb') as f: shutil.copyfileobj(response, f) - return True + return file_name + + +def rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status="CREATE_COMPLETE"): + heat = _get_heat_client() + for stack in heat.stacks.list(): + if status == None and stack.stack_name == name: + # Found target stack + print "Found stack, name=" + stack.stack_name + return True + elif stack.stack_name == name and stack.stack_status==status: + print "Found stack, name=" + stack.stack_name + ", status=" + stack.stack_status + return True + return False def rubbos_env_prepare(template=None): - """ Prepare for rubbos running env - """ - #logger.info("generate heat template for the testcase based on template '%s'." % template) - pass + print "========== Prepare rubbos environment ==========" + logger.info("Generate heat template for the testcase based on template '%s'." % template) def rubbos_env_cleanup(): + print "========== Cleanup rubbos environment ==========" glance = _get_glance_client() heat = _get_heat_client() nova = _get_nova_client() - for stack in self.heat.stacks.list(): - heat.stacks.delete(stack.id) + for image in glance.images.list(): + if image.name.find("bottlenecks_rubbos") >= 0: + print "Delete image, id:" + str(image.id) + ", name:" + str(image.name) + glance.images.delete(image.id) + + for keypair in nova.keypairs.list(): + if keypair.name.find("bottlenecks_rubbos") >= 0: + print "Delete keypair, id:" + keypair.id + ", name:" + keypair.name + nova.keypairs.delete(keypair.id) - for image in self.glance.images.list(): - glance.images.delete(image.id) + for flavor in nova.flavors.list(): + if flavor.name.find("bottlenecks_rubbos") >= 0: + print "Delete flavor, id:" + str(flavor.id) + ", name:" + flavor.name + nova.flavors.delete(flavor.id) - for keypair in self.nova.keypairs.list(): - nova.keypairs.delete(keypair.id) + for stack in heat.stacks.list(): + if stack.stack_name.find("bottlenecks_rubbos") >= 0: + print "Delete stack, id: " + str(stack.id) + ", name:" + str(stack.stack_name + heat.stacks.delete(stack.id) - for flavor in self.nova.flavors.list(): - nova.flavors.delete(flavor.id) + timeInProgress = 0 + while rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status=None) and timeInProgress < 60: + time.sleep(5) + timeInProgress = timeInProgress + 5 - logger.info("openstack env cleanup") + if rubbos_stack_satisfy(name="bottlenecks_rubbos_stack", status=None) == True: + print "Failed to clean the stack" + return False + else: + return True -def rubbos_load_image(name=None): - file_url = '/tmp' - download_url(image_url,file_url) +def rubbos_create_images(src_url=None, image_name="bottlenecks_rubbos_image"): + print "========== Create rubbos image in OS ==========" + dest_dir = '/tmp' + file_name = _download_url(src_url, dest_dir) + #file_name = "bottlenecks-trusty-server.img" + if file_name == None: + return False glance = _get_glance_client() - image_args = {'name': name, - 'disk-format': 'qcow2', - 'container_format': 'bare', - 'file': '/tmp/bottlenecks-trusty-server.img'} - image = glance.images.create(**image_args) - if not (image.id): - logger.error("failed to upload rubbos image to openstack") - exit(-1) + imagefile = dest_dir + "/" + file_name + image = glance.images.create(name=image_name, disk_format="qcow2", container_format="bare") + with open(imagefile) as fimage: + glance.images.upload(image.id, fimage) + + timeInQueue = 0 + img_status = image.status + while img_status == "queued" and timeInQueue < 30: + print " image's status: " + img_status + time.sleep(1) + timeInQueue = timeInQueue + 1 + img_status = glance.images.get(image.id).status + + print "After %d seconds, the image's status is [%s]" %(timeInQueue, img_status) + return True if img_status == "active" else False + +def rubbos_create_keypairs(key_path, name="bottlenecks_rubbos_keypair"): + print "========== Add rubbos keypairs in OS ==========" + nova = _get_nova_client() + with open(key_path) as pkey: + nova.keypairs.create(name=name, public_key=pkey.read()) + +def rubbos_create_flavors(name="bottlenecks_rubbos_flavor", ram=4096, vcpus=2, disk=10): + print "========== Create rubbos flavors in OS ==========" + nova = _get_nova_client() + nova.flavors.create(name=name, ram=ram, vcpus=vcpus, disk=disk) -def rubbos_create_instance(template_file=None, rubbos_parameters=None): +def rubbos_create_instance(template_file, rubbos_parameters=None, stack_name="bottlenecks_rubbos_stack"): + print "========== Create rubbos instances ==========" heat = _get_heat_client() - template = open(template_file, 'r') - rubbos_stack = heat.stacks.create(stack_name='rubbos', template=template.read(), parameters=rubbos_parameters) - uid = rubbos_stack['stack']['id'] -def rubbos_stack_check(stack_name=None): - for stack in heat.stacks.list(): - if stack.stack_name == stack_name: - return stack.stack_status - return 'NOT_FOUND' + with open(template_file) as template: + stack = heat.stacks.create(stack_name=stack_name, template=template.read(), parameters=rubbos_parameters) + + stack_id = stack['stack']['id'] + stack_status = heat.stacks.get(stack_id).stack_status + + print "Created stack, id=" + stack_id + ", status=" + stack_status + + timeInProgress= 0 + while stack_status == "CREATE_IN_PROGRESS" and timeInProgress < 150: + print " stack's status: %s, after %d seconds" %(stack_status, timeInProgress) + time.sleep(5) + timeInProgress = timeInProgress + 5 + stack_status = heat.stacks.get(stack_id).stack_status + + print "After %d seconds, the stack's status is [%s]" %(timeInProgress, stack_status) + return True if stack_status == "CREATE_COMPLETE" else False def rubbos_run(): pass def main(): global Heat_template + global Bottlenecks_repo_dir global image_url - + Bottlenecks_repo_dir = "/tmp/opnfvrepo/bottlenecks" + #Bottlenecks_repo_dir = "/root/wyg/bottlenecks" # Test dir in local env + image_url = 'http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img' if not (args.conf): - logger.error("configuration files are not set for testcase") + logger.error("Configuration files are not set for testcase") exit(-1) else: Heat_template = args.conf - parameters={'image': 'rubbos', - 'key_name': 'rubbos-key', - 'public_net': 'net04_ext'} + parameters={'image': 'bottlenecks_rubbos_image', + 'key_name': 'bottlenecks_rubbos_keypair', + 'flavor': 'm1.small', + 'public_net': 'ext-net'} + + print "Heat_template_file: " + Heat_template + print "parameters:\n" + str(parameters) + + image_created = False + stack_created = False rubbos_env_prepare(Heat_template) rubbos_env_cleanup() - rubbos_load_image(name='rubbos') - rubbos_create_instance(template_file=Heat_template, rubbos_paramters=parameters) - time.sleep(400) - rubbos_stack_check(stack_name='rubbos') + + image_created = rubbos_create_images(image_url) + keyPath = Bottlenecks_repo_dir + "/utils/infra_setup/bottlenecks_key/bottlenecks_key.pub" + rubbos_create_keypairs(key_path=keyPath) + rubbos_create_flavors() + + if image_created == True: + stack_created = rubbos_create_instances(template_file=Heat_template, rubbos_parameters=parameters, stack_name="bottlenecks_rubbos_stack") + else: + print "Cannot create instances, as Failed to create image(s)." + exit (-1) + rubbos_run() rubbos_env_cleanup() diff --git a/testsuites/rubbos/run_rubbos.sh b/testsuites/rubbos/run_rubbos.sh deleted file mode 100755 index b7492adc..00000000 --- a/testsuites/rubbos/run_rubbos.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -############################################################################### -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -usage="Script to run the tests in rubbos. - -usage: - bash $(basename "$0") [-h|--help] [-s <test suite>] - -where: - -h|--help show the help text - -t|--test run specifif test case scenario - <test case> examples: - rubbos_1-1-1, rubbos_1-2-1 - -examples: - $(basename "$0") - $(basename "$0") -t rubbos_1-1-1" - -while [[ $# > 0 ]] - do - key="$1" - case $key in - -h|--help) - echo "$usage" - exit 0 - shift - ;; - -t|--test) - CASE="$2" - shift - ;; - *) - echo "unkown option $1 $2" - exit 1 - ;; - esac - shift -done - -BASEDIR=`dirname $0` - -#has been checked in upper layer run_tests.sh -#run tests -if [ "${CASE}" != "" ]; then - case_exec=(${CASE//,/}) - for case_exe in "${case_exec[@]}"; do - client_num=4 - tomcat_num=$(echo "$case_exe"| awk -F '-' '{print $2}') - mysql_num=$(echo "$case_exe"| awk -F '-' '{print $3') - - hosts=(rubbos_control rubbos_benchmark rubbos_httpd) - for((i=1; i <= client_num; i++)); do - hosts=(${hosts[*]} client$i) - done - for((i=1; i <= tomcat_num; i++)); do - hosts=(${hosts[*]} tomcat$i) - done - for((i=1; i <= mysql_num; i++)); do - hosts=(${hosts[*]} mysql$i) - done - bash $BOTTLENECKS_TOP_DIR/utils/infra_setup/heat_template/rubbos_heat_template/HOT_create_instance.sh - done -fi diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py index 7a7144f8..d511a08d 100644 --- a/utils/dashboard/process_data.py +++ b/utils/dashboard/process_data.py @@ -10,22 +10,66 @@ import subprocess as subp import sys -from collector import Collector +from rubbos_collector import RubbosCollector from uploader import Uploader +def printUsage(): + print "Usage: python process_data.py required_params(**) optional_params([])" + print " ** -i|--input input_data_dir" + print " ** -s|--suite suite_name" + print " ** -c|--conf conf_file" + print " [] -o|--output output_file" + print " [] -u|--upload yes|no" -#process data -if len(sys.argv)!=3: - print "Wrong arguments, please input 2 parameters, 1st as raw data path; "\ - "2nd as config yaml!!" - exit (1) -data_home = sys.argv[1] -conf = sys.argv[2] +def process(input_dir, suite_name): + result = dict() + if suite_name == "rubbos": + result = RubbosCollector().collect_data(input_dir) + return result -#1collect result -result = Collector().collect_data(data_home) -print "Result collected:\n%s" % result +def writeResult(output_file, result): + f = open(output_file, "w") + if isinstance(result, list): + for elem in result: + f.write(str(elem) + "\n") + f.close() -#2upload result -Uploader(conf).upload_result("rubbos", result) +def uploadResult(conf, suite_name, result): + Uploader(conf).upload_result(suite_name, result) +def main(): + if len(sys.argv) < 7 or len(sys.argv) % 2 == 0: + printUsage() + exit (1) + i = 1 + params = dict() + while (i < len(sys.argv)): + print sys.argv[i] + if sys.argv[i]=="-i" or sys.argv[i]=="--input": + params["input"] = sys.argv[i+1] + if sys.argv[i]=="-s" or sys.argv[i]=="--suite": + params["suite"] = sys.argv[i+1] + if sys.argv[i]=="-c" or sys.argv[i]=="--conf": + params["conf"] = sys.argv[i+1] + if sys.argv[i]=="-o" or sys.argv[i]=="--output": + params["output"] = sys.argv[i+1] + if sys.argv[i]=="-u" or sys.argv[i]=="--upload": + params["upload"] = sys.argv[i+1] + i = i+2 + if not(params.has_key("input") and params.has_key("suite") and params.has_key("conf")): + print "Lack some required parameters." + exit (1) + + result = process(params["input"], params["suite"]) + print "Results:" + for elem in result: + print elem + + if params.has_key("output"): + writeResult(params["output"],result) + + if params.has_key("upload") and params["upload"].lower()=="yes": + uploadResult(params["conf"], params["suite"], result) + +if __name__=="__main__": + main() diff --git a/utils/dashboard/collector.py b/utils/dashboard/rubbos_collector.py index 1687f80f..c9851739 100755 --- a/utils/dashboard/collector.py +++ b/utils/dashboard/rubbos_collector.py @@ -14,33 +14,36 @@ def exec_shell(cmd): out,err = subp.Popen(cmd, stdout=subp.PIPE, shell=True).communicate() return out.strip() - def get_onetime_data(dir_name): cmd = "grep -in 'remote client nodes' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name client_node_num = int(exec_shell(cmd)) cmd = "grep -n 'Number of clients' %s/index.html|awk '{print $5}'|awk -F '<' '{print $1}'" % dir_name each_client_num = int(exec_shell(cmd)) total_client = (client_node_num+1) * each_client_num - cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'FNR%%4==0 {printf "%%s\\n", $3 }\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + + cmd = 'grep -n "throughput" %s/stat_client*.html |awk -F "<B>" \'{if (FNR%%2==0 && FNR%%4!=0) {printf "%%s\\n", $3}}\'|awk \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name throughput = int(exec_shell(cmd)) - return total_client, throughput + cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $4}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + request = int(exec_shell(cmd)) + + cmd = 'grep -n "Total" %s/stat_client*.html |awk -F "<B>" \'{if (FNR==4) {printf "%%s\\n", $5}}\'|awk -F "</B>" \'BEGIN{sum=0;}{sum=sum+$1;}END{print sum}\'' % dir_name + error_request = int(exec_shell(cmd)) + return total_client, throughput, request, error_request -class Collector(object): +class RubbosCollector(object): def __init__(self): pass - def collect_data(self, data_home): cmd = 'ls -l %s |grep ^d|awk \'{print $9}\'' % data_home result = [] for subdir in exec_shell(cmd).split('\n'): - total_client, throughput = get_onetime_data(data_home+'/'+subdir) - result.append({'client':total_client, 'throughput':throughput}) - result.sort(key=lambda x:x['client']) + total_client, throughput, request, error_request = get_onetime_data(data_home+'/'+subdir) + result.append({'client':total_client, 'throughput':throughput, 'request':request, 'error_request':error_request}) + result.sort(key=lambda x:x['client']) return result; - diff --git a/utils/dev_env/deploy/create_libvirt_vm.sh b/utils/dev_env/deploy/create_libvirt_vm.sh new file mode 100755 index 00000000..45ba2b8e --- /dev/null +++ b/utils/dev_env/deploy/create_libvirt_vm.sh @@ -0,0 +1,237 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +############################################## +# Usage: ./deploy.sh paras_conf outout_dir +############################################## +SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd) +tool_dir=${0%/*} + +function print_usage() +{ + echo "Usage: ./deploy.sh paras_conf output_dir" +} + +## sanity check +if [ ! -f $tool_dir/mac_generator.sh ] || \ + [ ! -f $tool_dir/libvirt_template.xml ] || \ + [ ! -f $tool_dir/meta-data.template ] || \ + [ ! -f $tool_dir/user-data.template ] || \ + [ ! -f $tool_dir/p-master-user-data.template ] || \ + [ ! -f $tool_dir/p-agent-user-data.template ]; then + echo "Lack some necessary files for this tool!" + echo "deploy.sh" + echo "mac_generator.sh" + echo "1 xml, and 4 template" + exit 1 +fi + +## Check input +if [ $# != 2 ]; then + print_usage + exit 1 +fi +if [ ! -f $1 ]; then + echo "Cannot find file: "$1 + exit 1 +fi +if [ -d $2 ]; then + echo "Ouput dir $2 exist!" + exit 1 +fi + +## Assign parameters +host_names="" +puppet_enable="true" +master_host="" +vm_mem= +vm_cpu_cores= +image_url= +image_name= +ipaddr_start= +trusted_ssh_pub_keys_file= +while read line +do + line=(${line//=/ }) + case ${line[0]} in + "host_names" ) + host_names=${line[1]} + ;; + "puppet_enable" ) + puppet_enable=${line[1]} + ;; + "master_host" ) + master_host=${line[1]} + ;; + "vm_mem" ) + vm_mem=${line[1]} + ;; + "vm_cpu_cores" ) + vm_cpu_cores=${line[1]} + ;; + "image_url" ) + image_url=${line[1]} + ;; + "image_name" ) + image_name=${line[1]} + ;; + "ipaddr_start" ) + ipaddr_start=${line[1]} + ;; + "trusted_ssh_pub_keys_file" ) + trusted_ssh_pub_keys_file=${line[1]} + ;; + esac +done < $1 + +echo "puppet_enable="$puppet_enable + +# Check parameters in conf file +if [ $puppet_enable == "true" ] ; then + if [ ${#master_host} == 0 ];then + echo "Should specify master_host!" + exit 1 + else + result=$(echo ${host_names} | grep "${master_host}") + if [ ${result} == "" ]; then + echo "Specified master_host is invalid!" + exit 1 + fi + fi +fi + +# Define and Prepare needed data +mac_arr= +hostname_arr=(${host_names//,/ }) +virt_num=${#hostname_arr[@]} +ip_arr=() +replaced_hosts="" +replaced_ssh_keys="" +output_dir=$2 +work_dir= +host_vm_dir= +cache_dir= +function init(){ + # Generate mac address + local mac_generator=${tool_dir}/mac_generator.sh + chmod +x $mac_generator + mac_str=$($mac_generator $virt_num) + mac_arr=($mac_str) + + # Generate hosts info + local ip="" + i=0 + for host in "${hostname_arr[@]}"; do + ip=${ipaddr_prefix}$((i+$ipaddr_idx)) + ip_arr+=($ip) + # Note the format, especially the space + replaced_hosts=${replaced_hosts}" "${ip}" "${host}"\n" + let i=i+1 + done + + # Generate ssh public keys + echo "## trusted_ssh_pub_keys_file --> "${trusted_ssh_pub_keys_file} + if [ ${trusted_ssh_pub_keys_file} != "" ] && [ -f ${trusted_ssh_pub_keys_file} ]; then + while read line + do + # Note the format, especially space + replaced_ssh_keys=${replaced_ssh_keys}" - "${line}"\n" + done < ${trusted_ssh_pub_keys_file} + # delete last "\n" in replaced_ssh_keysi + replaced_ssh_keys=${replaced_ssh_keys%\\n} + fi + + # Prepare needed folder and files + if [ ${output_dir:0-1} == "/" ]; then + output_dir=${output_dir%/*} + fi + output_file=$output_dir/hosts.info + mkdir -p $output_dir + touch $output_file + echo "## Output host_info file --> "$output_file + + work_dir=$output_dir/work + host_vm_dir=$work_dir/vm + cache_dir=$work_dir/cache + mkdir -p $work_dir + mkdir -p $host_vm_dir + mkdir -p $cache_dir + + # Cache img file + echo "## Cache img file" + curl --connect-timeout 10 -o ${cache_dir}/$image_name $image_url +} + +# Bring up instances/vms +sub_ip_arr=(${ipaddr_start//./ }) +ipaddr_prefix=${sub_ip_arr[0]}"."${sub_ip_arr[1]}"."${sub_ip_arr[2]}"." +ipaddr_idx=${sub_ip_arr[3]} +function bring_up() { + i=0 + while (($i < $virt_num)) + do + echo "Bring up a vm, hostname: ${hostname_arr[$i]}, ip: ${ip_arr[$i]}, mac: ${mac_arr[$i]}" + vm_dir=$host_vm_dir/${hostname_arr[$i]} + mkdir -p $vm_dir + cp ${cache_dir}/$image_name $vm_dir + + sed -e "s/REPLACE_IPADDR/${ip_arr[$i]}/g" \ + -e "s/REPLACE_GATEWAY/${ipaddr_prefix}1/g" \ + -e "s/REPLACE_HOSTNAME/${hostname_arr[$i]}/g" \ + ${tool_dir}/meta-data.template > ${cache_dir}/meta-data + + if [ ${puppet_enable} == "true" ]; then + # Use puppet user data + echo "hostname: "${hostname_arr[$i]} + if [ ${hostname_arr[$i]} == ${master_host} ]; then + cp ${tool_dir}/p-master-user-data.template ${cache_dir}/user-data.template + else + cp ${tool_dir}/p-agent-user-data.template ${cache_dir}/user-data.templatate + fi + sed -e "s#REPLACED_TRUSTED_PUB_SSH_KEYS#${replaced_ssh_keys}#g" \ + -e "s#REPLACED_HOSTS_INFO#${replaced_hosts}#g" \ + -e "s/REPLACED_PUPPET_MASTER_SERVER/${master_host}/g" \ + ${cache_dir}/user-data.template > ${cache_dir}/user-data + else + # Use common user data + echo "## Use common user-data.template" + cp ${tool_dir}/user-data.template ${cache_dir}/user-data + fi + + genisoimage -output seed.iso -volid cidata -joliet -rock ${cache_dir}/user-data ${cache_dir}/meta-data + mv seed.iso ${vm_dir}/ + # Create vm xml + sed -e "s/REPLACE_MEM/$vm_mem/g" \ + -e "s/REPLACE_CPU/$vm_cpu_cores/g" \ + -e "s/REPLACE_NAME/${hostname_arr[$i]}/g" \ + -e "s#REPLACE_IMAGE#$vm_dir/disk.img#g" \ + -e "s#REPLACE_SEED_IMAGE#$vm_dir/seed.iso#g" \ + -e "s/REPLACE_MAC_ADDR/${mac_arr[$i]}/g" \ + ${tool_dir}/libvirt_template.xml > ${vm_dir}/libvirt.xml + + echo "${ip_arr[$i]} ${hostname_arr[$i]}" >> $output_file + + echo "Will define xml from:"${vm_dir}"/libvirt.xml" + echo "start: "${hostname_arr[$i]} + sudo virsh define ${vm_dir}/libvirt.xml + sudo virsh start ${hostname_arr[$i]} + let i=i+1 + rm -rf ${cache_dir}/meta-data ${cache_dir}/user-data + done +} + +function clean(){ + rm -rf ${cache_dir} +} + +init +bring_up +clean diff --git a/utils/rubbos_dev_env_setup/libvirt_template.xml b/utils/dev_env/deploy/libvirt_template.xml index eddeff00..eddeff00 100644 --- a/utils/rubbos_dev_env_setup/libvirt_template.xml +++ b/utils/dev_env/deploy/libvirt_template.xml diff --git a/utils/rubbos_dev_env_setup/mac_generator.sh b/utils/dev_env/deploy/mac_generator.sh index ef78f5f6..f8bc5e08 100755 --- a/utils/rubbos_dev_env_setup/mac_generator.sh +++ b/utils/dev_env/deploy/mac_generator.sh @@ -1,6 +1,6 @@ #!/bin/bash ############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -8,8 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# this is a modified copy from compass4nfv/deploy/mac_generator.sh +## This script is used in 'create_libvirt_vm.sh' function mac_address_part() { hex_number=$(printf '%02x' $RANDOM) number_length=${#hex_number} @@ -33,4 +33,3 @@ for i in `seq $1`; do done echo ${machines} - diff --git a/utils/rubbos_dev_env_setup/meta-data_template b/utils/dev_env/deploy/meta-data.template index 43db6597..43db6597 100644 --- a/utils/rubbos_dev_env_setup/meta-data_template +++ b/utils/dev_env/deploy/meta-data.template diff --git a/utils/dev_env/deploy/p-agent-user-data.template b/utils/dev_env/deploy/p-agent-user-data.template new file mode 100644 index 00000000..3d778f1a --- /dev/null +++ b/utils/dev_env/deploy/p-agent-user-data.template @@ -0,0 +1,40 @@ +#cloud-config + +users: + - name: ubuntu + ssh-authorized-keys: +REPLACED_TRUSTED_PUB_SSH_KEYS + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + groups: sudo + shell: /bin/bash + +runcmd: + - restart ssh + - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb + - sudo dpkg -i puppetlabs-release-trusty.deb + - sudo apt-get update + - sudo apt-get -y install puppet + - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf + - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf + - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf + - sed -i '/^templatedir/d' /etc/puppet/puppet.conf + - sed -i 's/START=no/START=yes/g' /etc/default/puppet + +write_files: + - path: /etc/hosts + content: | + 127.0.0.1 localhost + + # The following lines are desirable for IPv6 capable hosts + ::1 ip6-localhost ip6-loopback + fe00::0 ip6-localnet + ff00::0 ip6-mcastprefix + ff02::1 ip6-allnodes + ff02::2 ip6-allrouters + ff02::3 ip6-allhosts + + # hosts info for all vms +REPLACED_HOSTS_INFO + +final_message: "The system with puppet is finally up, after $UPTIME seconds." diff --git a/utils/dev_env/deploy/p-master-user-data.template b/utils/dev_env/deploy/p-master-user-data.template new file mode 100644 index 00000000..446c9698 --- /dev/null +++ b/utils/dev_env/deploy/p-master-user-data.template @@ -0,0 +1,39 @@ +#cloud-config + +users: + - name: ubuntu + ssh-authorized-keys: +REPLACED_TRUSTED_PUB_SSH_KEYS + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCwmx8/86Vc1yqqplMPkm/w6wB2r5Yl6CjON85Ko5SzPEpnysJwO5ygbMClQTbzCUztIpFeO8D6xdWlhPmhMEJIWtRh+Dx0swB+ByMraGTP92xxKo2ukWzYI6eu25LmJX+a1lmVPmewQq+6dPzIwgCZvwD8UVe5L+d0EP2rb3iniQVoBzksUmhL3N6jp2Lgpscfi1yXFgjFdLpDsCcFcM+Q6wQUC2+qGBYa5RxqercWzISSbvAsYwC7UepmjFpcj3W/WX2QcOCDIlAzeURvC8PbOdzCtgeMhuhH2499BcIUSzPMpjzvpRiHr5SeF/zrPeW+2Ei1MfHi1qrBCxS8JBqt bottlenecks@bottlenecks.opnfv.org + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + groups: sudo + shell: /bin/bash + +runcmd: + - restart ssh + - wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb + - sudo dpkg -i puppetlabs-release-trusty.deb + - sudo apt-get update + - sudo apt-get -y install puppetserver + - sed -i '/^factpath/a basemodulepath=\/etc\/puppet\/modules' /etc/puppet/puppet.conf + - sed -i '/^factpath/a server=REPLACED_PUPPET_MASTER_SERVER' /etc/puppet/puppet.conf + - sed -i '/^factpath/a runinterval=1d' /etc/puppet/puppet.conf + - sed -i '/^templatedir/d' /etc/puppet/puppet.conf + +write_files: + - path: /etc/hosts + content: | + 127.0.0.1 localhost + + # The following lines are desirable for IPv6 capable hosts + ::1 ip6-localhost ip6-loopback + fe00::0 ip6-localnet + ff00::0 ip6-mcastprefix + ff02::1 ip6-allnodes + ff02::2 ip6-allrouters + ff02::3 ip6-allhosts + + # hosts info for all vms +REPLACED_HOSTS_INFO + +final_message: "The system with puppet is finally up, after $UPTIME seconds." diff --git a/utils/rubbos_dev_env_setup/user-data_template b/utils/dev_env/deploy/user-data.template index 77f616df..8692cde1 100644 --- a/utils/rubbos_dev_env_setup/user-data_template +++ b/utils/dev_env/deploy/user-data.template @@ -1,2 +1 @@ #cloud-config - diff --git a/utils/dev_env/paras.conf b/utils/dev_env/paras.conf new file mode 100644 index 00000000..1ebf104a --- /dev/null +++ b/utils/dev_env/paras.conf @@ -0,0 +1,20 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +## This is a sample configuration file, as one input parameter for 'create_libvirt_vm.sh' +host_names=test-master,test-agent +puppet_enable=true +master_host=test-master +vm_mem=4096 +vm_cpu_cores=4 +image_url=file:///home/ubuntu/trusty-server-cloudimg-amd64-btnks.img +image_name=disk.img +ipaddr_start=192.168.122.201 +trusted_ssh_pub_keys_file=/home/ubuntu/trusted_ssh_pub_keys diff --git a/utils/rubbos_dev_env_setup/deploy.sh b/utils/rubbos_dev_env_setup/deploy.sh deleted file mode 100755 index a4fbfb8b..00000000 --- a/utils/rubbos_dev_env_setup/deploy.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# this is a modified copy of compass4nfv/deploy/host_virtual.sh - -set -x - -SCRIPT_DIR=`cd ${BASH_SOURCE[0]%/*};pwd` -WORK_DIR=${SCRIPT_DIR}/work - -mkdir -p $WORK_DIR - -source ./env_config.sh - -host_vm_dir=$WORK_DIR/vm - - -function download_iso() -{ - mkdir -p ${WORK_DIR}/cache - curl --connect-timeout 10 -o ${WORK_DIR}/cache/$IMAGE_NAME $IMAGE_URL -} - - -function tear_down_machines() { - for i in $HOSTNAMES; do - echo "tear down machine:" $i - sudo virsh destroy $i - sudo virsh undefine $i - rm -rf $host_vm_dir/$i - done -} - - -function get_host_macs() { - local mac_generator=${SCRIPT_DIR}/mac_generator.sh - local machines= - - chmod +x $mac_generator - mac_array=$($mac_generator $VIRT_NUMBER) - machines=$(echo $mac_array) - - echo $machines -} - -function launch_host_vms() { - mac_array=($(get_host_macs)) - - echo ${mac_array[2]} - echo ${mac_array[*]} - - old_ifs=$IFS - IFS=, - tear_down_machines - echo "bringing up vms ${mac_array[*]}" - i=0 - for host in $HOSTNAMES; do - echo "creating vm disk for instance $host" \ - "ip ${IPADDR_PREFIX}$((i+11))" \ - "mac ${mac_array[$i]}" - vm_dir=$host_vm_dir/$host - mkdir -p $vm_dir - - cp ${WORK_DIR}/cache/$IMAGE_NAME $vm_dir - - # create seed.iso - sed -e "s/REPLACE_IPADDR/${IPADDR_PREFIX}$((i+11))/g" \ - -e "s/REPLACE_GATEWAY/${IPADDR_PREFIX}1/g" \ - -e "s/REPLACE_HOSTNAME/${host}/g" \ - meta-data_template \ - > meta-data - cp user-data_template user-data - genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data - cp seed.iso $vm_dir - - # create vm xml - sed -e "s/REPLACE_MEM/$VIRT_MEM/g" \ - -e "s/REPLACE_CPU/$VIRT_CPUS/g" \ - -e "s/REPLACE_NAME/$host/g" \ - -e "s#REPLACE_IMAGE#$vm_dir/disk.img#g" \ - -e "s#REPLACE_SEED_IMAGE#$vm_dir/seed.iso#g" \ - -e "s/REPLACE_MAC_ADDR/${mac_array[$i]}/g" \ - libvirt_template.xml \ - > $vm_dir/libvirt.xml - - sudo virsh define $vm_dir/libvirt.xml - sudo virsh start $host - let i=i+1 - done - IFS=$old_ifs - rm -rf meta-data user-data seed.iso -} - - -download_iso -launch_host_vms - -set +x - diff --git a/utils/rubbos_dev_env_setup/env_config.sh b/utils/rubbos_dev_env_setup/env_config.sh deleted file mode 100644 index 326b9a57..00000000 --- a/utils/rubbos_dev_env_setup/env_config.sh +++ /dev/null @@ -1,7 +0,0 @@ -export HOSTNAMES=${HOSTNAMES:-"control,httpd,mysql,tomcat,client1,client2,client3,client4,benchmark"} -export VIRT_NUMBER=9 -export VIRT_MEM=4096 -export VIRT_CPUS=4 -export IMAGE_URL=${IMAGE_URL:-"file:///bottlenecks/trusty-server-cloudimg-amd64-disk1.img"} -export IMAGE_NAME=disk.img -export IPADDR_PREFIX="192.168.122." |