diff options
Diffstat (limited to 'utils')
22 files changed, 431 insertions, 1355 deletions
diff --git a/utils/dashboard/process_data.py b/utils/dashboard/process_data.py deleted file mode 100644 index 4be09178..00000000 --- a/utils/dashboard/process_data.py +++ /dev/null @@ -1,79 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import sys -from rubbos_collector import RubbosCollector -from uploader import Uploader - - -def printUsage(): - print ("Usage: python process_data.py required_params(**)" - " optional_params([])") - print " ** -i|--input input_data_dir" - print " ** -s|--suite suite_name" - print " ** -c|--conf conf_file" - print " [] -o|--output output_file" - print " [] -u|--upload yes|no" - - -def process(input_dir, suite_name): - result = dict() - if suite_name == "rubbos": - result = RubbosCollector().collect_data(input_dir) - return result - - -def writeResult(output_file, result): - f = open(output_file, "w") - if isinstance(result, list): - for elem in result: - f.write(str(elem) + "\n") - f.close() - - -def uploadResult(conf, suite_name, result): - Uploader(conf).upload_result(suite_name, result) - - -def main(): - if len(sys.argv) < 7 or len(sys.argv) % 2 == 0: - printUsage() - exit(1) - i = 1 - params = dict() - while (i < len(sys.argv)): - if sys.argv[i] == "-i" or sys.argv[i] == "--input": - params["input"] = sys.argv[i + 1] - if sys.argv[i] == "-s" or sys.argv[i] == "--suite": - params["suite"] = sys.argv[i + 1] - if sys.argv[i] == "-c" or sys.argv[i] == "--conf": - params["conf"] = sys.argv[i + 1] - if sys.argv[i] == "-o" or sys.argv[i] == "--output": - params["output"] = sys.argv[i + 1] - if sys.argv[i] == "-u" or sys.argv[i] == "--upload": - params["upload"] = sys.argv[i + 1] - i = i + 2 - if not("input" in params and "suite" in params and "conf" in params): - print "Lack some required parameters." - exit(1) - - result = process(params["input"], params["suite"]) - print "Results:" - for elem in result: - print elem - - if "output" in params: - writeResult(params["output"], result) - - if "upload" in params and params["upload"].lower() == "yes": - uploadResult(params["conf"], params["suite"], result) - -if __name__ == "__main__": - main() diff --git a/utils/dashboard/uploader.py b/utils/dashboard/uploader.py deleted file mode 100755 index 97ffd38c..00000000 --- a/utils/dashboard/uploader.py +++ /dev/null @@ -1,66 +0,0 @@ -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd. and others -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import sys -import json -import requests -import yaml - - -class Uploader(object): - - def __init__(self, conf): - self.headers = {'Content-type': 'application/json'} - self.timeout = 5 - self.result = { - "project_name": "bottlenecks", - "description": "bottlenecks test cases result"} - - with open(conf) as stream: - dashboard_conf = yaml.load(stream) - self.result['pod_name'] = dashboard_conf['pod_name'] - self.result['installer'] = dashboard_conf['installer'] - self.result['version'] = dashboard_conf['version'] - self.target = dashboard_conf['target'] - - def upload_result(self, case_name, raw_data): - if self.target == '': - print('No target was set, so no data will be posted.') - return - self.result["case_name"] = case_name - self.result["details"] = raw_data - - try: - print('Result to be uploaded:\n %s' % json.dumps(self.result)) - res = requests.post(self.target, - data=json.dumps(self.result), - headers=self.headers, - timeout=self.timeout) - print( - 'Test result posting finished with status code %d.' % - res.status_code) - except Exception as err: - print ('Failed to record result data: %s', err) - - -def _test(): - - # data = '{"details": [{"client": 200, "throughput": 20}, - # {"client": 300, "throughput": 20}], "case_name": "rubbos"}' - if len(sys.argv) < 2: - print ("no argumens input!!") - exit(1) - - with open(sys.argv[1], 'r') as stream: - data = json.load(stream) - Uploader().upload_result(data) - - -if __name__ == "__main__": - _test() diff --git a/utils/env_prepare/config_prepare.sh b/utils/env_prepare/config_prepare.sh index 053c9da4..23923198 100644 --- a/utils/env_prepare/config_prepare.sh +++ b/utils/env_prepare/config_prepare.sh @@ -18,7 +18,7 @@ where: -h|--help show the help text -i|--installer input the name of the installer <installer> one of the following: - (compass, fuel, joid, apex) + (apex, compass, fuel, joid) --debug debug option switch examples: @@ -45,6 +45,10 @@ while [[ $# > 0 ]] redirect="/dev/stdout" shift ;; + *) + echo "unkown option $1 $2" + exit 1 + ;; esac shift done @@ -59,22 +63,27 @@ error () { exit 1 } -# Define Variables -echo "BOTTLENECKS INFO: Downloading Releng" +# Repo and configs RELENG_REPO="/home/releng" +BOTTLENECKS_CONFIG=/tmp +OPENRC=${BOTTLENECKS_CONFIG}/admin_rc.sh +OS_CACERT=${BOTTLENECKS_CONFIG}/os_cacert + + +############################################################################## +# Preparing scripts for openstack and pod configs for OPNFV Installers +############################################################################## +# Define Variables +info "Downloading Releng fetch_os_creds script for openstack/pod configs of OPNFV installers" + [ -d ${RELENG_REPO} ] && rm -rf ${RELENG_REPO} git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO} >${redirect} -echo "BOTTLENECKS INFO: Downloading Yardstick" +info "Downloading Yardstick for pod configs of OPNFV installers" YARDSTICK_REPO="/home/yardstick" [ -d ${YARDSTICK_REPO} ] && rm -rf ${YARDSTICK_REPO} git clone https://gerrit.opnfv.org/gerrit/yardstick ${YARDSTICK_REPO} >${redirect} -BOTTLENECKS_CONFIG=/tmp - -OPENRC=${BOTTLENECKS_CONFIG}/admin_rc.sh -OS_CACERT=${BOTTLENECKS_CONFIG}/os_cacert - # Preparing configuration files for testing if [[ ${INSTALLER_TYPE} != "" ]]; then # Preparing OpenStack RC and Cacert files @@ -84,73 +93,80 @@ if [[ ${INSTALLER_TYPE} != "" ]]; then INSTALLER_IP=192.168.200.2 if [[ ${BRANCH} == 'master' ]]; then ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${OS_CACERT} >${redirect} - if [[ -f ${OS_CACERT} ]]; then - echo "BOTTLENECKS INFO: successfully fetching os_cacert for openstack: ${OS_CACERT}" - else - echo "BOTTLENECKS ERROR: couldn't find os_cacert file: ${OS_CACERT}, please check if the it's been properly provided." - exit 1 - fi else ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} fi + elif [[ $INSTALLER_TYPE == 'apex' ]]; then + export BRANCH="stable/fraser" + INSTALLER_IP=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') + ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${OS_CACERT} >${redirect} + echo ${cmd} + ${cmd} else - error "The isntaller is not specified" - exit 1 - fi - - if [[ -f ${OPENRC} ]]; then - echo "BOTTLENECKS INFO: openstack credentials path is ${OPENRC}" - if [[ $INSTALLER_TYPE == 'compass' && ${BRANCH} == 'master' ]]; then - echo "BOTTLENECKS INFO: writing ${OS_CACERT} to ${OPENRC}" - echo "export OS_CACERT=${OS_CACERT}" >> ${OPENRC} - fi - cat ${OPENRC} - else - echo "BOTTLENECKS ERROR: couldn't find openstack rc file: ${OPENRC}, please check if the it's been properly provided." + error "The installer is not specified" exit 1 fi # Finding and crearting POD description files from different deployments - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - - if [ "$INSTALLER_TYPE" == "fuel" ]; then - echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa - fi + if [[ ${INSTALLER_TYPE} == 'compass' ]]; then + cmd="sudo cp ${YARDSTICK_REPO}/etc/yardstick/nodes/compass_sclab_virtual/pod.yaml \ + ${BOTTLENECKS_CONFIG}" + info ${cmd} + ${cmd} + elif [[ ${INSTALLER_TYPE} == 'apex' ]]; then + sudo pip install virtualenv - if [ "$INSTALLER_TYPE" == "apex" ]; then - echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa - fi + sudo virtualenv venv + source venv/bin/activate + sudo pip install -e ${RELENG_REPO}/modules/ >/dev/null + sudo pip install netaddr - if [[ ${INSTALLER_TYPE} == compass ]]; then - options="-u root -p root" - elif [[ ${INSTALLER_TYPE} == fuel ]]; then - options="-u root -p r00tme" - elif [[ ${INSTALLER_TYPE} == apex ]]; then options="-u stack -k /root/.ssh/id_rsa" - else - echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently." - fi - - if [[ ${INSTALLER_TYPE} != compass ]]; then cmd="sudo python ${RELENG_REPO}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \ - -i ${INSTALLER_IP} ${options} -f ${BOTTLENECKS_CONFIG}/pod.yaml \ - -s ${BOTTLENECKS_CONFIG}/id_rsa" - echo ${cmd} + -i ${INSTALLER_IP} ${options} -f ${BOTTLENECKS_CONFIG}/pod.yaml" + info ${cmd} ${cmd} - else - cmd="sudo cp ${YARDSTICK_REPO}/etc/yardstick/nodes/compass_sclab_virtual/pod.yaml \ - ${BOTTLENECKS_CONFIG}" - echo ${cmd} + + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + cmd="sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa" + info ${cmd} ${cmd} + + deactivate fi + + ############################################################################## + # Check the existence of the output configs for OPNFV Installers + ############################################################################## + # Checking the pod decription file if [ -f ${BOTTLENECKS_CONFIG}/pod.yaml ]; then - echo "FILE: ${BOTTLENECKS_CONFIG}/pod.yaml:" + info "FILE - ${BOTTLENECKS_CONFIG}/pod.yaml:" cat ${BOTTLENECKS_CONFIG}/pod.yaml else - echo "ERROR: cannot find file ${BOTTLENECKS_CONFIG}/pod.yaml. Please check if it is existing." + error "Cannot find file ${BOTTLENECKS_CONFIG}/pod.yaml. Please check if it is existing." sudo ls -al ${BOTTLENECKS_CONFIG} fi + + # Checking the openstack rc and os_cacert file + if [[ -f ${OPENRC} ]]; then + info "Opentack credentials path is ${OPENRC}" + if [[ -f ${OS_CACERT} ]]; then + info "Writing ${OS_CACERT} to ${OPENRC}" + echo "export OS_CACERT=${OS_CACERT}" >> ${OPENRC} + cat ${OPENRC} + else + info "Couldn't find openstack cacert file: ${OS_CACERT}, please check if the it's been properly provided." + fi + else + error "Couldn't find openstack rc file: ${OPENRC}, please check if the it's been properly provided." + exit 1 + fi + + # Checking ssh key id_rsa + if [[ -f "/tmp/id_rsa" ]]; then + info "Path of ssh key file for openstack nodes is /tmp/id_rsa" + else + info "Couldn't find the ssh key file for openstack nodes. If you are using user/pwd in pod.yaml, please ignore." + fi fi diff --git a/utils/env_prepare/quota_prepare.py b/utils/env_prepare/quota_prepare.py index 8dcdf3d6..367f7613 100644 --- a/utils/env_prepare/quota_prepare.py +++ b/utils/env_prepare/quota_prepare.py @@ -66,17 +66,28 @@ def quota_env_prepare(): "credential used, it should be set as True." .format(insecure)) - tenant_name = os.getenv("OS_TENANT_NAME") - cmd = ("openstack {} project list | grep ".format(insecure_option) + - tenant_name + - " | awk '{print $2}'") - - result = commands.getstatusoutput(cmd) - if result[0] == 0 and 'exception' not in result[1]: - LOG.info("Get %s project id is %s" % (tenant_name, result[1])) + quota_name = os.getenv("OS_PROJECT_NAME") + if quota_name: + cmd = ("openstack {} project list | grep ".format(insecure_option) + + quota_name + + " | awk '{print $2}'") + result = commands.getstatusoutput(cmd) + if result[0] == 0 and 'exception' not in result[1]: + LOG.info("Get %s project name is %s" % (quota_name, result[1])) + else: + LOG.error("can't get openstack project name") + return 1 else: - LOG.error("can't get openstack project id") - return 1 + quota_name = os.getenv("OS_TENANT_NAME") + cmd = ("openstack {} tenant list | grep ".format(insecure_option) + + quota_name + + " | awk '{print $2}'") + result = commands.getstatusoutput(cmd) + if result[0] == 0 and 'exception' not in result[1]: + LOG.info("Get %s tenant name is %s" % (quota_name, result[1])) + else: + LOG.error("can't get openstack tenant name") + return 1 openstack_id = result[1] @@ -85,7 +96,7 @@ def quota_env_prepare(): nova_q = nova_client.quotas.get(openstack_id).to_dict() neutron_q = neutron_client.show_quota(openstack_id) - LOG.info(tenant_name + "tenant nova and neutron quota(previous) :") + LOG.info(quota_name + " nova and neutron quotas (previous) :") LOG.info(nova_q) LOG.info(neutron_q) @@ -96,7 +107,7 @@ def quota_env_prepare(): nova_q = nova_client.quotas.get(openstack_id).to_dict() neutron_q = neutron_client.show_quota(openstack_id) - LOG.info(tenant_name + "tenant nova and neutron quota(now) :") + LOG.info(quota_name + " nova and neutron quotas (now) :") LOG.info(nova_q) LOG.info(neutron_q) return 0 diff --git a/utils/infra_setup/heat/common.py b/utils/infra_setup/heat/common.py index a0d6d83c..f0512b0f 100755 --- a/utils/infra_setup/heat/common.py +++ b/utils/infra_setup/heat/common.py @@ -66,14 +66,16 @@ def get_session_auth(): def get_session(): auth = get_session_auth() - try: - cacert = os.environ['OS_CACERT'] - except KeyError: - return session.Session(auth=auth) - else: - insecure = os.getenv('OS_INSECURE', '').lower() == 'true' - cacert = False if insecure else cacert + if os.getenv('OS_INSECURE', '').lower() == 'true': + cacert = False return session.Session(auth=auth, verify=cacert) + else: + try: + cacert = os.environ['OS_CACERT'] + except KeyError: + return session.Session(auth=auth) + else: + return session.Session(auth=auth, verify=cacert) def get_endpoint(service_type, endpoint_type='publicURL'): diff --git a/utils/infra_setup/heat/consts/__init__.py b/utils/infra_setup/heat/consts/__init__.py deleted file mode 100755 index e69de29b..00000000 --- a/utils/infra_setup/heat/consts/__init__.py +++ /dev/null diff --git a/utils/infra_setup/heat/consts/files.py b/utils/infra_setup/heat/consts/files.py deleted file mode 100755 index f148f103..00000000 --- a/utils/infra_setup/heat/consts/files.py +++ /dev/null @@ -1,36 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# ------------------------------------------------------ -# Configuration File -# ------------------------------------------------------ -GENERAL = 'General' - - -def get_sections(): - return [ - GENERAL, - # Add here new configurations... - ] - - -def get_sections_api(): - return [ - GENERAL, - # Add here new configurations... - ] - -# ------------------------------------------------------ -# General section parameters -# ------------------------------------------------------ -ITERATIONS = 'iterations' -TEMPLATE_DIR = 'template_dir' -TEMPLATE_NAME = 'template_base_name' -BENCHMARKS = 'benchmarks' -DEBUG = 'debug' diff --git a/utils/infra_setup/heat/consts/parameters.py b/utils/infra_setup/heat/consts/parameters.py deleted file mode 100755 index 15aa5b5d..00000000 --- a/utils/infra_setup/heat/consts/parameters.py +++ /dev/null @@ -1,16 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -# ------------------------------------------------------ -# Directories and file locations -# ------------------------------------------------------ -HEAT_DIR = 'heat/' -TEST_TEMPLATE_NAME = 'test_template' -TEMPLATE_EXTENSION = '.yaml' diff --git a/utils/infra_setup/heat_template/rubbos_heat_template/HOT_create_instance.sh b/utils/infra_setup/heat_template/rubbos_heat_template/HOT_create_instance.sh deleted file mode 100755 index 1a80d950..00000000 --- a/utils/infra_setup/heat_template/rubbos_heat_template/HOT_create_instance.sh +++ /dev/null @@ -1,325 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -x - -git_checkout() -{ - if git cat-file -e $1^{commit} 2>/dev/null; then - # branch, tag or sha1 object - git checkout $1 - else - # refspec / changeset - git fetch --tags --progress $2 $1 - git checkout FETCH_HEAD - fi -} - -bottlenecks_env_prepare() { - set -e - echo "Bottlenecks env prepare start $(date)" - git config --global http.sslVerify false - - if [ ! -d $BOTTLENECKS_REPO_DIR ]; then - git clone $BOTTLENECKS_REPO $BOTTLENECKS_REPO_DIR - fi - cd $BOTTLENECKS_REPO_DIR - git checkout master && git pull - git_checkout $BOTTLENECKS_BRANCH $BOTTLENECKS_REPO - cd - - - echo "Creating openstack credentials .." - if [ ! -d $RELENG_REPO_DIR ]; then - git clone $RELENG_REPO $RELENG_REPO_DIR - fi - cd $RELENG_REPO_DIR - git checkout master && git pull - git_checkout $RELENG_BRANCH $RELENG_REPO - cd - - - # Create openstack credentials - $RELENG_REPO_DIR/utils/fetch_os_creds.sh \ - -d /tmp/openrc \ - -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} - - source /tmp/openrc - - chmod 600 $KEY_PATH/bottlenecks_key - - echo "Bottlenecks env prepare end $(date)" - set +e -} - -wait_heat_stack_complete() { - retry=0 - while true - do - status=$(heat stack-list | grep bottlenecks | awk '{print $6}') - if [ x$status = x"CREATE_COMPLETE" ]; then - echo "bottlenecks stacke create complete" - heat stack-show bottlenecks - nova list | grep rubbos_ - break; - elif [ x$status = x"CREATE_FAILED" ]; then - echo "bottlenecks stacke create failed !!!" - heat stack-show bottlenecks - exit 1 - fi - - #if [ $BOTTLENECKS_DEBUG = True ]; then - if false; then - heat stack-show bottlenecks - nova list | grep rubbos_ - for i in $(nova list | grep rubbos_ | grep ERROR | awk '{print $2}') - do - nova show $i - done - fi - sleep 1 - let retry+=1 - if [[ $retry -ge $1 ]];then - echo "Heat stack create timeout, status $status !!!" - exit 1 - fi - done -} - -wait_rubbos_control_ok() { - control_ip=$(nova list | grep rubbos_control | awk '{print $13}') - - retry=0 - until timeout 3s ssh $ssh_args ubuntu@$control_ip "exit" >/dev/null 2>&1 - do - echo "retry connect rubbos control $retry" - sleep 1 - let retry+=1 - if [[ $retry -ge $1 ]];then - echo "rubbos control start timeout !!!" - exit 1 - fi - done - ssh $ssh_args ubuntu@$control_ip "uname -a" -} - -bottlenecks_check_instance_ok() -{ - echo "Bottlenecks check instance ok start $(date)" - - wait_heat_stack_complete 120 - wait_rubbos_control_ok 300 - nova list | grep rubbos_ - if [ $BOTTLENECKS_DEBUG = True ]; then - date - while true - do - for i in rubbos_benchmark rubbos_client1 rubbos_client2 rubbos_client3 \ - rubbos_client4 rubbos_control rubbos_httpd rubbos_mysql1 rubbos_tomcat1 - do - echo "logging $i" - nova console-log $i | tail -n 2 | grep Cloud-init | grep finished - if [ $? != 0 ]; then - break - fi - if [ $i = rubbos_tomcat1 ]; then - echo "all vm Cloud-init finished!" - date - return - fi - done - sleep 10 - done - fi - - echo "Bottlenecks check instance ok end $(date)" -} - -bottlenecks_create_instance() -{ - echo "Bottlenecks create instance using heat template start $(date)" - - echo "upload keypair" - nova keypair-add --pub_key $KEY_PATH/bottlenecks_key.pub $KEY_NAME - - echo "create flavor" - nova flavor-create $FLAVOR_NAME 200 4096 20 2 - - echo "use heat template to create stack" - cd $HOT_PATH - heat stack-create bottlenecks -f ${TEMPLATE_NAME} \ - -P "image=$IMAGE_NAME;key_name=$KEY_NAME;public_net=$EXTERNAL_NET;flavor=$FLAVOR_NAME" - - echo "Bottlenecks create instance using heat template end $(date)" -} - -bottlenecks_rubbos_wait_finish() -{ - echo "Start checking rubbos running status..." - retry=0 - while true - do - ssh $ssh_args ubuntu@$control_ip "FILE=/tmp/rubbos_finished; if [ -f \$FILE ]; then exit 0; else exit 1; fi" - if [ $? = 0 ]; then - echo "Rubbos test case successfully finished :)" - return 0 - fi - echo "Rubbos running $retry ..." - sleep 30 - let retry+=1 - if [[ $retry -ge $1 ]]; then - echo "Rubbos test case timeout :(" - return 1 - fi - done -} - -bottlenecks_rubbos_run() -{ - echo "Run Rubbos" - control_ip=$(nova list | grep rubbos_control | awk '{print $13}') - for i in rubbos_benchmark rubbos_client1 rubbos_client2 rubbos_client3 \ - rubbos_client4 rubbos_control rubbos_httpd rubbos_mysql1 \ - rubbos_tomcat1 - do - ip=$(nova list | grep $i | awk '{print $12}' | awk -F [=,] '{print $2}') - echo "$i=$ip" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - done - - nameserver_ip=$(grep -m 1 '^nameserver' \ - /etc/resolv.conf | awk '{ print $2 '}) - echo "nameserver_ip=$nameserver_ip" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - - echo "GERRIT_REFSPEC_DEBUG=$GERRIT_REFSPEC_DEBUG" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - echo "BOTTLENECKS_BRANCH=$BOTTLENECKS_BRANCH" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - - echo "NODE_NAME=$NODE_NAME" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - echo "INSTALLER_TYPE=$INSTALLER_TYPE" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - echo "BOTTLENECKS_VERSION=$BOTTLENECKS_VERSION" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - echo "BOTTLENECKS_DB_TARGET=$BOTTLENECKS_DB_TARGET" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - echo "PACKAGE_URL=$PACKAGE_URL" >> $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf - - scp $ssh_args -r \ - $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup \ - ubuntu@$control_ip:/tmp - ssh $ssh_args \ - ubuntu@$control_ip "bash /tmp/vm_dev_setup/setup_env.sh" & - - bottlenecks_rubbos_wait_finish 200 - - if [ x"$GERRIT_REFSPEC_DEBUG" != x ]; then - # TODO fix hard coded path - scp $ssh_args \ - ubuntu@$control_ip:"/bottlenecks/rubbos/rubbos_results/2015-01-20T081237-0700.tgz" /tmp - fi - - rm -rf $BOTTLENECKS_REPO_DIR/utils/infra_setup/vm_dev_setup/hosts.conf -} - -bottlenecks_cleanup() -{ - echo "Bottlenecks cleanup start $(date)" - - if heat stack-list; then - for stack in $(heat stack-list | grep -e bottlenecks | awk '{print $2}'); do - echo "clean up stack $stack" - heat stack-delete $stack || true - sleep 30 - done - fi - - if glance image-list; then - for image in $(glance image-list | grep -e $IMAGE_NAME | awk '{print $2}'); do - echo "clean up image $image" - glance image-delete $image || true - done - fi - - if nova keypair-list; then - for key in $(nova keypair-list | grep -e $KEY_NAME | awk '{print $2}'); do - echo "clean up key $key" - nova keypair-delete $key || true - done - fi - - if nova flavor-list; then - for flavor in $(nova flavor-list | grep -e $FLAVOR_NAME | awk '{print $2}'); do - echo "clean up flavor $flavor" - nova flavor-delete $flavor || true - done - fi - - echo "Bottlenecks cleanup end $(date)" -} - -bottlenecks_load_bottlenecks_image() -{ - echo "Bottlenecks load image start $(date)" - - curl --connect-timeout 10 -o /tmp/bottlenecks-trusty-server.img $IMAGE_URL -v - - result=$(glance image-create \ - --name $IMAGE_NAME \ - --disk-format qcow2 \ - --container-format bare \ - --file /tmp/bottlenecks-trusty-server.img) - echo "$result" - - rm -rf /tmp/bottlenecks-trusty-server.img - - IMAGE_ID_BOTTLENECKS=$(echo "$result" | grep " id " | awk '{print $(NF-1)}') - if [ -z "$IMAGE_ID_BOTTLENECKS" ]; then - echo 'failed to upload bottlenecks image to openstack' - exit 1 - fi - - echo "bottlenecks image end id: $IMAGE_ID_BOTTLENECKS $(date)" -} - -main() -{ - echo "main start $(date)" - - : ${BOTTLENECKS_DEBUG:='True'} - : ${BOTTLENECKS_REPO:='https://gerrit.opnfv.org/gerrit/bottlenecks'} - : ${BOTTLENECKS_REPO_DIR:='/tmp/opnfvrepo/bottlenecks'} - : ${BOTTLENECKS_BRANCH:='master'} # branch, tag, sha1 or refspec - : ${RELENG_REPO:='https://gerrit.opnfv.org/gerrit/releng'} - : ${RELENG_REPO_DIR:='/tmp/opnfvrepo/releng'} - : ${RELENG_BRANCH:='master'} # branch, tag, sha1 or refspec - : ${IMAGE_NAME:='bottlenecks-trusty-server'} - KEY_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/bottlenecks_key - HOT_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/heat_template - : ${KEY_NAME:='bottlenecks-key'} - : ${FLAVOR_NAME:='bottlenecks-flavor'} - : ${TEMPLATE_NAME:='bottlenecks_rubbos_hot.yaml'} - ssh_args="-o StrictHostKeyChecking=no -o BatchMode=yes -i $KEY_PATH/bottlenecks_key" - : ${EXTERNAL_NET:='net04_ext'} - : ${PACKAGE_URL:='http://artifacts.opnfv.org/bottlenecks'} - : ${NODE_NAME:='opnfv-jump-2'} - : ${INSTALLER_TYPE:='fuel'} - : ${INSTALLER_IP:='10.20.0.2'} - # TODO fix for dashboard - : ${BOTTLENECKS_VERSION:='master'} - : ${BOTTLENECKS_DB_TARGET:='213.77.62.197'} - IMAGE_URL=${PACKAGE_URL}/rubbos/bottlenecks-trusty-server.img - - bottlenecks_env_prepare - set -x - bottlenecks_cleanup - bottlenecks_load_bottlenecks_image - bottlenecks_create_instance - bottlenecks_check_instance_ok - bottlenecks_rubbos_run - bottlenecks_cleanup - echo "main end $(date)" -} - -main -set +x - diff --git a/utils/infra_setup/heat_template/vstf_heat_template/bottleneck_vstf.yaml b/utils/infra_setup/heat_template/vstf_heat_template/bottleneck_vstf.yaml deleted file mode 100644 index d9419929..00000000 --- a/utils/infra_setup/heat_template/vstf_heat_template/bottleneck_vstf.yaml +++ /dev/null @@ -1,252 +0,0 @@ -heat_template_version: 2013-05-23 -description: > - This template is used for creating a new environment on the Openstack Release L , - and the deployment will create three virtual machine on the compute node, one manager - and two agent vm included. Each vm will has a nic on the controlplane switch and two - agent vms will has a additional nic on the dataplane. -parameters: - #nova keypair-list to query available key pair - key_name: - type: string - description: Name of keypair to assign to servers - default: vstf-key - #nova image-list to query available images - image: - type: string - description: Name of image to use for servers - default: bottlenecks-trusty-server - #new addition image for the actual deployment - image_vstf_manager: - type: string - description: Name of image to use for servers - default: vstf-manager - image_vstf_tester: - type: string - description: Name of image to use for servers - default: vstf-tester - image_vstf_target: - type: string - description: Name of image to use for servers - default: vstf-target - #nova flavor-list to query available flavors - flavor: - type: string - description: Flavor to use for servers - default: m1.large - #nova net-list to query available - public_net: - type: string - description: > - ID or name of public network for which floating IP addresses will be allocated - default: net04_ext - - #private controlplane - private_net_name: - type: string - description: Name of private network to be created - default: vstf-private - private_net_cidr: - type: string - description: Private network address (CIDR notation) - default: "10.0.11.0/24" - private_net_gateway: - type: string - description: Private network gateway address - default: "10.0.11.1" - private_net_pool_start: - type: string - description: Start of private network IP address allocation pool - default: "10.0.11.2" - private_net_pool_end: - type: string - description: End of private network IP address allocation pool - default: "10.0.11.199" - - #testing dataplane - testing_net_name: - type: string - description: Name of private network to be created - default: bottlenecks-testing - testing_net_cidr: - type: string - description: Private network address (CIDR notation) - default: "10.0.20.0/24" - testing_net_gateway: - type: string - description: Private network gateway address - default: "10.0.20.1" - testing_net_pool_start: - type: string - description: Start of private network IP address allocation pool - default: "10.0.20.2" - testing_net_pool_end: - type: string - description: End of private network IP address allocation pool - default: "10.0.20.199" - - -resources: - #control plane - private_net: - type: OS::Neutron::Net - properties: - name: { get_param: private_net_name } - private_subnet: - type: OS::Neutron::Subnet - properties: - network_id: { get_resource: private_net } - cidr: { get_param: private_net_cidr } - gateway_ip: { get_param: private_net_gateway } - allocation_pools: - - start: { get_param: private_net_pool_start } - end: { get_param: private_net_pool_end } - - #dataplane - testing_net: - type: OS::Neutron::Net - properties: - name: { get_param: testing_net_name } - testing_subnet: - type: OS::Neutron::Subnet - properties: - network_id: { get_resource: testing_net } - cidr: { get_param: testing_net_cidr } - gateway_ip: { get_param: testing_net_gateway } - allocation_pools: - - start: { get_param: testing_net_pool_start } - end: { get_param: testing_net_pool_end } - - #router info - router: - type: OS::Neutron::Router - properties: - external_gateway_info: - network: { get_param: public_net } - router_interface: - type: OS::Neutron::RouterInterface - properties: - router_id: { get_resource: router } - subnet_id: { get_resource: private_subnet } - - #security_group - server_security_group: - type: OS::Neutron::SecurityGroup - properties: - description: vstf group for servers access. - name: vstf-security-group - rules: [ - {remote_ip_prefix: 0.0.0.0/0, - protocol: tcp, - port_range_min: 1, - port_range_max: 65535}, - {remote_ip_prefix: 0.0.0.0/0, - protocol: udp, - port_range_min: 1, - port_range_max: 65535}, - {remote_ip_prefix: 0.0.0.0/0, - protocol: icmp}] - - #nova server vstf manager definition info - vstf-manager: - type: OS::Nova::Server - properties: - name: vstf-manager - image: { get_param: image_vstf_manager } - flavor: { get_param: flavor } - key_name: { get_param: key_name } - networks: - - port: { get_resource: manager_control_port } - manager_control_port: - type: OS::Neutron::Port - properties: - network_id: { get_resource: private_net } - fixed_ips: - - subnet_id: { get_resource: private_subnet } - security_groups: [{ get_resource: server_security_group }] - manager_control_floating_ip: - type: OS::Neutron::FloatingIP - properties: - floating_network: { get_param: public_net } - port_id: { get_resource: manager_control_port } - - #nova server vstf target definition info - vstf-target: - type: OS::Nova::Server - properties: - name: vstf-target - image: { get_param: image_vstf_target } - flavor: { get_param: flavor } - key_name: { get_param: key_name } - networks: - - port: { get_resource: target_control_port } - - port: { get_resource: target_testing_port } - target_control_port: - type: OS::Neutron::Port - properties: - network_id: { get_resource: private_net } - fixed_ips: - - subnet_id: { get_resource: private_subnet } - security_groups: [{ get_resource: server_security_group }] - target_testing_port: - type: OS::Neutron::Port - properties: - network_id: { get_resource: testing_net } - fixed_ips: - - subnet_id: { get_resource: testing_subnet } - security_groups: [{ get_resource: server_security_group }] - target_control_floating_ip: - type: OS::Neutron::FloatingIP - properties: - floating_network: { get_param: public_net } - port_id: { get_resource: target_control_port } - - #nova server vstf tester definition info - vstf-tester: - type: OS::Nova::Server - properties: - name: vstf-tester - image: { get_param: image_vstf_tester } - flavor: { get_param: flavor } - key_name: { get_param: key_name } - networks: - - port: { get_resource: tester_control_port } - - port: { get_resource: tester_testing_port } - tester_control_port: - type: OS::Neutron::Port - properties: - network_id: { get_resource: private_net } - fixed_ips: - - subnet_id: { get_resource: private_subnet } - security_groups: [{ get_resource: server_security_group }] - tester_testing_port: - type: OS::Neutron::Port - properties: - network_id: { get_resource: testing_net } - fixed_ips: - - subnet_id: { get_resource: testing_subnet } - security_groups: [{ get_resource: server_security_group }] - tester_control_floating_ip: - type: OS::Neutron::FloatingIP - properties: - floating_network: { get_param: public_net } - port_id: { get_resource: tester_control_port } - -outputs: - manager_control_private_ip: - description: IP address of manager_control in private network - value: { get_attr: [ vstf-manager, first_address ] } - manager_control_public_ip: - description: Floating IP address of manager_control in public network - value: { get_attr: [ manager_control_floating_ip, floating_ip_address ] } - target_control_private_ip: - description: IP address of manager_control in private network - value: { get_attr: [ vstf-target, first_address ] } - target_control_public_ip: - description: Floating IP address of manager_control in public network - value: { get_attr: [ target_control_floating_ip, floating_ip_address ] } - tester_control_private_ip: - description: IP address of manager_control in private network - value: { get_attr: [ vstf-tester, first_address ] } - tester_control_public_ip: - description: Floating IP address of manager_control in public network - value: { get_attr: [ tester_control_floating_ip, floating_ip_address ] } diff --git a/utils/infra_setup/heat_template/vstf_heat_template/dashboard.json b/utils/infra_setup/heat_template/vstf_heat_template/dashboard.json deleted file mode 100755 index df6f64c3..00000000 --- a/utils/infra_setup/heat_template/vstf_heat_template/dashboard.json +++ /dev/null @@ -1,6 +0,0 @@ -{
- "pod_name": "unknown-pod",
- "installer": "fuel",
- "version": "unknown",
- "target": "http://213.77.62.197/results"
-}
\ No newline at end of file diff --git a/utils/infra_setup/heat_template/vstf_heat_template/launch_vstf.sh b/utils/infra_setup/heat_template/vstf_heat_template/launch_vstf.sh deleted file mode 100644 index ec615ec2..00000000 --- a/utils/infra_setup/heat_template/vstf_heat_template/launch_vstf.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -x - -STACK_NAME="bottlenecks_vstf_stack" -VM_MANAGER_USER="root" -VM_MANAGER_PASSWD="root" -VM_TARGET_USER="root" -VM_TARGET_PASSWD="root" -VM_TESTER_USER="root" -VM_TESTER_PASSWD="root" -RABBITMQ_PORT="5672" - -#load func -#source ./ssh.sh -#source ./scp.sh - -function fn_parser_ipaddress(){ - #parser and get output ipaddress - manager_control_private_ip=`heat output-show ${STACK_NAME} manager_control_private_ip | sed 's/\"//g'` - manager_control_public_ip=`heat output-show ${STACK_NAME} manager_control_public_ip | sed 's/\"//g'` - echo "manager_control_private_ip = ${manager_control_private_ip}" - #ping -c 5 ${manager_control_private_ip} - echo "manager_control_public_ip = ${manager_control_public_ip}" - ping -c 5 ${manager_control_public_ip} - target_control_private_ip=`heat output-show ${STACK_NAME} target_control_private_ip | sed 's/\"//g'` - target_control_public_ip=`heat output-show ${STACK_NAME} target_control_public_ip | sed 's/\"//g'` - echo "target_control_private_ip = ${target_control_private_ip}" - #ping -c 5 ${target_control_private_ip} - echo "target_control_public_ip = ${target_control_public_ip}" - ping -c 5 ${target_control_public_ip} - tester_control_private_ip=`heat output-show ${STACK_NAME} tester_control_private_ip | sed 's/\"//g'` - tester_control_public_ip=`heat output-show ${STACK_NAME} tester_control_public_ip | sed 's/\"//g'` - echo "tester_control_private_ip = ${tester_control_private_ip}" - #ping -c 5 ${tester_control_private_ip} - echo "tester_control_public_ip = ${tester_control_public_ip}" - ping -c 5 ${tester_control_public_ip} - - #get testing ipaddress - tester_testing_ip=`nova list | grep "vstf-tester" | grep "bottlenecks-testing" | awk -F'bottlenecks-testing=' '{print $2}' | awk '{print $1}'` - target_testing_ip=`nova list | grep "vstf-target" | grep "bottlenecks-testing" | awk -F'bottlenecks-testing=' '{print $2}' | awk '{print $1}'` - echo "tester_testing_ip = ${tester_testing_ip}" - echo "target_testing_ip = ${target_testing_ip}" - - #config ip for the testing plane - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${target_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${target_control_public_ip} "ifconfig eth1 ${target_testing_ip}/24;sleep 4" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${tester_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${tester_control_public_ip} "ifconfig eth1 ${tester_testing_ip}/24;sleep 4" - #ping with each other - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${target_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${target_control_public_ip} "ping -c 10 ${tester_testing_ip}" - - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${tester_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${tester_control_public_ip} "ping -c 10 ${target_testing_ip}" - - local ipaddr="" - for ipaddr in ${manager_control_private_ip} ${manager_control_public_ip} ${target_control_private_ip} \ - ${target_control_public_ip} ${tester_control_private_ip} ${tester_control_public_ip} - do - if [ "${ipaddr}x" == "x" ] - then - echo "[ERROR]The ipaddress is null ,get ip from heat output failed" - exit 1 - fi - done - - return 0 -} - -function fn_generate_amqp(){ - local node_type=$1 - if [ "${node_type}" == "manager" ] - then - return 0 - elif [ "${node_type}" == "target" -o "${node_type}" == "tester" ] - then - echo "[rabbit]" > ./vstf-${node_type}.ini - echo "user=guest" >> ./vstf-${node_type}.ini - echo "passwd=guest" >> ./vstf-${node_type}.ini - echo "host=${manager_control_private_ip}" >> ./vstf-${node_type}.ini - echo "port=${RABBITMQ_PORT}" >> ./vstf-${node_type}.ini - echo "id=\"${node_type}\"" >> ./vstf-${node_type}.ini - else - echo "[ERROR]node type ${node_type} does not exist" - exit 1 - fi - return 0 -} - -function fn_provision_agent_file(){ - - #apt-get -y install expect - #manager - fn_generate_amqp "manager" - - #target - fn_generate_amqp "target" - #scp_cmd ${target_control_public_ip} ${VM_TARGET_USER} ${VM_TARGET_PASSWD} "./vstf-target.ini" "/etc/vstf/amqp/amqp.ini" "file" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${target_control_public_ip} - sshpass -p root scp -o StrictHostKeyChecking=no "./vstf-target.ini" root@${target_control_public_ip}:/etc/vstf/amqp/amqp.ini - #tester - fn_generate_amqp "tester" - #scp_cmd ${tester_control_public_ip} ${VM_TESTER_USER} ${VM_TESTER_PASSWD} "./vstf-tester.ini" "/etc/vstf/amqp/amqp.ini" "file" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${tester_control_public_ip} - sshpass -p root scp -o StrictHostKeyChecking=no "./vstf-tester.ini" root@${tester_control_public_ip}:/etc/vstf/amqp/amqp.ini - - return 0 -} - -function fn_launch_vstf_process(){ - - #launch manager - local manager_cmd="vstf-manager stop;pkill vstf-manager;rm -rf /opt/vstf/vstf-server.pid;vstf-manager start --monitor ${manager_control_private_ip} --port ${RABBITMQ_PORT}" - #run_cmd ${manager_control_public_ip} ${VM_MANAGER_USER} ${VM_MANAGER_PASSWD} "${manager_cmd}" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${manager_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "ifconfig -a" - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "${manager_cmd}" - - #launch target agent - local target_cmd="vstf-agent stop;pkill vstf-agent;rm -rf /tmp/esp_rpc_client.pid;vstf-agent start --config_file=/etc/vstf/amqp/amqp.ini" - #run_cmd ${target_control_public_ip} ${VM_TARGET_USER} ${VM_TARGET_PASSWD} "${target_cmd}" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${target_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${target_control_public_ip} "ifconfig -a" - sshpass -p root ssh -o StrictHostKeyChecking=no root@${target_control_public_ip} "${target_cmd}" - - #launch tester agent - #run_cmd ${tester_control_public_ip} ${VM_TESTER_USER} ${VM_TESTER_PASSWD} "${target_cmd}" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${tester_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${tester_control_public_ip} "ifconfig -a" - sshpass -p root ssh -o StrictHostKeyChecking=no root@${tester_control_public_ip} "${target_cmd}" - - return 0 -} - -function main(){ - fn_parser_ipaddress - fn_provision_agent_file - fn_launch_vstf_process - cmd="rabbitmqctl list_queues" - sleep 20 - #${manager_control_public_ip} ${VM_MANAGER_USER} ${VM_MANAGER_PASSWD} "${cmd}" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${manager_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "${cmd}" - return 0 -} - -main -set +x diff --git a/utils/infra_setup/heat_template/vstf_heat_template/vstf_HOT_create_instance.sh b/utils/infra_setup/heat_template/vstf_heat_template/vstf_HOT_create_instance.sh deleted file mode 100755 index ce8ed439..00000000 --- a/utils/infra_setup/heat_template/vstf_heat_template/vstf_HOT_create_instance.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -ex - -GERRIT_REFSPEC_DEBUG=$1 - -echo "vstf DEBUG test" -echo "vstf workflow goes here" - -bottlenecks_env_prepare() -{ - if [ -d $BOTTLENECKS_REPO_DIR ]; then - rm -rf ${BOTTLENECKS_REPO_DIR} - fi - - mkdir -p ${BOTTLENECKS_REPO_DIR} - git config --global http.sslVerify false - git clone ${BOTTLENECKS_REPO} ${BOTTLENECKS_REPO_DIR} - if [ x"$GERRIT_REFSPEC_DEBUG" != x ]; then - cd ${BOTTLENECKS_REPO_DIR} - git fetch $BOTTLENECKS_REPO $GERRIT_REFSPEC_DEBUG && git checkout FETCH_HEAD - cd - - fi - - #obtain installer(openstack) IP, etc, use rubbos's temporarily, later we can amend this - source $BOTTLENECKS_REPO_DIR/rubbos/rubbos_scripts/1-1-1/scripts/env_preparation.sh -} - -vstf_cleanup() -{ - echo "[INFO]Begin to clean up vstf heat-stack ,glance images and keypairs" - #heat stack-delete bottlenecks - sleep 30 - if heat stack-list; then - for stack in $(heat stack-list | grep -e " vstf " | awk '{print $2}'); do - echo "[INFO]clean up stack $stack" - heat stack-delete $stack || true - sleep 30 - done - fi - - echo "begin to clean the image" - glance image-delete ${MANAGER_IMAGE_NAME};glance image-delete "${TARGET_IMAGE_NAME}";glance image-delete "${TESTER_IMAGE_NAME}" - if glance image-list; then - for image in $(glance image-list | grep -e "${MANAGER_IMAGE_NAME}" | awk '{print $2}'); do - echo "[INFO]clean up image $image" - glance image-delete $image || true - done - for image in $(glance image-list | grep -e "${TARGET_IMAGE_NAME}" | awk '{print $2}'); do - echo "[INFO]clean up image $image" - glance image-delete $image || true - done - for image in $(glance image-list | grep -e "${TESTER_IMAGE_NAME}" | awk '{print $2}'); do - echo "[INFO]clean up image $image" - glance image-delete $image || true - done - fi - - if nova keypair-list; then - for key in $(nova keypair-list | grep -e $KEY_NAME | awk '{print $2}'); do - echo "[INFO]clean up key $key" - nova keypair-delete $key || true - done - fi - - #check the default flavor m1.large existing - if nova flavor-list; then - flag=`nova flavor-list | grep "m1.large "` - echo "[INFO]the flavor m1.large num is $flag" - fi - - #delete image file - rm -rf /tmp/vstf-manager.img;rm -rf /tmp/vstf-agent.img ;rm -rf /tmp/vstf-agent_1.img - return 0 - -} - -vstf_register() -{ - echo "[INFO]download vstf images" - #download vstf-manager and vstf-agent image - #curl --connect-timeout 10 -o /tmp/vstf-manager.img $MANAGER_IMAGE_URL -v - #curl --connect-timeout 10 -o /tmp/vstf-agent.img $AGENT_IMAGE_URL -v - curl --connect-timeout 10 -o /tmp/vstf-manager.img $MANAGER_IMAGE_URL -v - curl --connect-timeout 10 -o /tmp/vstf-agent.img $MANAGER_IMAGE_URL -v - curl --connect-timeout 10 -o /tmp/vstf-agent_1.img $MANAGER_IMAGE_URL -v - #echo "begin to test downloading from vstf directory!!!!!!" - #curl --connect-timeout 10 -o /tmp/vstf-test.txt - #echo "begin to cat /tmp/vstf-test.txt vstf directory!!!!!!" - #cat /tmp/vstf-test.txt - #register - echo "[INFO]register vstf manager and agent images" - result=$(glance image-create \ - --name $MANAGER_IMAGE_NAME \ - --disk-format qcow2 \ - --container-format bare \ - --file /tmp/vstf-manager.img) - echo "Manager image register result $result." - - result=$(glance image-create \ - --name $TESTER_IMAGE_NAME \ - --disk-format qcow2 \ - --container-format bare \ - --file /tmp/vstf-agent.img) - echo "Agent image register result $result." - - result=$(glance image-create \ - --name $TARGET_IMAGE_NAME \ - --disk-format qcow2 \ - --container-format bare \ - --file /tmp/vstf-agent_1.img) - echo "Agent image register result $result." - - glance image-list - - rm -rf /tmp/vstf-manager.img;rm -rf /tmp/vstf-agent.img ;rm -rf /tmp/vstf-agent_1.img -} - -#vstf logic function here -vstf_create_heat_template() -{ - echo "create vstf instance using heat template" - echo "upload keypair" - nova keypair-add --pub_key $KEY_PATH/bottlenecks_key.pub $KEY_NAME - nova keypair-list - echo "use heat template to create stack" - cd ${HOT_PATH} - heat stack-create vstf -f ${TEMPLATE_NAME} - -} - -wait_heat_stack_complete() -{ - retry=0 - while true - do - status=$(heat stack-list | grep vstf | awk '{print $6}') - if [ x$status = x"CREATE_COMPLETE" ]; then - echo "vstf stacke create complete" - heat stack-show vstf - nova list | grep vstf- - break; - elif [ x$status = x"CREATE_FAILED" ]; then - echo "bottlenecks stacke create failed !!!" - heat stack-show vstf - exit 1 - fi - - if [ "$BOTTLENECKS_DEBUG" == "True" ]; then - heat stack-show vstf - nova list | grep vstf- - for i in $(nova list | grep "vstf-" | grep ERROR | awk '{print $2}') - do - nova show $i - done - fi - sleep 1 - let retry+=1 - if [[ $retry -ge $1 ]];then - echo "Heat vstf stack create timeout, status $status !!!" - exit 1 - fi - done -} - - -vstf_check_instance_ok() -{ - wait_heat_stack_complete 120 - - return 0 -} - -vstf_launch() -{ - cd ${HOT_PATH} - bash -x ./launch_vstf.sh - -} - -vstf_test() -{ - cd ${HOT_PATH} - bash -x ./vstf_test.sh -} - -main() -{ - echo "bottlenecks vstf: create instances with heat template" - - BOTTLENECKS_REPO=https://gerrit.opnfv.org/gerrit/bottlenecks - BOTTLENECKS_REPO_DIR=/tmp/opnfvrepo_vstf/bottlenecks - #vstf parameter here - MANAGER_IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/vstf-manager-new.img - AGENT_IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/vstf-agent-new.img - #MANAGER_IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img - #AGENT_IMAGE_URL=http://artifacts.opnfv.org/bottlenecks/rubbos/bottlenecks-trusty-server.img - MANAGER_IMAGE_NAME="vstf-manager" - TESTER_IMAGE_NAME="vstf-tester" - TARGET_IMAGE_NAME="vstf-target" - - KEY_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/bottlenecks_key - HOT_PATH=$BOTTLENECKS_REPO_DIR/utils/infra_setup/heat_template/vstf_heat_template - KEY_NAME=vstf-key - #use the default openstack flavor m1.large - FLAVOR_NAME="m1.large" - TEMPLATE_NAME=bottleneck_vstf.yaml - PUBLIC_NET_NAME=net04_ext - - #load adminrc - bottlenecks_env_prepare - #vstf function here - vstf_cleanup - vstf_register - vstf_create_heat_template - vstf_check_instance_ok - heat stack-list - nova list - sleep 100 - vstf_launch - sleep 30 - vstf_test - sleep 10 - echo "[INFO]bottleneck vstf testsuite done ,results in the directory ${HOT_PATH}/result" - echo "[INFO]Begin to clean up the vstf heat-stack and image" - vstf_cleanup - sleep 30 - heat stack-list - nova list - -} - -main -set +ex diff --git a/utils/infra_setup/heat_template/vstf_heat_template/vstf_test.sh b/utils/infra_setup/heat_template/vstf_heat_template/vstf_test.sh deleted file mode 100644 index 94e42e23..00000000 --- a/utils/infra_setup/heat_template/vstf_heat_template/vstf_test.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -set -x - -VM_MANAGER_USER="root" -VM_MANAGER_PASSWD="root" -STACK_NAME="bottlenecks_vstf_stack" - -function fn_parser_ipaddress(){ - #parser and get output ipaddress - manager_control_private_ip=`heat output-show ${STACK_NAME} manager_control_private_ip | sed 's/\"//g'` - manager_control_public_ip=`heat output-show ${STACK_NAME} manager_control_public_ip | sed 's/\"//g'` - - local ipaddr="" - for ipaddr in ${manager_control_private_ip} ${manager_control_public_ip} - do - if [ "${ipaddr}x" == "x" ] - then - echo "[ERROR]The ipaddress is null ,get ip from heat output failed" - exit 1 - fi - done - - return 0 -} - -function fn_vstf_test_config(){ - #get testing ipaddress - tester_testing_ip=`nova list | grep "vstf-tester" | grep "bottlenecks-testing" | awk -F'bottlenecks-testing=' '{print $2}' | awk '{print $1}'` - target_testing_ip=`nova list | grep "vstf-target" | grep "bottlenecks-testing" | awk -F'bottlenecks-testing=' '{print $2}' | awk '{print $1}'` - echo "tester_testing_ip = ${tester_testing_ip}" - echo "target_testing_ip = ${target_testing_ip}" - #setting testting ipaddress - local cmd="vstfadm settings ${tester_testing_ip} ${target_testing_ip}" - echo "$cmd" - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${manager_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "${cmd}" - - return 0 -} - -function fn_testing_scenario(){ - local head_cmd="vstfadm perf-test " - local test_length_list="64 128 256 512 1024" - local test_scenario_list="Tu-1 Tu-3" - local test_tool="netperf" - local protocol="udp" - local test_type="frameloss" - for scene in ${test_scenario_list} - do - local cmd="${head_cmd} ${scene} ${test_tool} ${protocol} ${test_type} \"${test_length_list}\" > /root/${scene}-result.txt" - echo ${cmd} - - ssh-keygen -f "/home/jenkins-ci/.ssh/known_hosts" -R ${manager_control_public_ip} - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "${cmd}" - sleep 10 - done - return 0 -} - -function fn_result(){ - local test_scenario_list="Tu-1 Tu-3" - mkdir ./result - rm -rf ./result/* - for scene in ${test_scenario_list} - do - sshpass -p root ssh -o StrictHostKeyChecking=no root@${manager_control_public_ip} "cat /root/${scene}-result.txt" - sshpass -p root scp -o StrictHostKeyChecking=no root@${manager_control_public_ip}:/root/${scene}-result.txt "./result/${scene}" - done - return 0 -} - -function fn_upload(){ - python vstf_collector.py --config dashboard.json --dir result -} - -function main(){ - fn_parser_ipaddress - fn_vstf_test_config - fn_testing_scenario - fn_result - fn_upload - return 0 -} - -main -set +x diff --git a/utils/infra_setup/runner/yardstick.py b/utils/infra_setup/runner/yardstick.py index 616bcc52..6071585c 100644 --- a/utils/infra_setup/runner/yardstick.py +++ b/utils/infra_setup/runner/yardstick.py @@ -45,7 +45,14 @@ def yardstick_command_parser(debug, cidr, outfile, parameter): cmd += " --output-file " + outfile image_name = config.bottlenecks_config["yardstick_image_name"] parameter["image_name"] = image_name - print parameter + DEPLOY_SCENARIO = os.getenv("DEPLOY_SCENARIO") + RAM_NUM = os.getenv("RAM_NUM") + if DEPLOY_SCENARIO: + if "ovs" in DEPLOY_SCENARIO: + parameter["dpdk_enabled"] = True + if RAM_NUM: + parameter["ram_num"] = RAM_NUM + LOG.info(parameter) if parameter is not None: cmd += " --task-args " + '"' + str(parameter) + '"' return cmd diff --git a/utils/dashboard/__init__.py b/utils/k8s_setup/__init__.py index e69de29b..e69de29b 100644 --- a/utils/dashboard/__init__.py +++ b/utils/k8s_setup/__init__.py diff --git a/utils/k8s_setup/golang_install.sh b/utils/k8s_setup/golang_install.sh new file mode 100644 index 00000000..06c54cee --- /dev/null +++ b/utils/k8s_setup/golang_install.sh @@ -0,0 +1,100 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +usage="Script to install and config golang of specific version. + +usage: + bash $(basename "$0") [-h|--help] [-v|--version <version>] [--debug] + +where: + -h|--help show the help text + -v|--version input the version of golang + --debug debug option switch +examples: + $(basename "$0") -v 1.10.3" + +# Debug option +redirect="/dev/null" + +# Process input variables +while [[ $# > 0 ]] + do + key="$1" + case $key in + -h|--help) + echo "$usage" + exit 0 + shift + ;; + -v|--version) + GOLANG_VERSION="$2" + shift + ;; + --debug) + redirect="/dev/stdout" + shift + ;; + *) + echo "unkown option $1 $2" + exit 1 + ;; + esac + shift +done + +#set -e + +echo "=======Downloading golang of version: ${GOLANG_VERSION}========" + +if [[ -f go${GOLANG_VERSION}.linux-amd64.tar.gz ]]; then + rm go${GOLANG_VERSION}.linux-amd64.tar.gz +fi +curl -O https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz >${redirect} + +echo "Installing golang of version: ${GOLANG_VERSION}" +if [[ -d /usr/local/go ]]; then + rm -rf /usr/local/go +fi + +tar -C /usr/local -xzf go${GOLANG_VERSION}.linux-amd64.tar.gz >${redirect} + +if [[ -d $HOME/go ]]; then + rm -rf ${HOME}/go + mkdir ${HOME}/go + mkdir ${HOME}/go/bin + mkdir ${HOME}/go/src +else + mkdir ${HOME}/go + mkdir ${HOME}/go/bin + mkdir ${HOME}/go/src +fi + +echo "Adding golang env to ~/.bashrc" +GOROOT=/usr/local/go +GOPATH=${HOME}/go + +if [[ $(cat ${HOME}/.bashrc | grep GOROOT) ]]; then + echo "golang env alreay in ${HOME}/.bashrc" +else + cat <<EOF >> ${HOME}/.bashrc + +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin +EOF +fi + +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin + +echo "Running go version command:" +go version + +echo "=======Installation of golang-${GOLANG_VERSION} complete=======" + diff --git a/utils/k8s_setup/k8s_config_pre.sh b/utils/k8s_setup/k8s_config_pre.sh new file mode 100644 index 00000000..f57add78 --- /dev/null +++ b/utils/k8s_setup/k8s_config_pre.sh @@ -0,0 +1,80 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +K8S_CONFIG="/tmp/k8s_config" + +usage="Script to prepare kubenetes test configurations. + +usage: + bash $(basename "$0") [-h|--help] [-i|--installer <installer typer>] [-c|--config <k8s config>] + +where: + -h|--help show the help text + -i|--installer specify the installer for the system to be monitored + <installer type> + one of the following: + (compass) +examples: + $(basename "$0") -i compass" + + +info () { + logger -s -t "BOTTLENECKS INFO" "$*" +} + +error () { + logger -s -t "BOTTLENECKS ERROR" "$*" + exit 1 +} + +# Process input variables +while [[ $# > 0 ]] + do + key="$1" + case $key in + -h|--help) + echo "$usage" + exit 0 + shift + ;; + -i|--installer) + INSTALLER_TYPE="$2" + shift + ;; + -c|--config) + K8S_CONFIG="$2" + shift + ;; + *) + error "unkown input options $1 $2" + exit 1 + ;; + esac + shift +done + +if [[ ${INSTALLER_TYPE} == 'compass' ]]; then + sshpass -p root scp -o StrictHostKeyChecking=no root@10.1.0.50:~/.kube/config ${K8S_CONFIG} +else + echo "BOTTLENECKS EROOR: unrecognized installer" +fi + +if [[ -f ${K8S_CONFIG} ]]; then + if [[ -d ~/.kube ]]; then + cp ${K8S_CONFIG} ~/.kube/config + echo "BOTTLENECKS INFO: copying k8s config to ~./kube" + else + mkdir ~/.kube + cp ${K8S_CONFIG} ~/.kube/config + echo "BOTTLENECKS INFO: copying k8s config to ~./kube" + fi +else + echo "BOTTLENECKS ERROR: k8s config file does no exit (${K8S_CONFIG})" + exit 1 +fi diff --git a/utils/dashboard/dashboard.yaml b/utils/k8s_setup/k8s_env.sh index 016c6d8d..855dea2f 100644 --- a/utils/dashboard/dashboard.yaml +++ b/utils/k8s_setup/k8s_env.sh @@ -1,15 +1,13 @@ +#!/bin/bash ############################################################################## -# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. -# +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## ---- +export GOROOT=/usr/local/go +export GOPATH=${HOME}/go +export PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin -pod_name: REPLACE_NODE_NAME -installer: REPLACE_INSTALLER_TYPE -version: REPLACE_VERSION -target: http://REPLACE_BOTTLENECKS_DB_TARGET/results diff --git a/utils/k8s_setup/k8s_utils.py b/utils/k8s_setup/k8s_utils.py new file mode 100644 index 00000000..ad2cc991 --- /dev/null +++ b/utils/k8s_setup/k8s_utils.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +import commands +import json +import utils.logger as log +from kubernetes import client, watch + + +LOG = log.Logger(__name__).getLogger() +INSTALLER_TYPE = os.getenv("INSTALLER_TYPE") +K8S_UTILS = "/home/opnfv/bottlenecks/utils/k8s_setup" + + +def get_config_path(INSTALLER_TYPE=None, K8S_CONFIG_PATH="/tmp/k8s_config"): + if INSTALLER_TYPE: + CMD = "bash " + K8S_UTILS + "/k8s_config_pre.sh -i " \ + + INSTALLER_TYPE + \ + " -c " + K8S_CONFIG_PATH + LOG.info("Executing command: " + CMD) + os.popen(CMD) + else: + if not os.path.exists(K8S_CONFIG_PATH): + raise Exception("Must at least specify the path \ +of k8s config!") + return K8S_CONFIG_PATH + + +def get_core_api(version='v1'): + if version.lower() == 'v1': + API = client.CoreV1Api() + LOG.info(API) + else: + raise Exception("Must input a valid verison!") + return API + + +def get_apps_api(version='v1'): + if version.lower() == 'v1': + API = client.AppsV1Api() + LOG.info(API) + else: + raise Exception("Must input a valid verison!") + return API + + +def get_namespace_status(namespace): + CMD = ("kubectl get ns | grep %s" % namespace) + namespace_existed = commands.getstatusoutput(CMD) + return namespace_existed + + +def get_deployment_status(name, namespace): + CMD = ("kubectl get deployment --namespace={} | grep {}".format( + namespace, name)) + deployment_existed = commands.getstatusoutput(CMD) + return deployment_existed + + +def get_available_pods(name, namespace): + CMD = ("kubectl get deployment --namespace={} | grep {}".format( + namespace, name) + " | awk '{print $4}'") + available_pods = commands.getstatusoutput(CMD) + return int(available_pods[1]) + + +def watch_namespace(namespace, count=3, stop=None, request_timeout=0): + w = watch.Watch() + LOG.debug("Watch object generated: {}".format(w)) + LOG.info("Watch stream generated: {}".format( + w.stream(namespace, _request_timeout=request_timeout))) + for event in w.stream(namespace, _request_timeout=request_timeout): + LOG.info("Event: %s %s" % + (event['type'], event['object'].metadata.name)) + if event['object'].metadata.name == stop: + LOG.info("Namesapce successfully added.\n") + w.stop() + count -= 1 + if not count: + LOG.info("Ended.\n") + w.stop() + + +def write_json(data, file_name): + with open(file_name, "a") as f: + f.write(json.dumps(data, f)) + f.write("\n") diff --git a/utils/k8s_setup/kubectl_install.sh b/utils/k8s_setup/kubectl_install.sh new file mode 100644 index 00000000..22c6197b --- /dev/null +++ b/utils/k8s_setup/kubectl_install.sh @@ -0,0 +1,33 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Tech and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +OS_TYPE=$(uname -a) +OS_UBUNTU=$(echo $OS_TYPE | grep Ubuntu) + +if [[ $OS_UBUNTU ]]; then + apt-get update && apt-get install -y apt-transport-https + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + touch /etc/apt/sources.list.d/kubernetes.list + echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list + apt-get update + apt-get install -y kubectl +else + cat <<EOF > /etc/yum.repos.d/kubernetes.repo +[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF + yum install -y kubectl +fi + diff --git a/utils/tools/dos2unix.sh b/utils/tools/dos2unix.sh new file mode 100644 index 00000000..6bfc8f9c --- /dev/null +++ b/utils/tools/dos2unix.sh @@ -0,0 +1,5 @@ +#!/bin/bash +for file in `find ./` +do +vi +':w ++ff=unix' +':q' ${file} +done |