diff options
Diffstat (limited to 'utils')
| -rw-r--r-- | utils/env_prepare/config_prepare.sh | 129 | ||||
| -rw-r--r-- | utils/env_prepare/moon_prepare.bash | 35 | ||||
| -rw-r--r-- | utils/env_prepare/moon_prepare.py | 25 | ||||
| -rw-r--r-- | utils/env_prepare/quota_prepare.py | 56 | ||||
| -rw-r--r-- | utils/env_prepare/stack_prepare.py | 47 | ||||
| -rw-r--r-- | utils/infra_setup/heat/manager.py | 2 | ||||
| -rw-r--r-- | utils/infra_setup/runner/docker_env.py | 2 | ||||
| -rw-r--r-- | utils/infra_setup/runner/yardstick.py | 15 | ||||
| -rw-r--r-- | utils/parser.py | 5 |
9 files changed, 235 insertions, 81 deletions
diff --git a/utils/env_prepare/config_prepare.sh b/utils/env_prepare/config_prepare.sh index 053c9da4..b13b5a02 100644 --- a/utils/env_prepare/config_prepare.sh +++ b/utils/env_prepare/config_prepare.sh @@ -18,7 +18,7 @@ where: -h|--help show the help text -i|--installer input the name of the installer <installer> one of the following: - (compass, fuel, joid, apex) + (apex, compass, fuel, joid) --debug debug option switch examples: @@ -45,6 +45,10 @@ while [[ $# > 0 ]] redirect="/dev/stdout" shift ;; + *) + echo "unkown option $1 $2" + exit 1 + ;; esac shift done @@ -59,22 +63,27 @@ error () { exit 1 } -# Define Variables -echo "BOTTLENECKS INFO: Downloading Releng" +# Repo and configs RELENG_REPO="/home/releng" +BOTTLENECKS_CONFIG=/tmp +OPENRC=${BOTTLENECKS_CONFIG}/admin_rc.sh +OS_CACERT=${BOTTLENECKS_CONFIG}/os_cacert + + +############################################################################## +# Preparing scripts for openstack and pod configs for OPNFV Installers +############################################################################## +# Define Variables +info "Downloading Releng fetch_os_creds script for openstack/pod configs of OPNFV installers" + [ -d ${RELENG_REPO} ] && rm -rf ${RELENG_REPO} git clone https://gerrit.opnfv.org/gerrit/releng ${RELENG_REPO} >${redirect} -echo "BOTTLENECKS INFO: Downloading Yardstick" +info "Downloading Yardstick for pod configs of OPNFV installers" YARDSTICK_REPO="/home/yardstick" [ -d ${YARDSTICK_REPO} ] && rm -rf ${YARDSTICK_REPO} git clone https://gerrit.opnfv.org/gerrit/yardstick ${YARDSTICK_REPO} >${redirect} -BOTTLENECKS_CONFIG=/tmp - -OPENRC=${BOTTLENECKS_CONFIG}/admin_rc.sh -OS_CACERT=${BOTTLENECKS_CONFIG}/os_cacert - # Preparing configuration files for testing if [[ ${INSTALLER_TYPE} != "" ]]; then # Preparing OpenStack RC and Cacert files @@ -84,73 +93,79 @@ if [[ ${INSTALLER_TYPE} != "" ]]; then INSTALLER_IP=192.168.200.2 if [[ ${BRANCH} == 'master' ]]; then ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${OS_CACERT} >${redirect} - if [[ -f ${OS_CACERT} ]]; then - echo "BOTTLENECKS INFO: successfully fetching os_cacert for openstack: ${OS_CACERT}" - else - echo "BOTTLENECKS ERROR: couldn't find os_cacert file: ${OS_CACERT}, please check if the it's been properly provided." - exit 1 - fi else ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} >${redirect} fi + elif [[ $INSTALLER_TYPE == 'apex' ]]; then + INSTALLER_IP=$(sudo virsh domifaddr undercloud | grep -Eo '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}') + ${RELENG_REPO}/utils/fetch_os_creds.sh -d ${OPENRC} -i ${INSTALLER_TYPE} -a ${INSTALLER_IP} -o ${OS_CACERT} >${redirect} + echo ${cmd} + ${cmd} else - error "The isntaller is not specified" - exit 1 - fi - - if [[ -f ${OPENRC} ]]; then - echo "BOTTLENECKS INFO: openstack credentials path is ${OPENRC}" - if [[ $INSTALLER_TYPE == 'compass' && ${BRANCH} == 'master' ]]; then - echo "BOTTLENECKS INFO: writing ${OS_CACERT} to ${OPENRC}" - echo "export OS_CACERT=${OS_CACERT}" >> ${OPENRC} - fi - cat ${OPENRC} - else - echo "BOTTLENECKS ERROR: couldn't find openstack rc file: ${OPENRC}, please check if the it's been properly provided." + error "The installer is not specified" exit 1 fi # Finding and crearting POD description files from different deployments - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - - if [ "$INSTALLER_TYPE" == "fuel" ]; then - echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sshpass -p r00tme sudo scp $ssh_options root@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa - fi + if [[ ${INSTALLER_TYPE} == 'compass' ]]; then + cmd="sudo cp ${YARDSTICK_REPO}/etc/yardstick/nodes/compass_sclab_virtual/pod.yaml \ + ${BOTTLENECKS_CONFIG}" + info ${cmd} + ${cmd} + elif [[ ${INSTALLER_TYPE} == 'apex' ]]; then + sudo pip install virtualenv - if [ "$INSTALLER_TYPE" == "apex" ]; then - echo "Fetching id_rsa file from jump_server $INSTALLER_IP..." - sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa - fi + sudo virtualenv venv + source venv/bin/activate + sudo pip install -e ${RELENG_REPO}/modules/ >/dev/null + sudo pip install netaddr - if [[ ${INSTALLER_TYPE} == compass ]]; then - options="-u root -p root" - elif [[ ${INSTALLER_TYPE} == fuel ]]; then - options="-u root -p r00tme" - elif [[ ${INSTALLER_TYPE} == apex ]]; then options="-u stack -k /root/.ssh/id_rsa" - else - echo "Don't support to generate pod.yaml on ${INSTALLER_TYPE} currently." - fi - - if [[ ${INSTALLER_TYPE} != compass ]]; then cmd="sudo python ${RELENG_REPO}/utils/create_pod_file.py -t ${INSTALLER_TYPE} \ - -i ${INSTALLER_IP} ${options} -f ${BOTTLENECKS_CONFIG}/pod.yaml \ - -s ${BOTTLENECKS_CONFIG}/id_rsa" - echo ${cmd} + -i ${INSTALLER_IP} ${options} -f ${BOTTLENECKS_CONFIG}/pod.yaml" + info ${cmd} ${cmd} - else - cmd="sudo cp ${YARDSTICK_REPO}/etc/yardstick/nodes/compass_sclab_virtual/pod.yaml \ - ${BOTTLENECKS_CONFIG}" - echo ${cmd} + + ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + cmd="sudo scp $ssh_options stack@${INSTALLER_IP}:~/.ssh/id_rsa ${BOTTLENECKS_CONFIG}/id_rsa" + info ${cmd} ${cmd} + + deactivate fi + + ############################################################################## + # Check the existence of the output configs for OPNFV Installers + ############################################################################## + # Checking the pod decription file if [ -f ${BOTTLENECKS_CONFIG}/pod.yaml ]; then - echo "FILE: ${BOTTLENECKS_CONFIG}/pod.yaml:" + info "FILE - ${BOTTLENECKS_CONFIG}/pod.yaml:" cat ${BOTTLENECKS_CONFIG}/pod.yaml else - echo "ERROR: cannot find file ${BOTTLENECKS_CONFIG}/pod.yaml. Please check if it is existing." + error "Cannot find file ${BOTTLENECKS_CONFIG}/pod.yaml. Please check if it is existing." sudo ls -al ${BOTTLENECKS_CONFIG} fi + + # Checking the openstack rc and os_cacert file + if [[ -f ${OPENRC} ]]; then + info "Opentack credentials path is ${OPENRC}" + if [[ -f ${OS_CACERT} ]]; then + info "Writing ${OS_CACERT} to ${OPENRC}" + echo "export OS_CACERT=${OS_CACERT}" >> ${OPENRC} + cat ${OPENRC} + else + error "Couldn't find openstack cacert file: ${OS_CACERT}, please check if the it's been properly provided." + fi + else + error "Couldn't find openstack rc file: ${OPENRC}, please check if the it's been properly provided." + exit 1 + fi + + # Checking ssh key id_rsa + if [[ -f "/tmp/id_rsa" ]]; then + info "Path of ssh key file for openstack nodes is /tmp/id_rsa" + else + error "Couldn't find the ssh key file for openstack nodes. If you are using user/pwd in pod.yaml, please ignore." + fi fi diff --git a/utils/env_prepare/moon_prepare.bash b/utils/env_prepare/moon_prepare.bash new file mode 100644 index 00000000..7625418c --- /dev/null +++ b/utils/env_prepare/moon_prepare.bash @@ -0,0 +1,35 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +if grep -q "cadvisor" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf +then + sed -e "/cadvisor-port=0/d" -i /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + systemctl daemon-reload + systemctl restart kubelet + +fi +if kubectl get po -n monitoring |grep prometheus-k8s |grep -q Running +then + echo "monitoring k8s deployment has been done" +else + git clone https://github.com/coreos/prometheus-operator.git + cd prometheus-operator + kubectl apply -n kube-system -f bundle.yaml + cd contrib/kube-prometheus + sleep 10 + hack/cluster-monitoring/deploy +fi + +while ! $(kubectl get po -n monitoring |grep prometheus-k8s |grep -q Running);do + echo "waiting for monitoring deployment finish!" + sleep 10 +done + +echo "waiting for monitoring tool works" +sleep 60 diff --git a/utils/env_prepare/moon_prepare.py b/utils/env_prepare/moon_prepare.py new file mode 100644 index 00000000..41739454 --- /dev/null +++ b/utils/env_prepare/moon_prepare.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import utils.infra_setup.passwordless_SSH.ssh as localssh + + +def moon_envprepare(host_info): + if ("password") in host_info: + client = localssh.SSH(user=host_info["user"], + host=host_info["ip"], + password=host_info["password"]) + else: + client = localssh.SSH(user=host_info["user"], + host=host_info["ip"], + key_filename=host_info["keyfile"]) + with open("/home/opnfv/bottlenecks/utils/env_prepare/moon_prepare.bash", + "rb") as stdin_file: + client.run("cat > ~/bottlenecks_envprepare.bash", stdin=stdin_file) + client.execute("sudo bash ~/bottlenecks_envprepare.bash") diff --git a/utils/env_prepare/quota_prepare.py b/utils/env_prepare/quota_prepare.py index 267e70ab..367f7613 100644 --- a/utils/env_prepare/quota_prepare.py +++ b/utils/env_prepare/quota_prepare.py @@ -42,18 +42,52 @@ nova_quota = {"ram": -1, "injected_file_path_bytes": -1} +def check_https_enabled(): + LOG.debug("Check if https is enabled in OpenStack") + os_auth_url = os.getenv('OS_AUTH_URL') + if os_auth_url.startswith('https'): + LOG.debug("https is enabled") + return True + LOG.debug("https is not enabled") + return False + + def quota_env_prepare(): - tenant_name = os.getenv("OS_TENANT_NAME") - cmd = ("openstack project list | grep " + - tenant_name + - " | awk '{print $2}'") + https_enabled = check_https_enabled() + insecure_option = '' + insecure = os.getenv('OS_INSECURE',) + if https_enabled: + LOG.info("https is enabled") + if insecure: + if insecure.lower() == "true": + insecure_option = ' --insecure ' + else: + LOG.warn("Env variable OS_INSECURE is {}: if https + no " + "credential used, it should be set as True." + .format(insecure)) - result = commands.getstatusoutput(cmd) - if result[0] == 0: - LOG.info("Get %s project id is %s" % (tenant_name, result[1])) + quota_name = os.getenv("OS_PROJECT_NAME") + if quota_name: + cmd = ("openstack {} project list | grep ".format(insecure_option) + + quota_name + + " | awk '{print $2}'") + result = commands.getstatusoutput(cmd) + if result[0] == 0 and 'exception' not in result[1]: + LOG.info("Get %s project name is %s" % (quota_name, result[1])) + else: + LOG.error("can't get openstack project name") + return 1 else: - LOG.error("can't get openstack project id") - return 1 + quota_name = os.getenv("OS_TENANT_NAME") + cmd = ("openstack {} tenant list | grep ".format(insecure_option) + + quota_name + + " | awk '{print $2}'") + result = commands.getstatusoutput(cmd) + if result[0] == 0 and 'exception' not in result[1]: + LOG.info("Get %s tenant name is %s" % (quota_name, result[1])) + else: + LOG.error("can't get openstack tenant name") + return 1 openstack_id = result[1] @@ -62,7 +96,7 @@ def quota_env_prepare(): nova_q = nova_client.quotas.get(openstack_id).to_dict() neutron_q = neutron_client.show_quota(openstack_id) - LOG.info(tenant_name + "tenant nova and neutron quota(previous) :") + LOG.info(quota_name + " nova and neutron quotas (previous) :") LOG.info(nova_q) LOG.info(neutron_q) @@ -73,7 +107,7 @@ def quota_env_prepare(): nova_q = nova_client.quotas.get(openstack_id).to_dict() neutron_q = neutron_client.show_quota(openstack_id) - LOG.info(tenant_name + "tenant nova and neutron quota(now) :") + LOG.info(quota_name + " nova and neutron quotas (now) :") LOG.info(nova_q) LOG.info(neutron_q) return 0 diff --git a/utils/env_prepare/stack_prepare.py b/utils/env_prepare/stack_prepare.py index c7dae390..6b9bc510 100644 --- a/utils/env_prepare/stack_prepare.py +++ b/utils/env_prepare/stack_prepare.py @@ -14,6 +14,7 @@ from utils.logger import Logger from utils.parser import Parser as config import utils.infra_setup.heat.manager as utils import utils.infra_setup.runner.docker_env as docker_env +import utils.infra_setup.heat.manager as client_manager LOG = Logger(__name__).getLogger() @@ -37,16 +38,6 @@ def _prepare_env_daemon(test_yardstick): config.bottlenecks_config["yardstick_rc_dir"]) docker_env.docker_exec_cmd(yardstick_contain, cmd) - file_orig = ("/home/opnfv/repos/yardstick/etc" - "/yardstick/yardstick.conf.sample") - file_after = "/etc/yardstick/yardstick.conf" - cmd = "cp %s %s" % (file_orig, - file_after) - docker_env.docker_exec_cmd(yardstick_contain, - cmd) - cmd = "sed -i '12s/http/file/g' /etc/yardstick/yardstick.conf" - docker_env.docker_exec_cmd(yardstick_contain, - cmd) # update the external_network _source_file(rc_file) @@ -83,7 +74,14 @@ def _source_file(rc_file): p = subprocess.Popen(". %s; env" % rc_file, stdout=subprocess.PIPE, shell=True) output = p.communicate()[0] - env = dict((line.split('=', 1) for line in output.splitlines())) + output_lines = output.splitlines() + env = list() + for line in output_lines: + if '=' in line: + env.append(tuple(line.split('=', 1))) + + env = dict(env) +# env = dict((line.split('=', 1) for line in output_lines)) os.environ.update(env) return env @@ -103,3 +101,30 @@ def _append_external_network(rc_file): except OSError as e: if e.errno != errno.EEXIST: raise + + +def prepare_image(image_name, image_dir): + glance_client = client_manager._get_glance_client() + if not os.path.isfile(image_dir): + LOG.error("Error: file %s does not exist.", image_dir) + return None + try: + images = glance_client.images.list() + image_id = next((i.id for i in images if i.name == image_name), None) + if image_id is not None: + LOG.info("Image %s already exists.", image_name) + else: + LOG.info("Creating image '%s' from '%s'...", image_name, image_dir) + + image = glance_client.images.create( + name=image_name, visibility="public", disk_format="qcow2", + container_format="bare") + image_id = image.id + with open(image_dir) as image_data: + glance_client.images.upload(image_id, image_data) + return image_id + except Exception: # pylint: disable=broad-except + LOG.error( + "Error [create_glance_image(glance_client, '%s', '%s')]", + image_name, image_dir) + return None diff --git a/utils/infra_setup/heat/manager.py b/utils/infra_setup/heat/manager.py index 5c181ae6..1a360b78 100644 --- a/utils/infra_setup/heat/manager.py +++ b/utils/infra_setup/heat/manager.py @@ -35,7 +35,7 @@ def _get_neutron_client(): return neutron_client -def stack_create_images( +def create_images( imagefile=None, image_name="bottlenecks_image"): print "========== Create image in OS ==========" diff --git a/utils/infra_setup/runner/docker_env.py b/utils/infra_setup/runner/docker_env.py index 54478ffc..438d3d19 100644 --- a/utils/infra_setup/runner/docker_env.py +++ b/utils/infra_setup/runner/docker_env.py @@ -45,7 +45,7 @@ def env_yardstick(docker_name): volume = get_self_volume() yardstick_tag = os.getenv("Yardstick_TAG") if yardstick_tag is None: - yardstick_tag = "danube.3.1" + yardstick_tag = "latest" env_docker = client.containers.run(image="opnfv/yardstick:%s" % yardstick_tag, privileged=True, diff --git a/utils/infra_setup/runner/yardstick.py b/utils/infra_setup/runner/yardstick.py index 559b9c10..3eeeee6b 100644 --- a/utils/infra_setup/runner/yardstick.py +++ b/utils/infra_setup/runner/yardstick.py @@ -15,15 +15,27 @@ At present, This file contain the following function: 4.how the process of task.''' import sys +import os import time import requests import json +import urllib import utils.logger as logger +from utils.parser import Parser as config +import utils.env_prepare.stack_prepare as env headers = {"Content-Type": "application/json"} LOG = logger.Logger(__name__).getLogger() +def yardstick_image_prepare(): + if not os.path.exists(config.bottlenecks_config["yardstick_image_dir"]): + urllib.urlretrieve(config.bottlenecks_config["image_url"], + config.bottlenecks_config["yardstick_image_dir"]) + env.prepare_image(config.bottlenecks_config["yardstick_image_name"], + config.bottlenecks_config["yardstick_image_dir"]) + + def yardstick_command_parser(debug, cidr, outfile, parameter): cmd = "yardstick" if debug: @@ -31,6 +43,9 @@ def yardstick_command_parser(debug, cidr, outfile, parameter): cmd += " task start " cmd += str(cidr) cmd += " --output-file " + outfile + image_name = config.bottlenecks_config["yardstick_image_name"] + parameter["image_name"] = image_name + LOG.info(parameter) if parameter is not None: cmd += " --task-args " + '"' + str(parameter) + '"' return cmd diff --git a/utils/parser.py b/utils/parser.py index b46a3b91..c0c10721 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -45,6 +45,11 @@ class Parser(): cls.bottlenecks_config["pod_info"] = common_config['pod_info'] cls.bottlenecks_config["yardstick_rc_dir"] = \ common_config['yardstick_rc_dir'] + cls.bottlenecks_config["yardstick_image_dir"] = \ + common_config['yardstick_image_dir'] + cls.bottlenecks_config["image_url"] = common_config['image_url'] + cls.bottlenecks_config["yardstick_image_name"] = \ + common_config['yardstick_image_name'] cls.config_dir_check(cls.bottlenecks_config["log_dir"]) @classmethod |