diff options
| -rw-r--r-- | config/config.yaml | 5 | ||||
| -rw-r--r-- | utils/env_prepare/__init__.py | 8 | ||||
| -rw-r--r-- | utils/env_prepare/stack_prepare.py | 79 | ||||
| -rwxr-xr-x | utils/infra_setup/fetch_os_creds.sh | 169 | ||||
| -rw-r--r-- | utils/infra_setup/heat/manager.py | 7 | ||||
| -rw-r--r-- | utils/logger.py | 19 | ||||
| -rw-r--r-- | utils/parser.py | 59 |
7 files changed, 145 insertions, 201 deletions
diff --git a/config/config.yaml b/config/config.yaml index cf380e27..702a0c8c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,2 +1,5 @@ common_config: - log_dir: '/tmp/bottlenecks_POSCA.log'
\ No newline at end of file + log_dir: '/tmp/' + rc_dir: '/tmp/admin_rc.sh' + releng_dir: '/home/opnfv/releng' + fetch_os_file: 'utils/fetch_os_creds.sh'
\ No newline at end of file diff --git a/utils/env_prepare/__init__.py b/utils/env_prepare/__init__.py new file mode 100644 index 00000000..b124dfa9 --- /dev/null +++ b/utils/env_prepare/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/utils/env_prepare/stack_prepare.py b/utils/env_prepare/stack_prepare.py new file mode 100644 index 00000000..37b523d1 --- /dev/null +++ b/utils/env_prepare/stack_prepare.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import subprocess +import errno +from utils.logger import Logger +from utils.parser import Parser as config +import utils.infra_setup.heat.manager as utils + +LOG = Logger(__name__).getLogger() + + +def _prepare_env_daemon(): + + installer_ip = os.environ.get('INSTALLER_IP', 'undefined') + installer_type = os.environ.get('INSTALLER_TYPE', 'undefined') + + rc_file = config.bottlenecks_config["rc_dir"] + + _get_remote_rc_file(rc_file, installer_ip, installer_type) + + _source_file(rc_file) + + _append_external_network(rc_file) + + # update the external_network + _source_file(rc_file) + + +def _get_remote_rc_file(rc_file, installer_ip, installer_type): + + RELENG_DIR = config.bottlenecks_config["releng_dir"] + OS_FETCH_SCRIPT = config.bottlenecks_config["fetch_os"] + os_fetch_script = os.path.join(RELENG_DIR, OS_FETCH_SCRIPT) + + try: + cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, + '-a', installer_ip] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p.communicate()[0] + + if p.returncode != 0: + LOG.debug('Failed to fetch credentials from installer') + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def _source_file(rc_file): + p = subprocess.Popen(". %s; env" % rc_file, stdout=subprocess.PIPE, + shell=True) + output = p.communicate()[0] + env = dict((line.split('=', 1) for line in output.splitlines())) + os.environ.update(env) + return env + + +def _append_external_network(rc_file): + neutron_client = utils._get_neutron_client() + networks = neutron_client.list_networks()['networks'] + try: + ext_network = next(n['name'] for n in networks if n['router:external']) + except StopIteration: + LOG.warning("Can't find external network") + else: + cmd = 'export EXTERNAL_NETWORK=%s' % ext_network + try: + with open(rc_file, 'a') as f: + f.write(cmd + '\n') + except OSError as e: + if e.errno != errno.EEXIST: + raise diff --git a/utils/infra_setup/fetch_os_creds.sh b/utils/infra_setup/fetch_os_creds.sh deleted file mode 100755 index 4ac7162e..00000000 --- a/utils/infra_setup/fetch_os_creds.sh +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# jose.lausuch@ericsson.com matthew.lijun@huawei.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -usage() { - echo "usage: $0 -d <destination> -i <installer_type> -a <installer_ip>" >&2 -} - -info () { - logger -s -t "fetch_os_creds.info" "$*" -} - - -error () { - logger -s -t "fetch_os_creds.error" "$*" - exit 1 -} - - -verify_connectivity() { - local ip=$1 - info "Verifying connectivity to $ip..." - for i in $(seq 0 10); do - if ping -c 1 -W 1 $ip > /dev/null; then - info "$ip is reachable!" - return 0 - fi - sleep 1 - done - error "Can not talk to $ip." -} - - - -#Get options -while getopts ":d:i:a:h:" optchar; do - case "${optchar}" in - d) dest_path=${OPTARG} ;; - i) installer_type=${OPTARG} ;; - a) installer_ip=${OPTARG} ;; - *) echo "Non-option argument: '-${OPTARG}'" >&2 - usage - exit 2 - ;; - esac -done - -# set vars from env if not provided by user as options -dest_path=${dest_path:-$HOME/opnfv-openrc.sh} -installer_type=${installer_type:-$INSTALLER_TYPE} -installer_ip=${installer_ip:-$INSTALLER_IP} - -if [ -z $dest_path ] || [ -z $installer_type ] || [ -z $installer_ip ]; then - usage - exit 2 -fi - -# Checking if destination path is valid -if [ -d $dest_path ]; then - error "Please provide the full destination path for the credentials file including the filename" -else - # Check if we can create the file (e.g. path is correct) - touch $dest_path || error "Cannot create the file specified. Check that the path is correct and run the script again." -fi - - -ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -# Start fetching the files -if [ "$installer_type" == "fuel" ]; then - #ip_fuel="10.20.0.2" - verify_connectivity $installer_ip - - # Check if controller is alive (online='True') - controller_ip=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ - 'fuel node | grep controller | grep True | awk "{print \$10}" | tail -1') &> /dev/null - - if [ -z $controller_ip ]; then - error "The controller $controller_ip is not up. Please check that the POD is correctly deployed." - fi - - info "Fetching rc file from controller $controller_ip..." - sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ - "scp $ssh_options ${controller_ip}:/root/openrc ." &> /dev/null - sshpass -p r00tme scp 2>/dev/null $ssh_options root@${installer_ip}:~/openrc $dest_path &> /dev/null - - #This file contains the mgmt keystone API, we need the public one for our rc file - admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') - public_ip=$(sshpass -p r00tme ssh $ssh_options root@${installer_ip} \ - "ssh ${controller_ip} 'source openrc; keystone endpoint-list'" \ - | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) &> /dev/null - #| grep http | head -1 | cut -d '|' -f 4 | sed 's/v1\/.*/v1\//' | sed 's/ //g') &> /dev/null - #NOTE: this is super ugly sed 's/v1\/.*/v1\//'OS_AUTH_URL - # but sometimes the output of endpoint-list is like this: http://172.30.9.70:8004/v1/%(tenant_id)s - - -elif [ "$installer_type" == "foreman" ]; then - #ip_foreman="172.30.10.73" - controller="oscontroller1.opnfv.com" - verify_connectivity $installer_ip - - # Check if controller is alive (here is more difficult to get the ip from a command like "fuel node") - sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ - "sshpass -p Op3nStack ssh $ssh_options root@${controller} 'ls'" &> /dev/null - if [ $? -ne 0 ]; then - error "The controller ${controller} is not up. Please check that the POD is correctly deployed." - fi - - info "Fetching openrc from a Foreman Controller '${controller}'..." - sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ - "sshpass -p Op3nStack scp $ssh_options root@${controller}:~/keystonerc_admin ." &> /dev/null - sshpass -p vagrant scp $ssh_options root@${installer_ip}:~/keystonerc_admin $dest_path &> /dev/null - - #This file contains the mgmt keystone API, we need the public one for our rc file - admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') - public_ip=$(sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ - "sshpass -p Op3nStack ssh $ssh_options root@${controller} \ - 'source keystonerc_admin;keystone endpoint-list'" \ - | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) &> /dev/null - -elif [ "$installer_type" == "compass" ]; then - verify_connectivity $installer_ip - controller_ip=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ip} \ - 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \ - | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host1\"/) {print $(i+1);break;}}' \ - | grep -oP "\d+.\d+.\d+.\d+") - - if [ -z $controller_ip ]; then - error "The controller $controller_ip is not up. Please check that the POD is correctly deployed." - fi - - info "Fetching rc file from controller $controller_ip..." - sshpass -p root ssh 2>/dev/null $ssh_options root@${installer_ip} \ - "scp $ssh_options ${controller_ip}:/opt/admin-openrc.sh ." &> /dev/null - sshpass -p root scp 2>/dev/null $ssh_options root@${installer_ip}:~/admin-openrc.sh $dest_path &> /dev/null - echo 'export OS_REGION_NAME=regionOne' >> $dest_path - - info "This file contains the mgmt keystone API, we need the public one for our rc file" - admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') - info "admin_ip: $admin_ip" - public_ip=$(sshpass -p root ssh $ssh_options root@${installer_ip} \ - "ssh ${controller_ip} 'source /opt/admin-openrc.sh; keystone endpoint-list'" \ - | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) - info "public_ip: $public_ip" - -else - error "Installer $installer is not supported by this script" -fi - - - -if [ "$public_ip" == "" ]; then - error "Cannot retrieve the public IP from keystone" -fi - -info "Keystone public IP is $public_ip" -sed -i "/OS_AUTH_URL/c\export OS_AUTH_URL=\'$public_ip'" $dest_path - -echo "-------- Credentials: --------" -cat $dest_path - -exit 0 diff --git a/utils/infra_setup/heat/manager.py b/utils/infra_setup/heat/manager.py index 3a270ac1..5c181ae6 100644 --- a/utils/infra_setup/heat/manager.py +++ b/utils/infra_setup/heat/manager.py @@ -11,6 +11,7 @@ import time import common as op_utils from glanceclient.client import Client as GlanceClient from novaclient.client import Client as NovaClient +from neutronclient.v2_0.client import Client as NeutronClient def _get_glance_client(): @@ -28,6 +29,12 @@ def _get_nova_client(): session=sess) +def _get_neutron_client(): + sess = op_utils.get_session() + neutron_client = NeutronClient(session=sess) + return neutron_client + + def stack_create_images( imagefile=None, image_name="bottlenecks_image"): diff --git a/utils/logger.py b/utils/logger.py index 0f93bca7..23d749b6 100644 --- a/utils/logger.py +++ b/utils/logger.py @@ -19,15 +19,16 @@ import logging import os -# from bottlenecks_cfg import Bottlenecks_cfg as bn_cfg +from utils.parser import Parser as conf class Logger: def __init__(self, logger_name): - #if user set --debug as a cli parameter - #we will set this variable “Debug” to output debug info. + # if user set --debug as a cli parameter + # we will set this variable "Debug" to output debug info. DEBUG = os.getenv('DEBUG') + conf = Parser() self.logger = logging.getLogger(logger_name) self.logger.propagate = 0 @@ -47,13 +48,15 @@ class Logger: ch.setLevel(logging.INFO) self.logger.addHandler(ch) -# result_path = bn_cfg.['log_dir'] -# if not os.path.exists(result_path): -# os.makedirs(result_path) - hdlr = logging.FileHandler('/tmp/bottlenecks.log') + result_path = conf.bottlenecks_config["log_dir"] + if not os.path.exists(result_path): + os.makedirs(result_path) + result_file = os.path.join(result_path, 'bottlenecks.log') + hdlr = logging.FileHandler(result_file) + hdlr.setFormatter(formatter) hdlr.setLevel(logging.DEBUG) self.logger.addHandler(hdlr) def getLogger(self): - return self.logger
\ No newline at end of file + return self.logger diff --git a/utils/parser.py b/utils/parser.py index 7ddf02a2..ce5096a3 100644 --- a/utils/parser.py +++ b/utils/parser.py @@ -7,50 +7,63 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from logger import Logger import os import yaml class Parser(): - def __init__(self): - self.code_dir = os.path.dirname(os.path.abspath(__file__)) - self.root_dir = os.path.dirname(self.code_dir) - self.test_dir = os.path.join(self.root_dir, 'testsuites') + + bottlenecks_config = {} + + @classmethod + def config_init(cls): + cls.code_dir = os.path.dirname(os.path.abspath(__file__)) + cls.root_dir = os.path.dirname(cls.code_dir) + cls.test_dir = os.path.join(cls.root_dir, 'testsuites') config_dir = os.path.join( - self.root_dir, + cls.root_dir, 'config', 'config.yaml') + with open(config_dir) as file: - log_info = yaml.load(file) - self.logdir = log_info['common_config'] - self.LOG = Logger(__name__).getLogger() - - def config_read(self, testcase, story_name): - self.LOG.info("begin to parser config file!") - testcase_parser = {} - self.story_dir = os.path.join( - self.test_dir, + config_info = yaml.load(file) + common_config = config_info['common_config'] + cls.bottlenecks_config["releng_dir"] = common_config["releng_dir"] + cls.bottlenecks_config["fetch_os"] = common_config["fetch_os_file"] + cls.bottlenecks_config["log_dir"] = common_config['log_dir'] + cls.bottlenecks_config["rc_dir"] = common_config['rc_dir'] + cls.config_dir_check(cls.bottlenecks_config["log_dir"]) + + @classmethod + def config_read(cls, testcase, story_name): + story_dir = os.path.join( + cls.test_dir, testcase, 'testsuite_story', story_name) - with open(self.story_dir) as file: - self.LOG.info('testsuite:' + testcase + 'story:' + story_name) + with open(story_dir) as file: story_parser = yaml.load(file) for case_name in story_parser['testcase']: testcase_dir = os.path.join( - self.test_dir, + cls.test_dir, testcase, 'testcase_cfg', case_name) with open(testcase_dir) as f: - self.LOG.info('story: %s, testcase: %s' % (story_name, case_name)) - testcase_parser[case_name] = yaml.load(f) + cls.bottlenecks_config[case_name] = yaml.load(f) + + return cls.bottlenecks_config - return testcase_parser + @classmethod + def config_dir_check(cls, dirname): + if dirname is None: + dirname = '/tmp/' + if not os.path.exists(dirname): + os.makedirs(dirname) - def config_parser(self, testcase_cfg, parameters): + @staticmethod + def config_parser(testcase_cfg, parameters): test_cfg = testcase_cfg['test_config'] stack_cfg = testcase_cfg['stack_config'] # TO-DO add cli parameters to stack_config. - return test_cfg, stack_cfg
\ No newline at end of file + return test_cfg, stack_cfg |