summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--func/driver.py96
-rw-r--r--func/env_setup.py59
-rwxr-xr-xfunc/fetch_compute_ips.sh117
-rw-r--r--tests/cli_test.py4
-rw-r--r--tests/driver_test.py75
-rw-r--r--tests/env_setup_test.py49
6 files changed, 324 insertions, 76 deletions
diff --git a/func/driver.py b/func/driver.py
index 33dbe320..291591b1 100644
--- a/func/driver.py
+++ b/func/driver.py
@@ -8,53 +8,67 @@
##############################################################################
import os
import json
-from collections import defaultdict
+import logging
class Driver:
def __init__(self):
- print "Class driver initialized\n"
- print os.environ['PWD']
- self.dic_json = defaultdict()
+ logging.info("Class driver initialized\n")
+ logging.info(os.environ['PWD'])
+ self.installer_username = {'fuel': 'root',
+ 'joid': 'ubuntu',
+ 'apex': 'heat-admin'}
- def drive_bench(self, benchmark, roles, benchmark_fname, benchmark_detail=None, pip_dict=None, proxy_info=None):
+ @staticmethod
+ def merge_two_dicts(x, y):
+ '''
+ It is from http://stackoverflow.com/questions/38987/
+ how-can-i-merge-two-python-dictionaries-in-a-single-expression
+ '''
+ z = x.copy()
+ z.update(y)
+ return z
+
+ def get_common_var_json(self, benchmark_fname, benchmark_detail, pip_dict, proxy_info):
+ common_json = {'Dest_dir': 'results',
+ 'ip1': '',
+ 'ip2': '',
+ 'installer': str(os.environ['INSTALLER_TYPE']),
+ 'workingdir': str(os.environ['PWD']),
+ 'fname': str(benchmark_fname),
+ 'username': self.installer_username[str(os.environ['INSTALLER_TYPE'])]}
+ common_json.update(benchmark_detail) if benchmark_detail else None
+ common_json.update(proxy_info) if proxy_info else None
+ return common_json
+
+ def get_special_var_json(self, role, roles, benchmark_detail, pip_dict):
+ special_json = {}
+ index = roles.index(role) + 1
+ special_json.update({'role': role[0]})
+ private_ip = pip_dict[0][1] if pip_dict[0][1][0] else 'NONE'
+ map(lambda x: special_json.update({'ip' + str(index): x}), role[1])\
+ if benchmark_detail and (role[0] == '1-server') else None
+ map(lambda x: special_json.update({'privateip' + str(index): private_ip}), role[1])\
+ if benchmark_detail and (role[0] == '1-server') else None
+ return special_json
+
+ def run_ansible_playbook(self, benchmark, extra_vars):
+ extra_vars_json = json.dumps(dict(extra_vars.items()))
+ logging.info(extra_vars_json)
+ run_play = 'ansible-playbook ./benchmarks/playbooks/{0}.yaml' \
+ '--private-key=./data/QtipKey -i ./data/hosts --extra-vars \'{1}\'' \
+ .format(benchmark, extra_vars_json)
+ os.system(run_play)
+
+ def drive_bench(self, benchmark, roles, benchmark_fname,
+ benchmark_detail=None, pip_dict=None, proxy_info=None):
roles = sorted(roles)
pip_dict = sorted(pip_dict)
- result_dir = 'results'
- benchmark_name = benchmark + '.yaml'
- self.dic_json['Dest_dir'] = str(result_dir)
- self.dic_json['ip1'] = ''
- self.dic_json['ip2'] = ''
- self.dic_json['installer'] = str(os.environ['INSTALLER_TYPE'])
- self.dic_json['workingdir'] = str(os.environ['PWD'])
- self.dic_json['fname'] = str(benchmark_fname)
- self.dic_json['username'] = str('root')
-
- for key in proxy_info.keys():
- self.dic_json[key] = proxy_info[key]
-
- if os.environ['INSTALLER_TYPE'] == str('joid'):
- self.dic_json['username'] = str('ubuntu')
- if os.environ['INSTALLER_TYPE'] == str('apex'):
- self.dic_json['username'] = str('heat-admin')
- for k, v in benchmark_detail:
- self.dic_json[k] = v
- for k, v in roles:
- self.dic_json['role'] = k
- index = 1
- if benchmark_detail is not None:
- for values in v:
- if k == '1-server':
- print values, 'saving IP'
- self.dic_json['ip' + str(index)] = str(values)
- if pip_dict[0][1][0]:
- self.dic_json['privateip' + str(index)] = pip_dict[0][1]
- if not pip_dict[0][1][0]:
- self.dic_json['privateip' + str(index)] = 'NONE'
- index = index + 1
- dic_json = json.dumps(dict(self.dic_json.items()))
- print dic_json
- run_play = 'ansible-playbook ./benchmarks/playbooks/{0} --private-key=./data/QtipKey -i ./data/hosts --extra-vars \'{1}\''.format(benchmark_name, dic_json)
- os.system(run_play)
+ var_json = self.get_common_var_json(benchmark_fname, benchmark_detail, pip_dict, proxy_info)
+ map(lambda role: self.run_ansible_playbook
+ (benchmark, self.merge_two_dicts(var_json,
+ self.get_special_var_json(role, roles,
+ benchmark_detail,
+ pip_dict))), roles)
diff --git a/func/env_setup.py b/func/env_setup.py
index f59883a0..1f86f0ea 100644
--- a/func/env_setup.py
+++ b/func/env_setup.py
@@ -1,5 +1,5 @@
##############################################################################
-# Copyright (c) 2015 Dell Inc and others.
+# Copyright (c) 2016 Dell Inc, ZTE and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
@@ -14,9 +14,16 @@ import yaml
import time
import paramiko
import socket
+from os.path import expanduser
+import random
+import logging
+
+LOG = logging.getLogger(__name__)
+LOG.setLevel(logging.DEBUG)
class Env_setup:
+
roles_ip_list = [] # ROLE and its corresponding IP address list
ip_pw_list = [] # IP and password, this will be used to ssh
roles_dict = defaultdict(list)
@@ -112,10 +119,58 @@ class Env_setup:
break
print ('\n\n %s is UP \n\n ' % ipvar)
- def get_host_machine_info(self, host_tag):
+ @staticmethod
+ def fetch_compute_ips():
+ LOG.info("Fetch compute ips through installer")
+ ips = []
+
+ installer_type = os.environ['INSTALLER_TYPE']
+ installer_ip = os.environ['INSTALLER_IP']
+ if installer_type.down.lower() != "fuel" or "compass":
+ raise RuntimeError("%s is not supported" % installer_type)
+ if installer_ip:
+ raise RuntimeError("undefine environment variable INSTALLER_IP")
+
+ cmd = "bash ./fetch_compute_ip.sh -i %s -a %s" % \
+ (installer_type, installer_ip)
+ os.system(cmd)
+ home = expanduser("~")
+ os.chdir(home)
+ with open("ips.log", "r") as file:
+ data = file.read()
+ if data:
+ ips.extend(data.rstrip('\n').split('\n'))
+ LOG.info("All compute ips: %s" % ips)
+ return ips
+
+ def check_machine_ips(self, host_tag):
+ LOG.info("Check machine ips")
+ ips = self.fetch_compute_ips()
+ ips_num = len(ips)
+ num = len(host_tag)
+ if num > ips_num:
+ err = "host num %s > compute ips num %s" % (num, ips_num)
+ raise RuntimeError(err)
+ for x in range(num):
+ hostlabel = 'machine_' + str(x + 1)
+ if host_tag[hostlabel]['ip']:
+ if host_tag[hostlabel]['ip'] in ips:
+ info = "%s's ip %s is defined by test case yaml file" % \
+ (hostlabel, host_tag[hostlabel]['ip'])
+ LOG.info(info)
+ else:
+ err = "%s is not in %s" % (host_tag[hostlabel]['ip'], ips)
+ raise RuntimeError(err)
+ else:
+ host_tag[hostlabel]['ip'] = random.choice(ips)
+ info = "assign ip %s to %s" % (host_tag[hostlabel]['ip'], hostlabel)
+ ips.remove(host_tag[hostlabel]['ip'])
+
+ def get_host_machine_info(self, host_tag):
num = len(host_tag)
offset = len(self.roles_ip_list)
+ self.check_machine_ips(host_tag)
for x in range(num):
hostlabel = 'machine_' + str(x + 1)
self.roles_ip_list.insert(
diff --git a/func/fetch_compute_ips.sh b/func/fetch_compute_ips.sh
new file mode 100755
index 00000000..c1cc4c6e
--- /dev/null
+++ b/func/fetch_compute_ips.sh
@@ -0,0 +1,117 @@
+#!/bin/bash
+##############################################################################
+#Copyright (c) 2016 Ericsson AB, ZTE and others.
+#jose.lausuch@ericsson.com
+#wu.zhihui1@zte.com.cn
+#All rights reserved. This program and the accompanying materials
+#are made available under the terms of the Apache License, Version 2.0
+#which accompanies this distribution, and is available at
+#http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+usage(){
+ echo "usage: $0 [-v] -i <installer_type> -a <installer_ip>" >&2
+ echo "[-v] Virtualized deployment" >&2
+}
+
+info() {
+ logger -s -t "fetch_compute_info.info" "$*"
+}
+
+
+error() {
+ logger -s -t "fetch_compute_info.error" "$*"
+ exit 1
+}
+
+verify_connectivity(){
+ local ip=$1
+ info "Verifying connectivity to $ip..."
+ for i in $(seq 0 10); do
+ if ping -c 1 -W 1 $ip > /dev/null; then
+ info "$ip is reachable!"
+ return 0
+ fi
+ sleep 1
+ done
+ error "Can not talk to $ip."
+}
+
+:${DEPLOY_TYPE:=''}
+
+#Getoptions
+whilegetopts ":d:i:a:h:v" optchar; do
+ case "${optchar}" in
+ i) installer_type=${OPTARG} ;;
+ a) installer_ip=${OPTARG} ;;
+ v) DEPLOY_TYPE="virt" ;;
+ *) echo "Non-option argument: '-${OPTARG}'" >&2
+ usage
+ exit 2
+ ;;
+ esac
+done
+
+#set vars from env if not provided by user as options
+installer_type=${installer_type:-$INSTALLER_TYPE}
+installer_ip=${installer_ip:-$INSTALLER_IP}
+
+if[ -z $installer_type ] || [ -z $installer_ip ]; then
+ usage
+ exit 2
+fi
+
+ssh_options="-oUserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+
+#Start fetching compute ip
+if[ "$installer_type" == "fuel" ]; then
+ verify_connectivity $installer_ip
+
+ env=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \
+ 'fuel env'|grep operational|head -1|awk '{print $1}') &> /dev/null
+ if [ -z $env ]; then
+ error "No operational environment detected in Fuel"
+ fi
+ env_id="${FUEL_ENV:-$env}"
+
+ # Check if compute is alive (online='True')
+ IPS=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \
+ "fuel node --env ${env_id} | grep compute | grep 'True\| 1' | awk -F\| '{print \$5}' " | \
+ sed 's/ //g') &> /dev/null
+
+
+elif[ "$installer_type" == "apex" ]; then
+ echo "not implement now"
+ exit 1
+
+elif[ "$installer_type" == "compass" ]; then
+ # need test
+ verify_connectivity $installer_ip
+ IPS=$(sshpass -p'root' ssh 2>/dev/null $ssh_options root@${installer_ip} \
+ 'mysql -ucompass -pcompass -Dcompass -e"select * from cluster;"' \
+ | awk -F"," '{for(i=1;i<NF;i++)if($i~/\"host[4-5]\"/) {print $(i+1);}}' \
+ | grep -oP "\d+.\d+.\d+.\d+")
+
+elif[ "$installer_type" == "joid" ]; then
+ echo "not implement now"
+ exit 1
+
+elif[ "$installer_type" == "foreman" ]; then
+ echo "not implement now"
+ exit 1
+
+else
+ error "Installer $installer is not supported by this script"
+fi
+
+if[ -z $IPS ]; then
+ error "The compute node $IPS are not up. Please check that the POD is correctly deployed."
+else
+ echo "-------- all compute node ips: --------"
+ touch $HOME/ips.log
+ echo "$IPS" > $HOME/ips.log
+ echo $IPS
+fi
+
+exit0
diff --git a/tests/cli_test.py b/tests/cli_test.py
index f12e8fed..bd31d987 100644
--- a/tests/cli_test.py
+++ b/tests/cli_test.py
@@ -16,5 +16,5 @@ class TestClass:
def test_cli_error(self, capfd, test_input, expected):
with pytest.raises(SystemExit):
cli(test_input)
- resout, reserr = capfd.readouterr()
- assert expected in resout
+ resout, reserr = capfd.readouterr()
+ assert expected in resout
diff --git a/tests/driver_test.py b/tests/driver_test.py
index 39adc939..a5b13588 100644
--- a/tests/driver_test.py
+++ b/tests/driver_test.py
@@ -16,20 +16,20 @@ class TestClass:
'https_proxy': 'http://10.20.0.1:8118',
'no_proxy': 'localhost,127.0.0.1,10.20.*,192.168.*'},
'fuel'],
- {'Dest_dir': 'results',
- 'ip1': '',
- 'ip2': '',
- 'installer': 'fuel',
- 'workingdir': '/home',
- 'fname': 'iperf_bm.yaml',
- 'username': 'root',
- 'http_proxy': 'http://10.20.0.1:8118',
- 'https_proxy': 'http://10.20.0.1:8118',
- 'no_proxy': 'localhost,127.0.0.1,10.20.*,192.168.*',
- 'duration': 20,
- 'protocol': 'tcp',
- 'bandwidthGbps': 0,
- "role": "host"}),
+ [{'Dest_dir': 'results',
+ 'ip1': '',
+ 'ip2': '',
+ 'installer': 'fuel',
+ 'workingdir': '/home',
+ 'fname': 'iperf_bm.yaml',
+ 'username': 'root',
+ 'http_proxy': 'http://10.20.0.1:8118',
+ 'https_proxy': 'http://10.20.0.1:8118',
+ 'no_proxy': 'localhost,127.0.0.1,10.20.*,192.168.*',
+ 'duration': 20,
+ 'protocol': 'tcp',
+ 'bandwidthGbps': 0,
+ "role": "host"}]),
(["iperf",
[('1-server', ['10.20.0.13']), ('2-host', ['10.20.0.15'])],
"iperf_vm.yaml",
@@ -37,18 +37,29 @@ class TestClass:
[("10.20.0.13", [None]), ("10.20.0.15", [None])],
{},
'joid'],
- {'Dest_dir': 'results',
- 'ip1': '10.20.0.13',
- 'ip2': '',
- 'installer': 'joid',
- "privateip1": "NONE",
- 'workingdir': '/home',
- 'fname': 'iperf_vm.yaml',
- 'username': 'ubuntu',
- 'duration': 20,
- 'protocol': 'tcp',
- 'bandwidthGbps': 0,
- "role": "2-host"})
+ [{'Dest_dir': 'results',
+ 'ip1': '10.20.0.13',
+ 'ip2': '',
+ 'installer': 'joid',
+ "privateip1": "NONE",
+ 'workingdir': '/home',
+ 'fname': 'iperf_vm.yaml',
+ 'username': 'ubuntu',
+ 'duration': 20,
+ 'protocol': 'tcp',
+ 'bandwidthGbps': 0,
+ "role": "1-server"},
+ {'Dest_dir': 'results',
+ 'ip1': '',
+ 'ip2': '',
+ 'installer': 'joid',
+ 'workingdir': '/home',
+ 'fname': 'iperf_vm.yaml',
+ 'username': 'ubuntu',
+ 'duration': 20,
+ 'protocol': 'tcp',
+ 'bandwidthGbps': 0,
+ "role": "2-host"}])
])
@mock.patch('func.driver.os.system')
def test_driver_success(self, mock_system, test_input, expected):
@@ -57,9 +68,11 @@ class TestClass:
k.start()
dri = Driver()
dri.drive_bench(test_input[0], test_input[1], test_input[2], test_input[3], test_input[4], test_input[5])
- call = mock_system.call_args
+ call_list = mock_system.call_args_list
k.stop()
- call_args, call_kwargs = call
- real_call = call_args[0].split('extra-vars \'')[1]
- real_call = real_call[0: len(real_call) - 1]
- assert json.loads(real_call) == json.loads(json.dumps(expected))
+ print call_list
+ for call in call_list:
+ call_args, call_kwargs = call
+ real_call = call_args[0].split('extra-vars \'')[1]
+ real_call = real_call[0: len(real_call) - 1]
+ assert json.loads(real_call) == json.loads(json.dumps(expected[call_list.index(call)]))
diff --git a/tests/env_setup_test.py b/tests/env_setup_test.py
index 9112ff94..cc3c6b60 100644
--- a/tests/env_setup_test.py
+++ b/tests/env_setup_test.py
@@ -1,6 +1,16 @@
+##############################################################################
+# Copyright (c) 2016 ZTE and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
import pytest
import filecmp
from func.env_setup import Env_setup
+import mock
class TestClass:
@@ -31,6 +41,8 @@ class TestClass:
print (test_input)
print (expected)
test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
benchmark, vm_para, details, proxy = \
test_class.parse(test_input)
assert benchmark == expected[0]
@@ -40,12 +52,16 @@ class TestClass:
def test_parse_vm_error(self):
test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
with pytest.raises(KeyError) as excinfo:
test_class.parse("tests/test_case/vm_error.yaml")
assert "benchmark" in str(excinfo.value)
def test_update_ansible(self):
test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
test_class.parse("tests/test_case/bm_without_proxy.yaml")
test_class.update_ansible()
result = filecmp.cmp('tests/output/hosts', 'data/hosts')
@@ -53,7 +69,40 @@ class TestClass:
def test_ping(self, capfd):
test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["127.0.0.1", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
test_class.parse("tests/test_case/bm_ping.yaml")
test_class.call_ping_test()
resout, reserr = capfd.readouterr()
assert '127.0.0.1 is UP' in resout
+
+ def test_check_machine_ips_without_ip(self):
+ test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
+ inputs = {"machine_1": {"ip": "", "pw": "", "role": "host"},
+ "machine_2": {"ip": "", "pw": "", "role": "host"}}
+ test_class.check_machine_ips(inputs)
+ assert inputs["machine_1"]['ip'] in ["10.20.0.28", "10.20.0.29"]
+ assert inputs["machine_2"]['ip'] in ["10.20.0.28", "10.20.0.29"]
+ assert inputs["machine_1"]['ip'] != inputs["machine_2"]['ip']
+
+ def test_check_machine_ips_with_ip(self):
+ test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
+ inputs = {"machine_1": {"ip": "10.20.0.28", "pw": "", "role": "host"},
+ "machine_2": {"ip": "10.20.0.29", "pw": "", "role": "host"}}
+ test_class.check_machine_ips(inputs)
+ assert inputs["machine_1"]['ip'] in ["10.20.0.28", "10.20.0.29"]
+ assert inputs["machine_2"]['ip'] in ["10.20.0.28", "10.20.0.29"]
+ assert inputs["machine_1"]['ip'] != inputs["machine_2"]['ip']
+
+ def test_check_machine_ips_with_invalid_ip(self):
+ test_class = Env_setup()
+ mock_ips = mock.Mock(return_value=["10.20.0.28", "10.20.0.29"])
+ test_class.fetch_compute_ips = mock_ips
+ inputs = {"machine_1": {"ip": "10.20.0.3", "pw": "", "role": "host"},
+ "machine_2": {"ip": "10.20.0.4", "pw": "", "role": "host"}}
+ with pytest.raises(RuntimeError):
+ test_class.check_machine_ips(inputs)