From 45fabbab6a4107df8a32f09045cf955afeb2e4ac Mon Sep 17 00:00:00 2001 From: Dan Radez Date: Mon, 20 Jun 2016 05:56:34 -0400 Subject: Syntax updates and new tests - syntax updates to match pep8 standards - tests to cover apex_python_utils.py Change-Id: Ifac06fdbb97266f1b574b20610979b6965d6dd55 Signed-off-by: Dan Radez --- .gitignore | 2 + build/Makefile | 15 ++- build/opnfv-apex-common.spec | 2 +- ci/deploy.sh | 8 +- ci/test.sh | 11 +- lib/common-functions.sh | 2 +- lib/python/apex-python-utils.py | 206 --------------------------------- lib/python/apex/deploy_env.py | 53 +++++---- lib/python/apex/ip_utils.py | 41 ++----- lib/python/apex/network_environment.py | 61 +++++----- lib/python/apex/network_settings.py | 9 +- lib/python/apex_python_utils.py | 206 +++++++++++++++++++++++++++++++++ tests/test_apex_deploy_env.py | 17 +-- tests/test_apex_ip_utils.py | 9 +- tests/test_apex_network_environment.py | 3 +- tests/test_apex_python_utils_py.py | 80 +++++++++++++ 16 files changed, 399 insertions(+), 326 deletions(-) delete mode 100755 lib/python/apex-python-utils.py create mode 100755 lib/python/apex_python_utils.py create mode 100644 tests/test_apex_python_utils_py.py diff --git a/.gitignore b/.gitignore index 33a0451b..f6ef061f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ *~ +*.pyc .*.sw? +.coverage /docs_build/ /docs_output/ /releng/ diff --git a/build/Makefile b/build/Makefile index 0763d361..84305618 100644 --- a/build/Makefile +++ b/build/Makefile @@ -66,12 +66,23 @@ $(RPMCOM): .PHONY: python-tests python-tests: + # clean previous coverage data + rm -rf ../tests/.coverage + rm -rf ../tests/htmlcov # run nose tests - cd ../tests && PYTHONPATH=../lib/python/ nosetests-3.4 . --with-coverage --cover-package apex + cd ../tests && PYTHONPATH=../lib/python/ nosetests-3.4 . --with-coverage --cover-package apex --cover-package apex_python_utils --cover-html --cover-min-percentage 85 + # generate reports - cd ../tests && coverage3 html --include '*lib/python/*' cd ../tests && coverage3 report --include '*lib/python/*' -m +####################### +# PYTHON PEP8 CHECK # +####################### + +.PHONY: python-pep8-check +python-pep8-check: + pep8 ../lib/python + pep8 ../tests ############### # UNDERCLOUD # diff --git a/build/opnfv-apex-common.spec b/build/opnfv-apex-common.spec index d5532411..5abd4493 100644 --- a/build/opnfv-apex-common.spec +++ b/build/opnfv-apex-common.spec @@ -53,7 +53,7 @@ install config/network/network_settings_v6.yaml %{buildroot}%{_sysconfdir}/opnfv mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/python/apex install lib/common-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/ install lib/utility-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/ -install lib/python/apex-python-utils.py %{buildroot}%{_var}/opt/opnfv/lib/python/ +install lib/python/apex_python_utils.py %{buildroot}%{_var}/opt/opnfv/lib/python/ mkdir -p %{buildroot}%{python3_sitelib}/apex/ install lib/python/apex/__init__.py %{buildroot}%{python3_sitelib}/apex/ install lib/python/apex/deploy_env.py %{buildroot}%{python3_sitelib}/apex/ diff --git a/ci/deploy.sh b/ci/deploy.sh index 9f678153..e5d47bec 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -124,7 +124,7 @@ parse_setting_value() { ##parses network settings yaml into globals parse_network_settings() { local output - if output=$(python3.4 -B $LIB/python/apex-python-utils.py parse-net-settings -s $NETSETS -i $net_isolation_enabled -e $CONFIG/network-environment.yaml); then + if output=$(python3.4 -B $LIB/python/apex_python_utils.py parse-net-settings -s $NETSETS -i $net_isolation_enabled -e $CONFIG/network-environment.yaml); then echo -e "${blue}${output}${reset}" eval "$output" else @@ -136,7 +136,7 @@ parse_network_settings() { ##parses deploy settings yaml into globals parse_deploy_settings() { local output - if output=$(python3.4 -B $LIB/python/apex-python-utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then + if output=$(python3.4 -B $LIB/python/apex_python_utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then echo -e "${blue}${output}${reset}" eval "$output" else @@ -664,12 +664,12 @@ function configure_undercloud { ovs_dpdk_bridge='' fi - if ! controller_nic_template=$(python3.4 -B $LIB/python/apex-python-utils.py nic-template -r controller -s $NETSETS -i $net_isolation_enabled -t $CONFIG/nics-template.yaml.jinja2 -n "$enabled_network_list" -e "br-ex" -af $ip_addr_family); then + if ! controller_nic_template=$(python3.4 -B $LIB/python/apex_python_utils.py nic-template -r controller -s $NETSETS -i $net_isolation_enabled -t $CONFIG/nics-template.yaml.jinja2 -n "$enabled_network_list" -e "br-ex" -af $ip_addr_family); then echo -e "${red}ERROR: Failed to generate controller NIC heat template ${reset}" exit 1 fi - if ! compute_nic_template=$(python3.4 -B $LIB/python/apex-python-utils.py nic-template -r compute -s $NETSETS -i $net_isolation_enabled -t $CONFIG/nics-template.yaml.jinja2 -n "$enabled_network_list" -e $ext_net_type -af $ip_addr_family -d "$ovs_dpdk_bridge"); then + if ! compute_nic_template=$(python3.4 -B $LIB/python/apex_python_utils.py nic-template -r compute -s $NETSETS -i $net_isolation_enabled -t $CONFIG/nics-template.yaml.jinja2 -n "$enabled_network_list" -e $ext_net_type -af $ip_addr_family -d "$ovs_dpdk_bridge"); then echo -e "${red}ERROR: Failed to generate compute NIC heat template ${reset}" exit 1 fi diff --git a/ci/test.sh b/ci/test.sh index 22f9120b..4f8a0421 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -11,7 +11,7 @@ set -e # Make sure python dependencies are installed -for pkg in epel-release python34-devel python34-nose; do +for pkg in epel-release python34-devel python34-nose python-pep8; do if ! rpm -q ${pkg} > /dev/null; then if ! sudo yum install -y ${pkg}; then echo "Failed to install ${pkg} package..." @@ -25,12 +25,5 @@ if ! python3 -c "import coverage" &> /dev/null; then sudo easy_install-3.4 cover pushd ../build/ > /dev/null make python-tests -popd > /dev/null -pushd ../tests/ > /dev/null -percent=$(coverage3 report --include '*lib/python/*' -m | grep TOTAL | tr -s ' ' | awk '{ print $4 }' | cut -d % -f 1) -if [[ percent -lt 80 ]]; then - echo "Python Coverage: $percent" - echo "Does not meet 80% requirement" - exit 1 -fi +make python-pep8-check popd > /dev/null diff --git a/lib/common-functions.sh b/lib/common-functions.sh index 6b259ac1..079b0886 100644 --- a/lib/common-functions.sh +++ b/lib/common-functions.sh @@ -18,7 +18,7 @@ function find_ip { return 1 fi - python3.4 -B $LIB/python/apex-python-utils.py find-ip -i $1 + python3.4 -B $LIB/python/apex_python_utils.py find-ip -i $1 } ##attach interface to OVS and set the network config correctly diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py deleted file mode 100755 index 01e6b03b..00000000 --- a/lib/python/apex-python-utils.py +++ /dev/null @@ -1,206 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Feng Pan (fpan@redhat.com), Dan Radez (dradez@redhat.com) -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import argparse -import sys -import logging -import os -import yaml - -from copy import copy - -from jinja2 import Environment -from jinja2 import FileSystemLoader - -from apex import NetworkSettings -from apex import NetworkEnvironment -from apex import DeploySettings -from apex import ip_utils -from apex.common.constants import OPNFV_NETWORK_TYPES -from apex.common.constants import ADMIN_NETWORK - - -def parse_net_settings(args): - """ - Parse OPNFV Apex network_settings.yaml config file - and dump bash syntax to set environment variables - - Args: - - file: string - file to network_settings.yaml file - - network_isolation: bool - enable or disable network_isolation - """ - settings = NetworkSettings(args.net_settings_file, - args.network_isolation) - net_env = NetworkEnvironment(settings, args.net_env_file) - dump_yaml(net_env.get_netenv_settings(), '/tmp/network-environment.yaml') - settings.dump_bash() - - -def dump_yaml(data, file): - """ - Dumps data to a file as yaml - :param data: yaml to be written to file - :param file: filename to write to - :return: - """ - with open(file, "w") as fh: - yaml.dump(data, fh, default_flow_style=False) - - -def parse_deploy_settings(args): - settings = DeploySettings(args.file) - settings.dump_bash() - - -def find_ip(args): - """ - Get and print the IP from a specific interface - - Args: - - interface: string - network interface name - - address_family: int - 4 or 6, respective to ipv4 or ipv6 - """ - interface = ip_utils.get_interface(args.interface, - args.address_family) - if interface: - print(interface.ip) - - -def build_nic_template(args): - """ - Build and print a Triple-O nic template from jinja template - - Args: - - template: string - path to jinja template to load - - enabled_networks: comma delimited list - list of networks defined in net_env.py - - ext_net_type: string - interface or br-ex, defines the external network configuration - - address_family: string - 4 or 6, respective to ipv4 or ipv6 - - ovs_dpdk_bridge: string - bridge name to use as ovs_dpdk - """ - template_dir, template = args.template.rsplit('/', 1) - - settings = NetworkSettings(args.net_settings_file, - args.network_isolation).settings_obj - env = Environment(loader=FileSystemLoader(template_dir)) - template = env.get_template(template) - - # gather vlan values into a dict - net_list = copy(args.enabled_networks).split(' ') - net_list.remove(ADMIN_NETWORK) - vlans_vals = map(lambda x: settings[x]['vlan'], net_list) - vlans = dict(zip(net_list, vlans_vals)) - - print(template.render( - enabled_networks=args.enabled_networks, - role=args.role, - vlans=vlans, - external_net_type=args.ext_net_type, - external_net_af=args.address_family, - ovs_dpdk_bridge=args.ovs_dpdk_bridge)) - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--debug', action='store_true', default=False, - help="Turn on debug messages") - parser.add_argument('-l', '--log-file', default='/var/log/apex/apex.log', - dest='log_file', help="Log file to log to") - subparsers = parser.add_subparsers() - - net_settings = subparsers.add_parser('parse-net-settings', - help='Parse network settings file') - net_settings.add_argument('-s', '--net-settings-file', - default='network-settings.yaml', - dest='net_settings_file', - help='path to network settings file') - net_settings.add_argument('-i', '--network-isolation', type=bool, - default=True, dest='network_isolation', - help='network isolation') - net_settings.add_argument('-e', '--net-env-file', - default="network-environment.yaml", - dest='net_env_file', - help='path to network environment file') - net_settings.set_defaults(func=parse_net_settings) - - get_int_ip = subparsers.add_parser('find-ip', - help='Find interface ip') - get_int_ip.add_argument('-i', '--interface', required=True, - help='Interface name') - get_int_ip.add_argument('-af', '--address-family', default=4, type=int, - choices=[4, 6], dest='address_family', - help='IP Address family') - get_int_ip.set_defaults(func=find_ip) - - nic_template = subparsers.add_parser('nic-template', - help='Build NIC templates') - nic_template.add_argument('-r', '--role', required=True, - choices=['controller', 'compute'], - help='Role template generated for') - nic_template.add_argument('-t', '--template', required=True, - dest='template', - help='Template file to process') - nic_template.add_argument('-s', '--net-settings-file', - default='network-settings.yaml', - dest='net_settings_file', - help='path to network settings file') - nic_template.add_argument('-i', '--network-isolation', type=bool, - default=True, dest='network_isolation', - help='network isolation') - nic_template.add_argument('-n', '--enabled-networks', required=True, - dest='enabled_networks', - help='enabled network list') - nic_template.add_argument('-e', '--ext-net-type', default='interface', - dest='ext_net_type', - choices=['interface', 'br-ex'], - help='External network type') - nic_template.add_argument('-af', '--address-family', type=int, default=4, - dest='address_family', help='IP address family') - nic_template.add_argument('-d', '--ovs-dpdk-bridge', - default=None, dest='ovs_dpdk_bridge', - help='OVS DPDK Bridge Name') - nic_template.set_defaults(func=build_nic_template) - - deploy_settings = subparsers.add_parser('parse-deploy-settings', - help='Parse deploy settings file') - deploy_settings.add_argument('-f', '--file', default='deploy_settings.yaml', - help='path to deploy settings file') - deploy_settings.set_defaults(func=parse_deploy_settings) - - args = parser.parse_args(sys.argv[1:]) - if args.debug: - logging.basicConfig(level=logging.DEBUG) - else: - apex_log_filename = args.log_file - os.makedirs(os.path.dirname(apex_log_filename), exist_ok=True) - logging.basicConfig(filename=apex_log_filename, - format='%(asctime)s %(levelname)s: %(message)s', - datefmt='%m/%d/%Y %I:%M:%S %p', - level=logging.DEBUG) - return parser, args - - -def main(parser, args): - if hasattr(args, 'func'): - args.func(args) - else: - parser.print_help() - exit(1) - -if __name__ == "__main__": - parser, args = parse_args() - main(parser, args) diff --git a/lib/python/apex/deploy_env.py b/lib/python/apex/deploy_env.py index 4c71b54f..1fe137e4 100644 --- a/lib/python/apex/deploy_env.py +++ b/lib/python/apex/deploy_env.py @@ -24,8 +24,9 @@ REQ_DEPLOY_SETTINGS = ['sdn_controller', OPT_DEPLOY_SETTINGS = ['performance'] VALID_ROLES = ['Controller', 'Compute', 'ObjectStorage'] -VALID_PERF_OPTS = ['kernel','nova'] -VALID_DATAPLANES = ['ovs','ovs_dpdk','fdio'] +VALID_PERF_OPTS = ['kernel', 'nova'] +VALID_DATAPLANES = ['ovs', 'ovs_dpdk', 'fdio'] + class DeploySettings: """ @@ -65,9 +66,9 @@ class DeploySettings: if setting == 'dataplane': if value not in VALID_DATAPLANES: planes = ' '.join(VALID_DATAPLANES) - raise DeploySettingsException("Invalid dataplane {} " - "specified. Valid dataplanes:" - " {}".format(value,planes)) + raise DeploySettingsException( + "Invalid dataplane {} specified. Valid dataplanes:" + " {}".format(value, planes)) for req_set in REQ_DEPLOY_SETTINGS: if req_set not in deploy_options: @@ -80,37 +81,43 @@ class DeploySettings: if not isinstance(deploy_options['performance'], dict): raise DeploySettingsException("Performance deploy_option" "must be a dictionary.") - for role,role_perf_sets in deploy_options['performance'].items(): + for role, role_perf_sets in deploy_options['performance'].items(): if role not in VALID_ROLES: raise DeploySettingsException("Performance role {}" "is not valid, choose" "from {}".format( - role," ".join(VALID_ROLES) + role, + " ".join(VALID_ROLES) )) for key in role_perf_sets: if key not in VALID_PERF_OPTS: - raise DeploySettingsException("Performance option {}" + raise DeploySettingsException("Performance option {} " "is not valid, choose" "from {}".format( - key," ".join( - VALID_PERF_OPTS))) - + key, + " ".join( + VALID_PERF_OPTS) + )) def _dump_performance(self): """ Creates performance settings string for bash consumption. - Output will be in the form of a list that can be iterated over in bash, - with each string being the direct input to the performance setting script - in the form to facilitate modification of the - correct image. + Output will be in the form of a list that can be iterated over in + bash, with each string being the direct input to the performance + setting script in the form to + facilitate modification of the correct image. """ bash_str = 'performance_options=(\n' - for role,settings in self.deploy_settings['deploy_options']['performance'].items(): - for category,options in settings.items(): - for key,value in options.items(): - bash_str += "\"{} {} {} {}\"\n".format(role, category, key, value) + deploy_options = self.deploy_settings['deploy_options'] + for role, settings in deploy_options['performance'].items(): + for category, options in settings.items(): + for key, value in options.items(): + bash_str += "\"{} {} {} {}\"\n".format(role, + category, + key, + value) bash_str += ')\n' bash_str += '\n' bash_str += 'performance_roles=(\n' @@ -126,11 +133,13 @@ class DeploySettings: Creates deploy settings array in bash syntax. """ bash_str = '' - for key,value in self.deploy_settings['deploy_options'].items(): + for key, value in self.deploy_settings['deploy_options'].items(): if not isinstance(value, bool): - bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key, value) + bash_str += "deploy_options_array[{}]=\"{}\"\n".format(key, + value) else: - bash_str += "deploy_options_array[{}]={}\n".format(key, value) + bash_str += "deploy_options_array[{}]={}\n".format(key, + value) return bash_str def dump_bash(self, path=None): diff --git a/lib/python/apex/ip_utils.py b/lib/python/apex/ip_utils.py index f51f227a..b039e26b 100644 --- a/lib/python/apex/ip_utils.py +++ b/lib/python/apex/ip_utils.py @@ -34,11 +34,11 @@ def get_ip_range(start_offset=None, count=None, end_offset=None, space, and end_offset will be calculated from the interface IP. 2 of start_offset, end_offset and count options must be provided: - - If start_offset and end_offset are provided, a range from start_offset - to end_offset will be returned. - - If count is provided, a range from either start_offset to (start_offset - +count) or (end_offset-count) to end_offset will be returned. The - IP range returned will be of size . + - If start_offset and end_offset are provided, a range from + start_offset to end_offset will be returned. + - If count is provided, a range from either start_offset to + (start_offset+count) or (end_offset-count) to end_offset will be + returned. The IP range returned will be of size . Both start_offset and end_offset must be greater than 0. Returns IP range in the format of "first_addr,second_addr" or exception @@ -47,7 +47,7 @@ def get_ip_range(start_offset=None, count=None, end_offset=None, if cidr: if count and start_offset and not end_offset: start_index = start_offset - end_index = start_offset + count -1 + end_index = start_offset + count - 1 elif count and end_offset and not start_offset: end_index = -1 - end_offset start_index = -1 - end_index - count + 1 @@ -81,7 +81,7 @@ def get_ip_range(start_offset=None, count=None, end_offset=None, else: if count and start_offset and not end_offset: start_ip = network[start_offset] - end_ip = start_ip + count -1 + end_ip = start_ip + count - 1 elif count and end_offset and not start_offset: end_ip = interface.ip - end_offset start_ip = end_ip - count + 1 @@ -135,27 +135,6 @@ def get_ip(offset, cidr=None, interface=None): return str(ip) -def generate_ip_range(args): - """ - Generate IP range in string format for given CIDR. - This function works for both IPv4 and IPv6. - - args is expected to contain the following members: - CIDR: any valid CIDR representation. - start_position: starting index, default to first address in subnet (1) - end_position: ending index, default to last address in subnet (-1) - - Returns IP range in string format. A single IP is returned if start and - end IPs are identical. - """ - cidr = ipaddress.ip_network(args.CIDR) - (start_index, end_index) = (args.start_position, args.end_position) - if cidr[start_index] == cidr[end_index]: - return str(cidr[start_index]) - else: - return ','.join(sorted([str(cidr[start_index]), str(cidr[end_index])])) - - def get_interface(nic, address_family=4): """ Returns interface object for a given NIC name in the system @@ -183,7 +162,7 @@ def get_interface(nic, address_family=4): return ipaddress.ip_interface(match.group()) else: logging.info("interface ip not found! ip address output:\n{}" - .format(output)) + .format(output)) return None @@ -228,8 +207,8 @@ def _validate_ip_range(start_ip, end_ip, cidr): """ ip_range = "{},{}".format(start_ip, end_ip) if end_ip <= start_ip: - logging.warning("IP range {} is invalid: end_ip should be greater than " - "starting ip".format(ip_range)) + logging.warning("IP range {} is invalid: end_ip should be greater " + "than starting ip".format(ip_range)) return False if start_ip not in ipaddress.ip_network(cidr): logging.warning('start_ip {} is not in network {}' diff --git a/lib/python/apex/network_environment.py b/lib/python/apex/network_environment.py index fd6f5286..fec6299d 100644 --- a/lib/python/apex/network_environment.py +++ b/lib/python/apex/network_environment.py @@ -9,7 +9,11 @@ import yaml import re -from .common import constants +from .common.constants import ADMIN_NETWORK +from .common.constants import PRIVATE_NETWORK +from .common.constants import STORAGE_NETWORK +from .common.constants import PUBLIC_NETWORK +from .common.constants import API_NETWORK PORTS = '/ports' # Resources defined by : @@ -70,27 +74,27 @@ class NetworkEnvironment: if not tht_dir: raise NetworkEnvException('Unable to parse THT Directory') - admin_cidr = net_settings[constants.ADMIN_NETWORK]['cidr'] + admin_cidr = net_settings[ADMIN_NETWORK]['cidr'] admin_prefix = str(admin_cidr.prefixlen) self.netenv_obj[param_def]['ControlPlaneSubnetCidr'] = admin_prefix self.netenv_obj[param_def]['ControlPlaneDefaultRoute'] = \ - net_settings[constants.ADMIN_NETWORK]['provisioner_ip'] - public_cidr = net_settings[constants.PUBLIC_NETWORK]['cidr'] + net_settings[ADMIN_NETWORK]['provisioner_ip'] + public_cidr = net_settings[PUBLIC_NETWORK]['cidr'] self.netenv_obj[param_def]['ExternalNetCidr'] = str(public_cidr) - if net_settings[constants.PUBLIC_NETWORK]['vlan'] != 'native': + if net_settings[PUBLIC_NETWORK]['vlan'] != 'native': self.netenv_obj[param_def]['ExternalNetworkVlanID'] = \ - net_settings[constants.PUBLIC_NETWORK]['vlan'] - public_range = net_settings[constants.PUBLIC_NETWORK][ - 'usable_ip_range'].split(',') + net_settings[PUBLIC_NETWORK]['vlan'] + public_range = \ + net_settings[PUBLIC_NETWORK]['usable_ip_range'].split(',') self.netenv_obj[param_def]['ExternalAllocationPools'] = \ [{'start': public_range[0], 'end': public_range[1] }] self.netenv_obj[param_def]['ExternalInterfaceDefaultRoute'] = \ - net_settings[constants.PUBLIC_NETWORK]['gateway'] + net_settings[PUBLIC_NETWORK]['gateway'] self.netenv_obj[param_def]['EC2MetadataIp'] = \ - net_settings[constants.ADMIN_NETWORK]['provisioner_ip'] + net_settings[ADMIN_NETWORK]['provisioner_ip'] self.netenv_obj[param_def]['DnsServers'] = net_settings['dns_servers'] if public_cidr.version == 6: @@ -103,24 +107,23 @@ class NetworkEnvironment: prefix = '' self.netenv_obj[reg][key] = tht_dir + prefix + postfix - - if constants.PRIVATE_NETWORK in enabled_networks: - priv_range = net_settings[constants.PRIVATE_NETWORK][ + if PRIVATE_NETWORK in enabled_networks: + priv_range = net_settings[PRIVATE_NETWORK][ 'usable_ip_range'].split(',') self.netenv_obj[param_def]['TenantAllocationPools'] = \ [{'start': priv_range[0], 'end': priv_range[1] }] - priv_cidr = net_settings[constants.PRIVATE_NETWORK]['cidr'] + priv_cidr = net_settings[PRIVATE_NETWORK]['cidr'] self.netenv_obj[param_def]['TenantNetCidr'] = str(priv_cidr) if priv_cidr.version == 6: postfix = '/tenant_v6.yaml' else: postfix = '/tenant.yaml' - if net_settings[constants.PRIVATE_NETWORK]['vlan'] != 'native': + if net_settings[PRIVATE_NETWORK]['vlan'] != 'native': self.netenv_obj[param_def]['TenantNetworkVlanID'] = \ - net_settings[constants.PRIVATE_NETWORK]['vlan'] + net_settings[PRIVATE_NETWORK]['vlan'] else: postfix = '/noop.yaml' @@ -129,8 +132,8 @@ class NetworkEnvironment: prefix = '' self.netenv_obj[reg][key] = tht_dir + prefix + postfix - if constants.STORAGE_NETWORK in enabled_networks: - storage_range = net_settings[constants.STORAGE_NETWORK][ + if STORAGE_NETWORK in enabled_networks: + storage_range = net_settings[STORAGE_NETWORK][ 'usable_ip_range'].split(',') self.netenv_obj[param_def]['StorageAllocationPools'] = \ [{'start': @@ -138,15 +141,15 @@ class NetworkEnvironment: 'end': storage_range[1] }] - storage_cidr = net_settings[constants.STORAGE_NETWORK]['cidr'] + storage_cidr = net_settings[STORAGE_NETWORK]['cidr'] self.netenv_obj[param_def]['StorageNetCidr'] = str(storage_cidr) if storage_cidr.version == 6: postfix = '/storage_v6.yaml' else: postfix = '/storage.yaml' - if net_settings[constants.STORAGE_NETWORK]['vlan'] != 'native': + if net_settings[STORAGE_NETWORK]['vlan'] != 'native': self.netenv_obj[param_def]['StorageNetworkVlanID'] = \ - net_settings[constants.STORAGE_NETWORK]['vlan'] + net_settings[STORAGE_NETWORK]['vlan'] else: postfix = '/noop.yaml' @@ -155,24 +158,22 @@ class NetworkEnvironment: prefix = '' self.netenv_obj[reg][key] = tht_dir + prefix + postfix - if constants.API_NETWORK in enabled_networks: - api_range = net_settings[constants.API_NETWORK][ + if API_NETWORK in enabled_networks: + api_range = net_settings[API_NETWORK][ 'usable_ip_range'].split(',') self.netenv_obj[param_def]['InternalApiAllocationPools'] = \ - [{'start': - api_range[0], - 'end': - api_range[1] + [{'start': api_range[0], + 'end': api_range[1] }] - api_cidr = net_settings[constants.API_NETWORK]['cidr'] + api_cidr = net_settings[API_NETWORK]['cidr'] self.netenv_obj[param_def]['InternalApiNetCidr'] = str(api_cidr) if api_cidr.version == 6: postfix = '/internal_api_v6.yaml' else: postfix = '/internal_api.yaml' - if net_settings[constants.API_NETWORK]['vlan'] != 'native': + if net_settings[API_NETWORK]['vlan'] != 'native': self.netenv_obj[param_def]['InternalApiNetworkVlanID'] = \ - net_settings[constants.API_NETWORK]['vlan'] + net_settings[API_NETWORK]['vlan'] else: postfix = '/noop.yaml' diff --git a/lib/python/apex/network_settings.py b/lib/python/apex/network_settings.py index 475082df..50dd15c3 100644 --- a/lib/python/apex/network_settings.py +++ b/lib/python/apex/network_settings.py @@ -19,8 +19,8 @@ class NetworkSettings: This class parses APEX network settings yaml file into an object. It generates or detects all missing fields for deployment. - The resulting object will be used later to generate network environment file - as well as configuring post deployment networks. + The resulting object will be used later to generate network environment + file as well as configuring post deployment networks. Currently the parsed object is dumped into a bash global definition file for deploy.sh consumption. This object will later be used directly as @@ -41,7 +41,7 @@ class NetworkSettings: """ if constants.ADMIN_NETWORK not in self.settings_obj or \ not utils.str2bool(self.settings_obj[constants.ADMIN_NETWORK].get( - 'enabled')): + 'enabled')): raise NetworkSettingsException("You must enable admin_network " "and configure it explicitly or " "use auto-detection") @@ -274,6 +274,3 @@ class NetworkSettingsException(Exception): def __str__(self): return self.value - - - diff --git a/lib/python/apex_python_utils.py b/lib/python/apex_python_utils.py new file mode 100755 index 00000000..1a3cb159 --- /dev/null +++ b/lib/python/apex_python_utils.py @@ -0,0 +1,206 @@ +############################################################################## +# Copyright (c) 2016 Feng Pan (fpan@redhat.com), Dan Radez (dradez@redhat.com) +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import argparse +import sys +import logging +import os +import yaml + +from copy import copy + +from jinja2 import Environment +from jinja2 import FileSystemLoader + +from apex import NetworkSettings +from apex import NetworkEnvironment +from apex import DeploySettings +from apex import ip_utils +from apex.common.constants import OPNFV_NETWORK_TYPES +from apex.common.constants import ADMIN_NETWORK + + +def parse_net_settings(args): + """ + Parse OPNFV Apex network_settings.yaml config file + and dump bash syntax to set environment variables + + Args: + - file: string + file to network_settings.yaml file + - network_isolation: bool + enable or disable network_isolation + """ + settings = NetworkSettings(args.net_settings_file, + args.network_isolation) + net_env = NetworkEnvironment(settings, args.net_env_file) + dump_yaml(net_env.get_netenv_settings(), '/tmp/network-environment.yaml') + settings.dump_bash() + + +def dump_yaml(data, file): + """ + Dumps data to a file as yaml + :param data: yaml to be written to file + :param file: filename to write to + :return: + """ + with open(file, "w") as fh: + yaml.dump(data, fh, default_flow_style=False) + + +def parse_deploy_settings(args): + settings = DeploySettings(args.file) + settings.dump_bash() + + +def find_ip(args): + """ + Get and print the IP from a specific interface + + Args: + - interface: string + network interface name + - address_family: int + 4 or 6, respective to ipv4 or ipv6 + """ + interface = ip_utils.get_interface(args.interface, + args.address_family) + if interface: + print(interface.ip) + + +def build_nic_template(args): + """ + Build and print a Triple-O nic template from jinja template + + Args: + - template: string + path to jinja template to load + - enabled_networks: comma delimited list + list of networks defined in net_env.py + - ext_net_type: string + interface or br-ex, defines the external network configuration + - address_family: string + 4 or 6, respective to ipv4 or ipv6 + - ovs_dpdk_bridge: string + bridge name to use as ovs_dpdk + """ + template_dir, template = args.template.rsplit('/', 1) + + settings = NetworkSettings(args.net_settings_file, + args.network_isolation).settings_obj + env = Environment(loader=FileSystemLoader(template_dir)) + template = env.get_template(template) + + # gather vlan values into a dict + net_list = copy(args.enabled_networks).split(' ') + net_list.remove(ADMIN_NETWORK) + vlans_vals = map(lambda x: settings[x]['vlan'], net_list) + vlans = dict(zip(net_list, vlans_vals)) + + print(template.render(enabled_networks=args.enabled_networks, + role=args.role, + vlans=vlans, + external_net_type=args.ext_net_type, + external_net_af=args.address_family, + ovs_dpdk_bridge=args.ovs_dpdk_bridge)) + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--debug', action='store_true', default=False, + help="Turn on debug messages") + parser.add_argument('-l', '--log-file', default='/var/log/apex/apex.log', + dest='log_file', help="Log file to log to") + subparsers = parser.add_subparsers() + + net_settings = subparsers.add_parser('parse-net-settings', + help='Parse network settings file') + net_settings.add_argument('-s', '--net-settings-file', + default='network-settings.yaml', + dest='net_settings_file', + help='path to network settings file') + net_settings.add_argument('-i', '--network-isolation', type=bool, + default=True, dest='network_isolation', + help='network isolation') + net_settings.add_argument('-e', '--net-env-file', + default="network-environment.yaml", + dest='net_env_file', + help='path to network environment file') + net_settings.set_defaults(func=parse_net_settings) + + get_int_ip = subparsers.add_parser('find-ip', + help='Find interface ip') + get_int_ip.add_argument('-i', '--interface', required=True, + help='Interface name') + get_int_ip.add_argument('-af', '--address-family', default=4, type=int, + choices=[4, 6], dest='address_family', + help='IP Address family') + get_int_ip.set_defaults(func=find_ip) + + nic_template = subparsers.add_parser('nic-template', + help='Build NIC templates') + nic_template.add_argument('-r', '--role', required=True, + choices=['controller', 'compute'], + help='Role template generated for') + nic_template.add_argument('-t', '--template', required=True, + dest='template', + help='Template file to process') + nic_template.add_argument('-s', '--net-settings-file', + default='network-settings.yaml', + dest='net_settings_file', + help='path to network settings file') + nic_template.add_argument('-i', '--network-isolation', type=bool, + default=True, dest='network_isolation', + help='network isolation') + nic_template.add_argument('-n', '--enabled-networks', required=True, + dest='enabled_networks', + help='enabled network list') + nic_template.add_argument('-e', '--ext-net-type', default='interface', + dest='ext_net_type', + choices=['interface', 'br-ex'], + help='External network type') + nic_template.add_argument('-af', '--address-family', type=int, default=4, + dest='address_family', help='IP address family') + nic_template.add_argument('-d', '--ovs-dpdk-bridge', + default=None, dest='ovs_dpdk_bridge', + help='OVS DPDK Bridge Name') + nic_template.set_defaults(func=build_nic_template) + + deploy_settings = subparsers.add_parser('parse-deploy-settings', + help='Parse deploy settings file') + deploy_settings.add_argument('-f', '--file', + default='deploy_settings.yaml', + help='path to deploy settings file') + deploy_settings.set_defaults(func=parse_deploy_settings) + + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + apex_log_filename = args.log_file + os.makedirs(os.path.dirname(apex_log_filename), exist_ok=True) + logging.basicConfig(filename=apex_log_filename, + format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', + level=logging.DEBUG) + if hasattr(args, 'func'): + args.func(args) + else: + parser.print_help() + exit(1) + +if __name__ == "__main__": + main() diff --git a/tests/test_apex_deploy_env.py b/tests/test_apex_deploy_env.py index 0cd144ef..648923d0 100644 --- a/tests/test_apex_deploy_env.py +++ b/tests/test_apex_deploy_env.py @@ -30,23 +30,23 @@ deploy_files = ('deploy_settings.yaml', 'os-onos-nofeature-ha.yaml') test_deploy_content = ( -'global_params:', -'deploy_options: string', -"""deploy_options: string + 'global_params:', + 'deploy_options: string', + """deploy_options: string global_params:""", -"""global_params: + """global_params: deploy_options: error: error """, -"""global_params: + """global_params: deploy_options: performance: string """, -"""global_params: + """global_params: deploy_options: dataplane: invalid """, -"""global_params: + """global_params: deploy_options: performance: Controller: @@ -78,7 +78,8 @@ class TestIpUtils(object): f = open('/tmp/apex_deploy_test_file', 'w') f.write(c) f.close() - assert_raises(DeploySettingsException, DeploySettings, '/tmp/apex_deploy_test_file') + assert_raises(DeploySettingsException, + DeploySettings, '/tmp/apex_deploy_test_file') def test_dump_bash(self): # the performance file has the most use of the function diff --git a/tests/test_apex_ip_utils.py b/tests/test_apex_ip_utils.py index dc3aadf7..60c8b2b8 100644 --- a/tests/test_apex_ip_utils.py +++ b/tests/test_apex_ip_utils.py @@ -60,9 +60,9 @@ class TestIpUtils(object): def test_get_interface(self): assert_equal(get_interface(''), None) assert_equal(get_interface('notreal'), None) - assert_is_instance(get_interface( - self.iface_name, - address_family=4), IPv4Address) + assert_is_instance(get_interface(self.iface_name, + address_family=4), + IPv4Address) # assert_is_instance(get_interface( # self.iface_name, # address_family=6), IPv6Address) @@ -93,8 +93,7 @@ class TestIpUtils(object): assert_regexp_matches(get_ip_range(interface=self.iface, end_offset=20, count=10), ip4_range_pattern) - @staticmethod - def test_get_ip_range_with_cidr(): + def test_get_ip_range_with_cidr(self): cidr = ip_network('10.10.10.0/24') assert_raises(IPUtilsException, get_ip_range, cidr=cidr) assert_regexp_matches(get_ip_range(cidr=cidr, start_offset=1, diff --git a/tests/test_apex_network_environment.py b/tests/test_apex_network_environment.py index 90c89073..2a8438fa 100644 --- a/tests/test_apex_network_environment.py +++ b/tests/test_apex_network_environment.py @@ -33,7 +33,8 @@ class TestNetworkEnvironment(object): """This method is run once after _each_ test method is executed""" def test_init(self): - assert_raises(NetworkEnvException, NetworkEnvironment, None, '../build/network-environment.yaml') + assert_raises(NetworkEnvException, NetworkEnvironment, + None, '../build/network-environment.yaml') def test_get_netenv_settings(self): ns = NetworkSettings('../config/network/network_settings.yaml', True) diff --git a/tests/test_apex_python_utils_py.py b/tests/test_apex_python_utils_py.py new file mode 100644 index 00000000..47c10092 --- /dev/null +++ b/tests/test_apex_python_utils_py.py @@ -0,0 +1,80 @@ +############################################################################## +# Copyright (c) 2016 Dan Radez (Red Hat) +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import sys + +from test_apex_ip_utils import get_default_gateway_linux +from apex_python_utils import main +from apex_python_utils import get_parser +from apex_python_utils import parse_net_settings +from apex_python_utils import parse_deploy_settings +from apex_python_utils import find_ip +from apex_python_utils import build_nic_template + +from nose.tools import assert_equal +from nose.tools import assert_raises + + +net_sets = '../config/network/network_settings.yaml' +net_env = '../build/network-environment.yaml' +deploy_sets = '../config/deploy/deploy_settings.yaml' +nic_template = '../build/nics-template.yaml.jinja2' + + +class TestCommonUtils(object): + @classmethod + def setup_class(klass): + """This method is run once for each class before any tests are run""" + klass.parser = get_parser() + klass.iface_name = get_default_gateway_linux() + + @classmethod + def teardown_class(klass): + """This method is run once for each class _after_ all tests are run""" + + def setUp(self): + """This method is run once before _each_ test method is executed""" + + def teardown(self): + """This method is run once after _each_ test method is executed""" + + def test_main(self): + sys.argv = ['apex_python_utils', '-l', '/dev/null'] + assert_raises(SystemExit, main) + sys.argv = ['apex_python_utils', '--debug', '-l', '/dev/null'] + assert_raises(SystemExit, main) + sys.argv = ['apex_python_utils', '-l', '/dev/null', + 'parse-deploy-settings', + '-f', deploy_sets] + assert_equal(main(), None) + + def test_parse_net_settings(self): + args = self.parser.parse_args(['parse-net-settings', + '-s', net_sets, + '-i', 'True', + '-e', net_env]) + assert_equal(parse_net_settings(args), None) + + def test_parse_deploy_settings(self): + args = self.parser.parse_args(['parse-deploy-settings', + '-f', deploy_sets]) + assert_equal(parse_deploy_settings(args), None) + + def test_find_ip(self): + args = self.parser.parse_args(['find-ip', + '-i', self.iface_name]) + assert_equal(find_ip(args), None) + + def test_build_nic_template(self): + args = self.parser.parse_args(['nic-template', + '-s', net_sets, + '-r', 'compute', + '-t', nic_template, + '-n', 'admin_network']) + assert_equal(build_nic_template(args), None) -- cgit 1.2.3-korg