Add python parsing library for network settings file.

Changes:
   - Implements network_settings.yaml file parsing in python.
   - Adds support for both IPv4 and IPv6 in network_settings.yaml
   - Adds support for api_network in network_settings.yaml
   - Removes bash library functions for network related functions.
   - Adds dependency to python34-yaml for apex-common package.

Note that support for ipv6 and api_network is not complete yet.
Proper configuriration of network environment and nic template
files will be added later.

Change-Id: I087f725dabedfef109c9de1f58ce2611da647e87
Signed-off-by: Feng Pan <fpan@redhat.com>

4 files changed, 523 insertions, 16 deletions

diff --git a/lib/python/apex-python-utils.py b/lib/python/apex-python-utils.py
new file mode 100755
index 00000000..802e8571
--- /dev/null
+++ b/lib/python/apex-python-utils.py
@@ -0,0 +1,67 @@
+##############################################################################
+# Copyright (c) 2016 Feng Pan (fpan@redhat.com)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import argparse
+import sys
+import apex
+import logging
+import os
+
+def parse_net_settings(settings_args):
+    settings = apex.NetworkSettings(settings_args.path,
+                                    settings_args.network_isolation)
+    settings.dump_bash()
+
+
+def find_ip(int_args):
+    interface = apex.ip_utils.get_interface(int_args.interface,
+                                      int_args.address_family)
+    if interface:
+        print(interface.ip)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--DEBUG', action='store_true', default=False,
+                    help="Turn on debug messages")
+subparsers = parser.add_subparsers()
+
+net_settings = subparsers.add_parser('parse_net_settings',
+                                     help='Parse network settings file')
+net_settings.add_argument('-n', '--path', default='network_settings.yaml',
+                          help='path to network settings file')
+net_settings.add_argument('-i', '--network_isolation', type=bool, default=True,
+                          help='network isolation')
+net_settings.set_defaults(func=parse_net_settings)
+
+get_int_ip = subparsers.add_parser('find_ip',
+                                   help='Find interface ip')
+get_int_ip.add_argument('-i', '--interface', required=True,
+                        help='Interface name')
+get_int_ip.add_argument('-af', '--address_family', default=4, type=int,
+                        choices=[4, 6],
+                        help='IP Address family')
+get_int_ip.set_defaults(func=find_ip)
+
+args = parser.parse_args(sys.argv[1:])
+if args.DEBUG:
+    logging.basicConfig(level=logging.DEBUG)
+else:
+    apex_log_filename = '/var/log/apex/apex.log'
+    os.makedirs(os.path.dirname(apex_log_filename), exist_ok=True)
+    logging.basicConfig(filename=apex_log_filename,
+                        format='%(asctime)s %(levelname)s: %(message)s',
+                        datefmt='%m/%d/%Y %I:%M:%S %p',
+                        level=logging.DEBUG)
+
+if hasattr(args, 'func'):
+    args.func(args)
+else:
+    parser.print_help()
+    exit(1)
diff --git a/lib/python/apex/__init__.py b/lib/python/apex/__init__.py
index 0c0ae6c6..88b066b2 100644
--- a/lib/python/apex/__init__.py
+++ b/lib/python/apex/__init__.py
@@ -7,3 +7,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
+
+from .net_env import NetworkSettings
diff --git a/lib/python/apex/ip_utils.py b/lib/python/apex/ip_utils.py
index 680ce7e0..d7099db5 100644
--- a/lib/python/apex/ip_utils.py
+++ b/lib/python/apex/ip_utils.py
@@ -1,4 +1,3 @@
-
 ##############################################################################
 # Copyright (c) 2016 Feng Pan (fpan@redhat.com) and others.
 #
@@ -10,6 +9,130 @@
 
 
 import ipaddress
+import subprocess
+import re
+import logging
+
+
+def get_ip_range(start_offset=None, count=None, end_offset=None,
+                 cidr=None, interface=None):
+    """
+    Generate IP range for a network (cidr) or an interface.
+
+    If CIDR is provided, it will take precedence over interface. In this case,
+    The entire CIDR IP address space is considered usable. start_offset will be
+    calculated from the network address, and end_offset will be calculated from
+    the last address in subnet.
+
+    If interface is provided, the interface IP will be used to calculate
+    offsets:
+        - If the interface IP is in the first half of the address space,
+        start_offset will be calculated from the interface IP, and end_offset
+        will be calculated from end of address space.
+        - If the interface IP is in the second half of the address space,
+        start_offset will be calculated from the network address in the address
+        space, and end_offset will be calculated from the interface IP.
+
+    2 of start_offset, end_offset and count options must be provided:
+        - If start_offset and end_offset are provided, a range from start_offset
+        to end_offset will be returned.
+        - If count is provided, a range from either start_offset to (start_offset
+        +count) or (end_offset-count) to end_offset will be returned. The
+        IP range returned will be of size <count>.
+    Both start_offset and end_offset must be greater than 0.
+
+    Returns IP range in the format of "first_addr,second_addr" or exception
+    is raised.
+    """
+    if cidr:
+        if count and start_offset and not end_offset:
+            start_index = start_offset
+            end_index = start_offset + count -1
+        elif count and end_offset and not start_offset:
+            end_index = -1 - end_offset
+            start_index = -1 - end_index - count + 1
+        elif start_offset and end_offset and not count:
+            start_index = start_offset
+            end_index = -1 - end_offset
+        else:
+            raise IPUtilsException("Argument error: must pass in exactly 2 of"
+                                   "start_offset, end_offset and count")
+
+        start_ip = cidr[start_index]
+        end_ip = cidr[end_index]
+        network = cidr
+    elif interface:
+        network = interface.network
+        number_of_addr = network.num_addresses
+        if interface.ip < network[int(number_of_addr / 2)]:
+            if count and start_offset and not end_offset:
+                start_ip = interface.ip + start_offset
+                end_ip = start_ip + count - 1
+            elif count and end_offset and not start_offset:
+                end_ip = network[-1 - end_offset]
+                start_ip = end_ip - count + 1
+            elif start_offset and end_offset and not count:
+                start_ip = interface.ip + start_offset
+                end_ip = network[-1 - end_offset]
+            else:
+                raise IPUtilsException(
+                    "Argument error: must pass in exactly 2 of"
+                    "start_offset, end_offset and count")
+        else:
+            if count and start_offset and not end_offset:
+                start_ip = network[start_offset]
+                end_ip = start_ip + count -1
+            elif count and end_offset and not start_offset:
+                end_ip = interface.ip - end_offset
+                start_ip = end_ip - count + 1
+            elif start_offset and end_offset and not count:
+                start_ip = network[start_offset]
+                end_ip = interface.ip - end_offset
+            else:
+                raise IPUtilsException(
+                    "Argument error: must pass in exactly 2 of"
+                    "start_offset, end_offset and count")
+
+    else:
+        raise IPUtilsException("Must pass in cidr or interface to generate"
+                               "ip range")
+
+    range_result = _validate_ip_range(start_ip, end_ip, network)
+    if range_result:
+        ip_range = "{},{}".format(start_ip, end_ip)
+        return ip_range
+    else:
+        raise IPUtilsException("Invalid IP range: {},{} for network {}"
+                               .format(start_ip, end_ip, network))
+
+
+def get_ip(offset, cidr=None, interface=None):
+    """
+    Returns an IP in a network given an offset.
+
+    Either cidr or interface must be provided, cidr takes precedence.
+
+    If cidr is provided, offset is calculated from network address.
+    If interface is provided, offset is calculated from interface IP.
+
+    offset can be positive or negative, but the resulting IP address must also
+    be contained in the same subnet, otherwise an exception will be raised.
+
+    returns a IP address object.
+    """
+    if cidr:
+        ip = cidr[0 + offset]
+        network = cidr
+    elif interface:
+        ip = interface.ip + offset
+        network = interface.network
+    else:
+        raise IPUtilsException("Must pass in cidr or interface to generate IP")
+
+    if ip not in network:
+        raise IPUtilsException("IP {} not in network {}".format(ip, network))
+    else:
+        return str(ip)
 
 
 def generate_ip_range(args):
@@ -22,7 +145,8 @@ def generate_ip_range(args):
     start_position: starting index, default to first address in subnet (1)
     end_position:  ending index, default to last address in subnet (-1)
 
-    Returns IP range in string format. A single IP is returned if start and end IPs are identical.
+    Returns IP range in string format. A single IP is returned if start and
+    end IPs are identical.
     """
     cidr = ipaddress.ip_network(args.CIDR)
     (start_index, end_index) = (args.start_position, args.end_position)
@@ -32,23 +156,95 @@ def generate_ip_range(args):
         return ','.join(sorted([str(cidr[start_index]), str(cidr[end_index])]))
 
 
-def main():
-    import argparse
-    import sys
+def get_interface(nic, address_family=4):
+    """
+    Returns interface object for a given NIC name in the system
+
+    Only global address will be returned at the moment.
+
+    Returns interface object if an address is found for the given nic,
+    otherwise returns None.
+    """
+    if not nic.strip():
+        logging.error("empty nic name specified")
+        return None
+    output = subprocess.getoutput("ip -{} addr show {} scope global"
+                                  .format(address_family, nic))
+    if address_family == 4:
+        pattern = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d{1,2}")
+    elif address_family == 6:
+        pattern = re.compile("([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}/\d{1,3}")
+    else:
+        raise IPUtilsException("Invalid address family: {}"
+                               .format(address_family))
+    match = re.search(pattern, output)
+    if match:
+        logging.info("found interface {} ip: {}".format(nic, match.group()))
+        return ipaddress.ip_interface(match.group())
+    else:
+        logging.info("interface ip not found! ip address output:\n{}"
+                        .format(output))
+        return None
+
+
+def find_gateway(interface):
+    """
+    Validate gateway on the system
+
+    Ensures that the provided interface object is in fact configured as default
+    route on the system.
+
+    Returns gateway IP (reachable from interface) if default route is found,
+    otherwise returns None.
+    """
+
+    address_family = interface.version
+    output = subprocess.getoutput("ip -{} route".format(address_family))
 
-    parser = argparse.ArgumentParser()
-    subparsers = parser.add_subparsers()
+    pattern = re.compile("default\s+via\s+(\S+)\s+")
+    match = re.search(pattern, output)
 
-    parser_gen_ip_range = subparsers.add_parser('generate_ip_range', help='Generate IP Range given CIDR')
-    parser_gen_ip_range.add_argument('CIDR', help='Network in CIDR notation')
-    parser_gen_ip_range.add_argument('start_position', type=int, help='Starting index')
-    parser_gen_ip_range.add_argument('end_position', type=int, help='Ending index')
-    parser_gen_ip_range.set_defaults(func=generate_ip_range)
+    if match:
+        gateway_ip = match.group(1)
+        reverse_route_output = subprocess.getoutput("ip route get {}"
+                                                    .format(gateway_ip))
+        pattern = re.compile("{}.+src\s+{}".format(gateway_ip, interface.ip))
+        if not re.search(pattern, reverse_route_output):
+            logging.warning("Default route doesn't match interface specified: "
+                            "{}".format(reverse_route_output))
+            return None
+        else:
+            return gateway_ip
+    else:
+        logging.warning("Can't find gateway address on system")
+        return None
+
+
+def _validate_ip_range(start_ip, end_ip, cidr):
+    """
+    Validates an IP range is in good order and the range is part of cidr.
+
+    Returns True if validation succeeds, False otherwise.
+    """
+    ip_range = "{},{}".format(start_ip, end_ip)
+    if end_ip <= start_ip:
+        logging.warning("IP range {} is invalid: end_ip should be greater than "
+                        "starting ip".format(ip_range))
+        return False
+    if start_ip not in ipaddress.ip_network(cidr):
+        logging.warning('start_ip {} is not in network {}'
+                        .format(start_ip, cidr))
+        return False
+    if end_ip not in ipaddress.ip_network(cidr):
+        logging.warning('end_ip {} is not in network {}'.format(end_ip, cidr))
+        return False
 
-    args = parser.parse_args(sys.argv[1:])
-    print(args.func(args))
+    return True
 
 
-if __name__ == '__main__':
-    main()
+class IPUtilsException(Exception):
+    def __init__(self, value):
+        self.value = value
 
+    def __str__(self):
+        return self.value
diff --git a/lib/python/apex/net_env.py b/lib/python/apex/net_env.py
new file mode 100644
index 00000000..ec46fe28
--- /dev/null
+++ b/lib/python/apex/net_env.py
@@ -0,0 +1,242 @@
+##############################################################################
+# Copyright (c) 2016 Feng Pan (fpan@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import yaml
+import logging
+import ipaddress
+from . import ip_utils
+
+
+ADMIN_NETWORK = 'admin_network'
+PRIVATE_NETWORK = 'private_network'
+PUBLIC_NETWORK = 'public_network'
+STORAGE_NETWORK = 'storage_network'
+API_NETWORK = 'api_network'
+OPNFV_NETWORK_TYPES = [ADMIN_NETWORK, PRIVATE_NETWORK, PUBLIC_NETWORK,
+                       STORAGE_NETWORK, API_NETWORK]
+
+
+class NetworkSettings:
+    """
+    This class parses APEX network settings yaml file into an object. It
+    generates or detects all missing fields for deployment.
+
+    The resulting object will be used later to generate network environment file
+    as well as configuring post deployment networks.
+
+    Currently the parsed object is dumped into a bash global definition file
+    for deploy.sh consumption. This object will later be used directly as
+    deployment script move to python.
+    """
+    def __init__(self, filename, network_isolation):
+        with open(filename, 'r') as network_settings_file:
+            self.settings_obj = yaml.load(network_settings_file)
+            self.network_isolation = network_isolation
+            self.enabled_network_list = []
+            self._validate_input()
+
+    def _validate_input(self):
+        """
+        Validates the network settings file and populates all fields.
+
+        NetworkSettingsException will be raised if validation fails.
+        """
+        if ADMIN_NETWORK not in self.settings_obj or \
+                self.settings_obj[ADMIN_NETWORK].get('enabled') != True:
+            raise NetworkSettingsException("You must enable admin_network "
+                                           "and configure it explicitly or "
+                                           "use auto-detection")
+        if self.network_isolation and \
+            (PUBLIC_NETWORK not in self.settings_obj or
+                self.settings_obj[PUBLIC_NETWORK].get('enabled') != True):
+            raise NetworkSettingsException("You must enable public_network "
+                                           "and configure it explicitly or "
+                                           "use auto-detection")
+
+        for network in OPNFV_NETWORK_TYPES:
+            if network in self.settings_obj:
+                if self.settings_obj[network].get('enabled') == True:
+                    logging.info("{} enabled".format(network))
+                    self._config_required_settings(network)
+                    self._config_ip_range(network=network,
+                                          setting='usable_ip_range',
+                                          start_offset=21, end_offset=21)
+                    self._config_optional_settings(network)
+                    self.enabled_network_list.append(network)
+                else:
+                    logging.info("{} disabled, will collapse with "
+                                 "admin_network".format(network))
+            else:
+                logging.info("{} is not in specified, will collapse with "
+                             "admin_network".format(network))
+
+    def _config_required_settings(self, network):
+        """
+        Configures either CIDR or bridged_interface setting
+
+        cidr takes precedence if both cidr and bridged_interface are specified
+        for a given network.
+
+        When using bridged_interface, we will detect network setting on the
+        given NIC in the system. The resulting config in settings object will
+        be an ipaddress.network object, replacing the NIC name.
+        """
+        cidr = self.settings_obj[network].get('cidr')
+        nic_name = self.settings_obj[network].get('bridged_interface')
+
+        if cidr:
+            cidr = ipaddress.ip_network(self.settings_obj[network]['cidr'])
+            self.settings_obj[network]['cidr'] = cidr
+            logging.info("{}_cidr: {}".format(network, cidr))
+            return 0
+        elif nic_name:
+            # If cidr is not specified, we need to know if we should find
+            # IPv6 or IPv4 address on the interface
+            if self.settings_obj[network].get('ipv6') == True:
+                address_family = 6
+            else:
+                address_family = 4
+            nic_interface = ip_utils.get_interface(nic_name, address_family)
+            if nic_interface:
+                self.settings_obj[network]['bridged_interface'] = nic_interface
+                logging.info("{}_bridged_interface: {}".
+                             format(network, nic_interface))
+                return 0
+            else:
+                raise NetworkSettingsException("Auto detection failed for {}: "
+                                               "Unable to find valid ip for "
+                                               "interface {}"
+                                               .format(network, nic_name))
+
+        else:
+            raise NetworkSettingsException("Auto detection failed for {}: "
+                                           "either bridge_interface or cidr "
+                                           "must be specified"
+                                           .format(network))
+
+    def _config_ip_range(self, network, setting, start_offset=None,
+                         end_offset=None, count=None):
+        """
+        Configures IP range for a given setting.
+
+        If the setting is already specified, no change will be made.
+
+        The spec for start_offset, end_offset and count are identical to
+        ip_utils.get_ip_range.
+        """
+        ip_range = self.settings_obj[network].get(setting)
+        interface = self.settings_obj[network].get('bridged_interface')
+
+        if not ip_range:
+            cidr = self.settings_obj[network].get('cidr')
+            ip_range = ip_utils.get_ip_range(start_offset=start_offset,
+                                             end_offset=end_offset,
+                                             count=count,
+                                             cidr=cidr,
+                                             interface=interface)
+            self.settings_obj[network][setting] = ip_range
+
+        logging.info("{}_{}: {}".format(network, setting, ip_range))
+
+    def _config_ip(self, network, setting, offset):
+        """
+        Configures IP for a given setting.
+
+        If the setting is already specified, no change will be made.
+
+        The spec for offset is identical to ip_utils.get_ip
+        """
+        ip = self.settings_obj[network].get(setting)
+        interface = self.settings_obj[network].get('bridged_interface')
+
+        if not ip:
+            cidr = self.settings_obj[network].get('cidr')
+            ip = ip_utils.get_ip(offset, cidr, interface)
+            self.settings_obj[network][setting] = ip
+
+        logging.info("{}_{}: {}".format(network, setting, ip))
+
+    def _config_optional_settings(self, network):
+        """
+        Configures optional settings:
+        - admin_network:
+            - provisioner_ip
+            - dhcp_range
+            - introspection_range
+        - public_network:
+            - provisioner_ip
+            - floating_ip
+            - gateway
+        """
+        if network == ADMIN_NETWORK:
+            self._config_ip(network, 'provisioner_ip', 1)
+            self._config_ip_range(network=network, setting='dhcp_range',
+                                  start_offset=2, count=9)
+            self._config_ip_range(network=network,
+                                  setting='introspection_range',
+                                  start_offset=11, count=9)
+        elif network == PUBLIC_NETWORK:
+            self._config_ip(network, 'provisioner_ip', 1)
+            self._config_ip_range(network=network,
+                                  setting='floating_ip',
+                                  end_offset=2, count=20)
+            self._config_gateway(network)
+
+    def _config_gateway(self, network):
+        """
+        Configures gateway setting for a given network.
+
+        If cidr is specified, we always use the first address in the address
+        space for gateway. Otherwise, we detect the system gateway.
+        """
+        gateway = self.settings_obj[network].get('gateway')
+        interface = self.settings_obj[network].get('bridged_interface')
+
+        if not gateway:
+            cidr = self.settings_obj[network].get('cidr')
+            if cidr:
+                gateway = ip_utils.get_ip(1, cidr)
+            else:
+                gateway = ip_utils.find_gateway(interface)
+
+            if gateway:
+                self.settings_obj[network]['gateway'] = gateway
+            else:
+                raise NetworkSettingsException("Failed to set gateway")
+
+        logging.info("{}_gateway: {}".format(network, gateway))
+
+
+    def dump_bash(self, path=None):
+        """
+        Prints settings for bash consumption.
+
+        If optional path is provided, bash string will be written to the file
+        instead of stdout.
+        """
+        bash_str = ''
+        for network in self.enabled_network_list:
+            for key, value in self.settings_obj[network].items():
+                bash_str += "{}_{}={}\n".format(network, key, value)
+        bash_str += "enabled_network_list='{}'\n" \
+            .format(' '.join(self.enabled_network_list))
+        if path:
+            with open(path, 'w') as file:
+                file.write(bash_str)
+        else:
+            print(bash_str)
+
+
+class NetworkSettingsException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+            return self.value