Add mapping option to influence nicN mapping order

Currently there's a fixed mapping between abstracted interface
names (nic1, nic2 etc) and the underlying biosdevname for the
device.

In many cases, this mapping based on system enumeration is
sufficient, but in some cases, particularly when you perform
detailed pre-deployment discovery of interfaces, you may wish
to alter the mapping independently of the config (e.g if the
config is in a heat template, and the discovery data is
provided at runtime).

So this adds a -m option to os-net-config, which enables a
mapping file to be provided, such that specific interfaces
may be mapped to their abstract names based on knowledge of
the devices or the networks they are connected to.

The mapping file has the following format, where em1 and em2 are
device names as detected by the OS (e.g biosdevname):

interface_mapping:
    nic1: em2
    nic2: em1

Or you can use the device MAC instead:

interface_mapping:
    nic1: 12:34:56:78:9a:bc
    nic2: 12:34:56:de:f0:12

Change-Id: I93e6d3ed733244834bb3c2126c91db705b4d9167

6 files changed, 184 insertions, 33 deletions

diff --git a/etc/os-net-config/samples/bond_mapped.yaml b/etc/os-net-config/samples/bond_mapped.yaml
new file mode 100644
index 0000000..9a118c9
--- /dev/null
+++ b/etc/os-net-config/samples/bond_mapped.yaml
@@ -0,0 +1,23 @@
+# Example showing use of the optional nicN abstraction
+# for device naming, which defaults to an ordered
+# translation to biodev names based on which interfaces
+# are active on the system.
+# Optionally the default mapping may be overriden by
+# a mapping file via the -m option.
+network_config:
+  -
+     type: ovs_bridge
+     name: br-ctlplane
+     use_dhcp: true
+     members:
+       -
+         type: ovs_bond
+         name: bond1
+         use_dhcp: true
+         members:
+           -
+             type: interface
+             name: nic1
+           -
+             type: interface
+             name: nic2
diff --git a/etc/os-net-config/samples/mapping.yaml b/etc/os-net-config/samples/mapping.yaml
new file mode 100644
index 0000000..5faec22
--- /dev/null
+++ b/etc/os-net-config/samples/mapping.yaml
@@ -0,0 +1,8 @@
+# This can be used with the -m option to override the
+# default mapping of the nicN aliases in configs
+# The mapping can specify either a device name or a mac address
+interface_mapping:
+    nic1: em3
+    nic2: em1
+    nic3: 12:34:56:de:f0:12
+    nic4: 12:34:56:78:9a:bc
diff --git a/os_net_config/cli.py b/os_net_config/cli.py
index 2df13c9..d22573d 100644
--- a/os_net_config/cli.py
+++ b/os_net_config/cli.py
@@ -38,6 +38,9 @@ def parse_opts(argv):
     parser.add_argument('-c', '--config-file', metavar='CONFIG_FILE',
                         help="""path to the configuration file.""",
                         default='/etc/os-net-config/config.yaml')
+    parser.add_argument('-m', '--mapping-file', metavar='MAPPING_FILE',
+                        help="""path to the interface mapping file.""",
+                        default='/etc/os-net-config/mapping.yaml')
     parser.add_argument('-p', '--provider', metavar='PROVIDER',
                         help="""The provider to use."""
                         """One of: ifcfg, eni, iproute.""",
@@ -94,6 +97,8 @@ def main(argv=sys.argv):
     opts = parse_opts(argv)
     configure_logger(opts.verbose, opts.debug)
     logger.info('Using config file at: %s' % opts.config_file)
+    if opts.mapping_file:
+        logger.info('Using mapping file at: %s' % opts.mapping_file)
     iface_array = []
 
     provider = None
@@ -116,6 +121,7 @@ def main(argv=sys.argv):
             logger.error('Unable to set provider for this operating system.')
             return 1
 
+    # Read config file containing network configs to apply
     if os.path.exists(opts.config_file):
         with open(opts.config_file) as cf:
             iface_array = yaml.load(cf.read()).get("network_config")
@@ -123,10 +129,23 @@ def main(argv=sys.argv):
     else:
         logger.error('No config file exists at: %s' % opts.config_file)
         return 1
+
     if not isinstance(iface_array, list):
         logger.error('No interfaces defined in config: %s' % opts.config_file)
         return 1
+
+    # Read the interface mapping file, if it exists
+    # This allows you to override the default network naming abstraction
+    # mappings by specifying a specific nicN->name or nicN->MAC mapping
+    if os.path.exists(opts.mapping_file):
+        with open(opts.mapping_file) as cf:
+            iface_mapping = yaml.load(cf.read()).get("interface_mapping")
+            logger.debug('interface_mapping JSON: %s' % str(iface_mapping))
+    else:
+        iface_mapping = None
+
     for iface_json in iface_array:
+        iface_json.update({'nic_mapping': iface_mapping})
         obj = objects.object_from_json(iface_json)
         provider.add_object(obj)
     files_changed = provider.apply(noop=opts.noop, cleanup=opts.cleanup)
diff --git a/os_net_config/objects.py b/os_net_config/objects.py
index 4a43528..929bcab 100644
--- a/os_net_config/objects.py
+++ b/os_net_config/objects.py
@@ -51,16 +51,45 @@ def _get_required_field(json, name, object_name):
     return field
 
 
-def _numbered_nics():
+def _numbered_nics(nic_mapping=None):
+    mapping = nic_mapping or {}
     global _NUMBERED_NICS
     if _NUMBERED_NICS:
         return _NUMBERED_NICS
     _NUMBERED_NICS = {}
     count = 0
-    for nic in utils.ordered_active_nics():
+    active_nics = utils.ordered_active_nics()
+    for nic in active_nics:
         count += 1
-        _NUMBERED_NICS["nic%i" % count] = nic
-        logger.info("nic%i mapped to: %s" % (count, nic))
+        nic_alias = "nic%i" % count
+        nic_mapped = mapping.get(nic_alias, nic)
+
+        # The mapping is either invalid, or specifies a mac
+        if nic_mapped not in active_nics:
+            for active in active_nics:
+                try:
+                    active_mac = utils.interface_mac(active)
+                except IOError:
+                    continue
+                if nic_mapped == active_mac:
+                    logger.debug("%s matches device %s" % (nic_mapped, active))
+                    nic_mapped = active
+                    break
+            else:
+                # The mapping can't specify a non-active or non-existent nic
+                logger.warning('interface %s is not in an active nic (%s)'
+                               % (nic_mapped, ', '.join(active_nics)))
+                continue
+
+        # Duplicate mappings are not allowed
+        if nic_mapped in _NUMBERED_NICS.values():
+            msg = ('interface %s already mapped, '
+                   'check mapping file for duplicates'
+                   % nic_mapped)
+            raise InvalidConfigException(msg)
+
+        _NUMBERED_NICS[nic_alias] = nic_mapped
+        logger.info("%s mapped to: %s" % (nic_alias, nic_mapped))
     return _NUMBERED_NICS
 
 
@@ -100,8 +129,8 @@ class _BaseOpts(object):
     """Base abstraction for logical port options."""
 
     def __init__(self, name, use_dhcp=False, use_dhcpv6=False, addresses=[],
-                 routes=[], mtu=1500, primary=False):
-        numbered_nic_names = _numbered_nics()
+                 routes=[], mtu=1500, primary=False, nic_mapping=None):
+        numbered_nic_names = _numbered_nics(nic_mapping)
         if name in numbered_nic_names:
             self.name = numbered_nic_names[name]
         else:
@@ -162,19 +191,23 @@ class _BaseOpts(object):
                 msg = 'Routes must be a list.'
                 raise InvalidConfigException(msg)
 
+        nic_mapping = json.get('nic_mapping')
+
         if include_primary:
-            return (use_dhcp, use_dhcpv6, addresses, routes, mtu, primary)
+            return (use_dhcp, use_dhcpv6, addresses, routes, mtu, primary,
+                    nic_mapping)
         else:
-            return (use_dhcp, use_dhcpv6, addresses, routes, mtu)
+            return (use_dhcp, use_dhcpv6, addresses, routes, mtu,
+                    nic_mapping)
 
 
 class Interface(_BaseOpts):
     """Base class for network interfaces."""
 
     def __init__(self, name, use_dhcp=False, use_dhcpv6=False, addresses=[],
-                 routes=[], mtu=1500, primary=False):
+                 routes=[], mtu=1500, primary=False, nic_mapping=None):
         super(Interface, self).__init__(name, use_dhcp, use_dhcpv6, addresses,
-                                        routes, mtu, primary)
+                                        routes, mtu, primary, nic_mapping)
 
     @staticmethod
     def from_json(json):
@@ -191,13 +224,14 @@ class Vlan(_BaseOpts):
     """
 
     def __init__(self, device, vlan_id, use_dhcp=False, use_dhcpv6=False,
-                 addresses=[], routes=[], mtu=1500, primary=False):
+                 addresses=[], routes=[], mtu=1500, primary=False,
+                 nic_mapping=None):
         name = 'vlan%i' % vlan_id
         super(Vlan, self).__init__(name, use_dhcp, use_dhcpv6, addresses,
-                                   routes, mtu, primary)
+                                   routes, mtu, primary, nic_mapping)
         self.vlan_id = int(vlan_id)
 
-        numbered_nic_names = _numbered_nics()
+        numbered_nic_names = _numbered_nics(nic_mapping)
         if device in numbered_nic_names:
             self.device = numbered_nic_names[device]
         else:
@@ -217,9 +251,9 @@ class OvsBridge(_BaseOpts):
 
     def __init__(self, name, use_dhcp=False, use_dhcpv6=False, addresses=[],
                  routes=[], mtu=1500, members=[], ovs_options=None,
-                 ovs_extra=[]):
+                 ovs_extra=[], nic_mapping=None):
         super(OvsBridge, self).__init__(name, use_dhcp, use_dhcpv6, addresses,
-                                        routes, mtu, False)
+                                        routes, mtu, False, nic_mapping)
         self.members = members
         self.ovs_options = ovs_options
         self.ovs_extra = ovs_extra
@@ -238,7 +272,8 @@ class OvsBridge(_BaseOpts):
     @staticmethod
     def from_json(json):
         name = _get_required_field(json, 'name', 'OvsBridge')
-        opts = _BaseOpts.base_opts_from_json(json, include_primary=False)
+        (use_dhcp, use_dhcpv6, addresses, routes, mtu, nic_mapping
+         ) = _BaseOpts.base_opts_from_json(json, include_primary=False)
         ovs_options = json.get('ovs_options')
         ovs_extra = json.get('ovs_extra', [])
         members = []
@@ -253,8 +288,10 @@ class OvsBridge(_BaseOpts):
                 msg = 'Members must be a list.'
                 raise InvalidConfigException(msg)
 
-        return OvsBridge(name, *opts, members=members, ovs_options=ovs_options,
-                         ovs_extra=ovs_extra)
+        return OvsBridge(name, use_dhcp=use_dhcp, use_dhcpv6=use_dhcpv6,
+                         addresses=addresses, routes=routes, mtu=mtu,
+                         members=members, ovs_options=ovs_options,
+                         ovs_extra=ovs_extra, nic_mapping=nic_mapping)
 
 
 class OvsBond(_BaseOpts):
@@ -262,9 +299,9 @@ class OvsBond(_BaseOpts):
 
     def __init__(self, name, use_dhcp=False, use_dhcpv6=False, addresses=[],
                  routes=[], mtu=1500, primary=False, members=[],
-                 ovs_options=None, ovs_extra=[]):
+                 ovs_options=None, ovs_extra=[], nic_mapping=None):
         super(OvsBond, self).__init__(name, use_dhcp, use_dhcpv6, addresses,
-                                      routes, mtu, primary)
+                                      routes, mtu, primary, nic_mapping)
         self.members = members
         self.ovs_options = ovs_options
         self.ovs_extra = ovs_extra
@@ -281,7 +318,8 @@ class OvsBond(_BaseOpts):
     @staticmethod
     def from_json(json):
         name = _get_required_field(json, 'name', 'OvsBond')
-        opts = _BaseOpts.base_opts_from_json(json)
+        (use_dhcp, use_dhcpv6, addresses, routes, mtu, nic_mapping
+         ) = _BaseOpts.base_opts_from_json(json, include_primary=False)
         ovs_options = json.get('ovs_options')
         ovs_extra = json.get('ovs_extra', [])
         members = []
@@ -296,5 +334,7 @@ class OvsBond(_BaseOpts):
                 msg = 'Members must be a list.'
                 raise InvalidConfigException(msg)
 
-        return OvsBond(name, *opts, members=members, ovs_options=ovs_options,
-                       ovs_extra=ovs_extra)
+        return OvsBond(name, use_dhcp=use_dhcp, use_dhcpv6=use_dhcpv6,
+                       addresses=addresses, routes=routes, mtu=mtu,
+                       members=members, ovs_options=ovs_options,
+                       ovs_extra=ovs_extra, nic_mapping=nic_mapping)
diff --git a/os_net_config/tests/base.py b/os_net_config/tests/base.py
index 1e87ba2..30f9d97 100644
--- a/os_net_config/tests/base.py
+++ b/os_net_config/tests/base.py
@@ -29,6 +29,7 @@ _TRUE_VALUES = ('True', 'true', '1', 'yes')
 class TestCase(testtools.TestCase):
 
     """Test case base class for all unit tests."""
+    stub_numbered_nics = True
 
     def setUp(self):
         """Run before each test method to initialize test environment."""
@@ -36,9 +37,10 @@ class TestCase(testtools.TestCase):
         super(TestCase, self).setUp()
         self.stubs = stubout.StubOutForTesting()
 
-        def test_numbered_nics():
+        def dummy_numbered_nics(nic_mapping=None):
             return {}
-        self.stubs.Set(objects, '_numbered_nics', test_numbered_nics)
+        if self.stub_numbered_nics:
+            self.stubs.Set(objects, '_numbered_nics', dummy_numbered_nics)
 
         test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
         try:
diff --git a/os_net_config/tests/test_objects.py b/os_net_config/tests/test_objects.py
index 2f43997..268a48b 100644
--- a/os_net_config/tests/test_objects.py
+++ b/os_net_config/tests/test_objects.py
@@ -15,9 +15,11 @@
 # under the License.
 
 import json
+import six
 
 from os_net_config import objects
 from os_net_config.tests import base
+from os_net_config import utils
 
 
 class TestRoute(base.TestCase):
@@ -93,9 +95,9 @@ class TestInterface(base.TestCase):
         self.assertEqual(True, interface.use_dhcp)
 
     def test_from_json_dhcp_nic1(self):
-        def test_numbered_nics():
+        def dummy_numbered_nics(nic_mapping=None):
             return {"nic1": "em3"}
-        self.stubs.Set(objects, '_numbered_nics', test_numbered_nics)
+        self.stubs.Set(objects, '_numbered_nics', dummy_numbered_nics)
 
         data = '{"type": "interface", "name": "nic1", "use_dhcp": true}'
         interface = objects.object_from_json(json.loads(data))
@@ -141,9 +143,9 @@ class TestVlan(base.TestCase):
         self.assertEqual(True, vlan.use_dhcp)
 
     def test_from_json_dhcp_nic1(self):
-        def test_numbered_nics():
+        def dummy_numbered_nics(nic_mapping=None):
             return {"nic1": "em4"}
-        self.stubs.Set(objects, '_numbered_nics', test_numbered_nics)
+        self.stubs.Set(objects, '_numbered_nics', dummy_numbered_nics)
 
         data = '{"type": "vlan", "device": "nic1", "vlan_id": 16,' \
                '"use_dhcp": true}'
@@ -175,9 +177,9 @@ class TestBridge(base.TestCase):
         self.assertEqual("br-foo", interface1.bridge_name)
 
     def test_from_json_dhcp_with_nic1(self):
-        def test_numbered_nics():
+        def dummy_numbered_nics(nic_mapping=None):
             return {"nic1": "em5"}
-        self.stubs.Set(objects, '_numbered_nics', test_numbered_nics)
+        self.stubs.Set(objects, '_numbered_nics', dummy_numbered_nics)
 
         data = """{
 "type": "ovs_bridge",
@@ -258,9 +260,9 @@ class TestBond(base.TestCase):
 
     def test_from_json_dhcp_with_nic1_nic2(self):
 
-        def test_numbered_nics():
+        def dummy_numbered_nics(nic_mapping=None):
             return {"nic1": "em1", "nic2": "em2"}
-        self.stubs.Set(objects, '_numbered_nics', test_numbered_nics)
+        self.stubs.Set(objects, '_numbered_nics', dummy_numbered_nics)
 
         data = """{
 "type": "ovs_bond",
@@ -285,3 +287,60 @@ class TestBond(base.TestCase):
         self.assertEqual("em1", interface1.name)
         interface2 = bridge.members[1]
         self.assertEqual("em2", interface2.name)
+
+
+class TestNumberedNicsMapping(base.TestCase):
+
+    # We want to test the function, not the dummy..
+    stub_numbered_nics = False
+
+    def tearDown(self):
+        super(TestNumberedNicsMapping, self).tearDown()
+        objects._NUMBERED_NICS = None
+
+    def _stub_active_nics(self, nics):
+        def dummy_ordered_active_nics():
+            return nics
+        self.stubs.Set(utils, 'ordered_active_nics', dummy_ordered_active_nics)
+
+    def test_numbered_nics_default(self):
+        self._stub_active_nics(['em1', 'em2'])
+        expected = {'nic1': 'em1', 'nic2': 'em2'}
+        self.assertEqual(expected, objects._numbered_nics())
+
+    def test_numbered_nics_mapped(self):
+        self._stub_active_nics(['em1', 'em2'])
+        mapping = {'nic1': 'em2', 'nic2': 'em1'}
+        expected = {'nic1': 'em2', 'nic2': 'em1'}
+        self.assertEqual(expected, objects._numbered_nics(nic_mapping=mapping))
+
+    def test_numbered_nics_mapped_partial(self):
+        self._stub_active_nics(['em1', 'em2', 'em3', 'em4'])
+        mapping = {'nic1': 'em2', 'nic2': 'em1'}
+        expected = {'nic1': 'em2', 'nic2': 'em1', 'nic3': 'em3', 'nic4': 'em4'}
+        self.assertEqual(expected, objects._numbered_nics(nic_mapping=mapping))
+
+    def test_numbered_nics_map_error_notactive(self):
+        self._stub_active_nics(['em1', 'em2'])
+        mapping = {'nic1': 'em3', 'nic2': 'em1'}
+        expected = {'nic2': 'em1'}
+        self.assertEqual(expected, objects._numbered_nics(nic_mapping=mapping))
+
+    def test_numbered_nics_map_error_duplicate(self):
+        self._stub_active_nics(['em1', 'em2'])
+        mapping = {'nic1': 'em1', 'nic2': 'em1'}
+        err = self.assertRaises(objects.InvalidConfigException,
+                                objects._numbered_nics, nic_mapping=mapping)
+        expected = 'em1 already mapped, check mapping file for duplicates'
+        self.assertIn(expected, six.text_type(err))
+
+    def test_numbered_nics_map_mac(self):
+        def dummy_interface_mac(name):
+            mac_map = {'em1': '12:34:56:78:9a:bc',
+                       'em2': '12:34:56:de:f0:12'}
+            return mac_map[name]
+        self.stubs.Set(utils, 'interface_mac', dummy_interface_mac)
+        self._stub_active_nics(['em1', 'em2'])
+        mapping = {'nic1': '12:34:56:de:f0:12', 'nic2': '12:34:56:78:9a:bc'}
+        expected = {'nic1': 'em2', 'nic2': 'em1'}
+        self.assertEqual(expected, objects._numbered_nics(nic_mapping=mapping))