moving inventory file parsing to python

Change-Id: Ib03728e8ffe9c65044b32b4348e9c1c88862c6e3
Signed-off-by: Dan Radez <dradez@redhat.com>

9 files changed, 208 insertions, 136 deletions

diff --git a/ci/deploy.sh b/ci/deploy.sh
index 91663dfc..00441975 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -179,8 +179,19 @@ parse_cmdline() {
     exit 1
   fi
 
-  if [[ -n "$virtual" && -n "$INVENTORY_FILE" ]]; then
-    echo -e "${red}ERROR: You should not specify an inventory with virtual deployments${reset}"
+  # inventory file usage validation
+  if [[ -n "$virtual" ]]; then
+      if [[ -n "$INVENTORY_FILE" ]]; then
+          echo -e "${red}ERROR: You should not specify an inventory file with virtual deployments${reset}"
+          exit 1
+      else
+          INVENTORY_FILE='/tmp/inventory-virt.yaml'
+      fi
+  elif [[ -z "$INVENTORY_FILE" ]]; then
+    echo -e "${red}ERROR: You must specify an inventory file for baremetal deployments! Exiting...${reset}"
+    exit 1
+  elif [[ ! -f "$INVENTORY_FILE" ]]; then
+    echo -e "{$red}ERROR: Inventory File: ${INVENTORY_FILE} does not exist! Exiting...${reset}"
     exit 1
   fi
 
@@ -194,16 +205,6 @@ parse_cmdline() {
     exit 1
   fi
 
-  if [[ ! -z "$INVENTORY_FILE" && ! -f "$INVENTORY_FILE" ]]; then
-    echo -e "{$red}ERROR: Inventory File: ${INVENTORY_FILE} does not exist! Exiting...${reset}"
-    exit 1
-  fi
-
-  if [[ -z "$virtual" && -z "$INVENTORY_FILE" ]]; then
-    echo -e "${red}ERROR: You must specify an inventory file for baremetal deployments! Exiting...${reset}"
-    exit 1
-  fi
-
   if [[ "$net_isolation_enabled" == "FALSE" && "$post_config" == "TRUE" ]]; then
     echo -e "${blue}INFO: Post Install Configuration will be skipped.  It is not supported with --flat${reset}"
     post_config="FALSE"
@@ -226,9 +227,8 @@ main() {
   setup_undercloud_vm
   if [ "$virtual" == "TRUE" ]; then
     setup_virtual_baremetal $VM_CPUS $VM_RAM
-  elif [ -n "$INVENTORY_FILE" ]; then
-    parse_inventory_file
   fi
+  parse_inventory_file
   configure_undercloud
   overcloud_deploy
   if [ "$post_config" == "TRUE" ]; then
diff --git a/lib/parse-functions.sh b/lib/parse-functions.sh
index 9c2ebff5..40cdb826 100755
--- a/lib/parse-functions.sh
+++ b/lib/parse-functions.sh
@@ -65,7 +65,7 @@ parse_network_settings() {
       done
   fi
 
-  if output=$(python3.4 -B $LIB/python/apex_python_utils.py parse-net-settings -s $NETSETS $net_isolation_arg -e $CONFIG/network-environment.yaml $parse_ext); then
+  if output=$(python3 -B $LIB/python/apex_python_utils.py parse-net-settings -s $NETSETS $net_isolation_arg -e $CONFIG/network-environment.yaml $parse_ext); then
       echo -e "${blue}${output}${reset}"
       eval "$output"
   else
@@ -88,7 +88,7 @@ parse_network_settings() {
 ##parses deploy settings yaml into globals
 parse_deploy_settings() {
   local output
-  if output=$(python3.4 -B $LIB/python/apex_python_utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then
+  if output=$(python3 -B $LIB/python/apex_python_utils.py parse-deploy-settings -f $DEPLOY_SETTINGS_FILE); then
       echo -e "${blue}${output}${reset}"
       eval "$output"
   else
@@ -99,85 +99,15 @@ parse_deploy_settings() {
 }
 
 ##parses baremetal yaml settings into compatible json
-##writes the json to $CONFIG/instackenv_tmp.json
+##writes the json to undercloud:instackenv.json
 ##params: none
 ##usage: parse_inventory_file
 parse_inventory_file() {
-  local inventory=$(parse_yaml $INVENTORY_FILE)
-  local node_list
-  local node_prefix="node"
-  local node_count=0
-  local node_total
-  local inventory_list
-
-  # detect number of nodes
-  for entry in $inventory; do
-    if echo $entry | grep -Eo "^nodes_node[0-9]+_" > /dev/null; then
-      this_node=$(echo $entry | grep -Eo "^nodes_node[0-9]+_")
-      if [[ "$inventory_list" != *"$this_node"* ]]; then
-        inventory_list+="$this_node "
-      fi
-    fi
-  done
-
-  inventory_list=$(echo $inventory_list | sed 's/ $//')
-
-  for node in $inventory_list; do
-    ((node_count+=1))
-  done
-
-  node_total=$node_count
-
-  if [[ "$node_total" -lt 5 && "$ha_enabled" == "True" ]]; then
-    echo -e "${red}ERROR: You must provide at least 5 nodes for HA baremetal deployment${reset}"
-    exit 1
-  elif [[ "$node_total" -lt 2 ]]; then
-    echo -e "${red}ERROR: You must provide at least 2 nodes for non-HA baremetal deployment${reset}"
-    exit 1
-  fi
-
-  eval $(parse_yaml $INVENTORY_FILE) || {
-    echo "${red}Failed to parse inventory.yaml. Aborting.${reset}"
-    exit 1
-  }
-
-  instackenv_output="
-{
- \"nodes\" : [
-
-"
-  node_count=0
-  for node in $inventory_list; do
-    ((node_count+=1))
-    node_output="
-        {
-          \"pm_password\": \"$(eval echo \${${node}ipmi_pass})\",
-          \"pm_type\": \"$(eval echo \${${node}pm_type})\",
-          \"mac\": [
-            \"$(eval echo \${${node}mac_address})\"
-          ],
-          \"cpu\": \"$(eval echo \${${node}cpus})\",
-          \"memory\": \"$(eval echo \${${node}memory})\",
-          \"disk\": \"$(eval echo \${${node}disk})\",
-          \"arch\": \"$(eval echo \${${node}arch})\",
-          \"pm_user\": \"$(eval echo \${${node}ipmi_user})\",
-          \"pm_addr\": \"$(eval echo \${${node}ipmi_ip})\",
-          \"capabilities\": \"$(eval echo \${${node}capabilities})\"
-"
-    instackenv_output+=${node_output}
-    if [ $node_count -lt $node_total ]; then
-      instackenv_output+="        },"
-    else
-      instackenv_output+="        }"
-    fi
-  done
-
-  instackenv_output+='
-  ]
-}
-'
-  #Copy instackenv.json to undercloud for baremetal
-  echo -e "{blue}Parsed instackenv JSON:\n${instackenv_output}${reset}"
+  if [ "$virtual" == "TRUE" ]; then inv_virt="--virtual"; fi
+  if [[ "$ha_enabled" == "True" ]]; then inv_ha="--ha"; fi
+  instackenv_output=$(python3 -B $LIB/python/apex_python_utils.py parse-inventory -f $INVENTORY_FILE $inv_virt $inv_ha)
+  #Copy instackenv.json to undercloud
+  echo -e "${blue}Parsed instackenv JSON:\n${instackenv_output}${reset}"
   ssh -T ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" <<EOI
 cat > instackenv.json << EOF
 $instackenv_output
diff --git a/lib/python/apex/__init__.py b/lib/python/apex/__init__.py
index 9993dc97..b2a45f7d 100644
--- a/lib/python/apex/__init__.py
+++ b/lib/python/apex/__init__.py
@@ -12,3 +12,4 @@ from .network_settings import NetworkSettings
 from .deploy_settings import DeploySettings
 from .network_environment import NetworkEnvironment
 from .clean import clean_nodes
+from .inventory import Inventory
diff --git a/lib/python/apex/inventory.py b/lib/python/apex/inventory.py
new file mode 100644
index 00000000..f4a33b28
--- /dev/null
+++ b/lib/python/apex/inventory.py
@@ -0,0 +1,76 @@
+##############################################################################
+# Copyright (c) 2016 Dan Radez (dradez@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import yaml
+import json
+
+
+class Inventory(dict):
+    """
+    This class parses an APEX inventory yaml file into an object. It
+    generates or detects all missing fields for deployment.
+
+    It then collapses one level of identifcation from the object to
+    convert it to a structure that can be dumped into a json file formatted
+    such that Triple-O can read the resulting json as an instackenv.json file.
+    """
+    def __init__(self, source, ha=True, virtual=False):
+        init_dict = {}
+        if type(source) is str:
+            with open(source, 'r') as network_settings_file:
+                yaml_dict = yaml.load(network_settings_file)
+            # collapse node identifiers from the structure
+            init_dict['nodes'] = list(map(lambda n: n[1],
+                                          yaml_dict['nodes'].items()))
+        else:
+            # assume input is a dict to build from
+            init_dict = source
+
+        # move ipmi_* to pm_*
+        # make mac a list
+        def munge_nodes(node):
+            node['pm_addr'] = node['ipmi_ip']
+            node['pm_password'] = node['ipmi_pass']
+            node['pm_user'] = node['ipmi_user']
+            node['mac'] = [node['mac_address']]
+
+            for i in ('ipmi_ip', 'ipmi_pass', 'ipmi_user', 'mac_address'):
+                del i
+
+            return node
+
+        super().__init__({'nodes': list(map(munge_nodes, init_dict['nodes']))})
+
+        # verify number of nodes
+        if ha and len(self['nodes']) < 5:
+            raise InventoryException('You must provide at least 5 '
+                                     'nodes for HA baremetal deployment')
+        elif len(self['nodes']) < 2:
+            raise InventoryException('You must provide at least 2 nodes '
+                                     'for non-HA baremetal deployment${reset}')
+
+        if virtual:
+            self['arch'] = 'x86_64'
+            self['host-ip'] = '192.168.122.1'
+            self['power_manager'] = \
+                'nova.virt.baremetal.virtual_power_driver.VirtualPowerManager'
+            self['seed-ip'] = ''
+            self['ssh-key'] = 'INSERT_STACK_USER_PRIV_KEY'
+            self['ssh-user'] = 'root'
+
+    def dump_instackenv_json(self):
+        print(json.dumps(dict(self), sort_keys=True, indent=4))
+
+
+class InventoryException(Exception):
+    def __init__(self, value):
+        self.value = value
+
+    def __str__(self):
+            return self.value
diff --git a/lib/python/apex_python_utils.py b/lib/python/apex_python_utils.py
index 1f5e407c..ebc49dc5 100755
--- a/lib/python/apex_python_utils.py
+++ b/lib/python/apex_python_utils.py
@@ -22,6 +22,7 @@ from jinja2 import FileSystemLoader
 from apex import NetworkSettings
 from apex import NetworkEnvironment
 from apex import DeploySettings
+from apex import Inventory
 from apex import ip_utils
 from apex.common.constants import ADMIN_NETWORK
 
@@ -66,6 +67,11 @@ def run_clean(args):
     apex.clean_nodes(args.file)
 
 
+def parse_inventory(args):
+    inventory = Inventory(args.file, ha=args.ha, virtual=args.virtual)
+    inventory.dump_instackenv_json()
+
+
 def find_ip(args):
     """
     Get and print the IP from a specific interface
@@ -128,7 +134,7 @@ def get_parser():
     parser.add_argument('-l', '--log-file', default='/var/log/apex/apex.log',
                         dest='log_file', help="Log file to log to")
     subparsers = parser.add_subparsers()
-
+    # parse-net-settings
     net_settings = subparsers.add_parser('parse-net-settings',
                                          help='Parse network settings file')
     net_settings.add_argument('-s', '--net-settings-file',
@@ -154,7 +160,7 @@ def get_parser():
                               help='Boolean to enable Controller Pre Config')
 
     net_settings.set_defaults(func=parse_net_settings)
-
+    # find-ip
     get_int_ip = subparsers.add_parser('find-ip',
                                        help='Find interface ip')
     get_int_ip.add_argument('-i', '--interface', required=True,
@@ -163,7 +169,7 @@ def get_parser():
                             choices=[4, 6], dest='address_family',
                             help='IP Address family')
     get_int_ip.set_defaults(func=find_ip)
-
+    # nic-template
     nic_template = subparsers.add_parser('nic-template',
                                          help='Build NIC templates')
     nic_template.add_argument('-r', '--role', required=True,
@@ -189,13 +195,28 @@ def get_parser():
                               default=None, dest='ovs_dpdk_bridge',
                               help='OVS DPDK Bridge Name')
     nic_template.set_defaults(func=build_nic_template)
-
+    # parse-deploy-settings
     deploy_settings = subparsers.add_parser('parse-deploy-settings',
                                             help='Parse deploy settings file')
     deploy_settings.add_argument('-f', '--file',
                                  default='deploy_settings.yaml',
                                  help='path to deploy settings file')
     deploy_settings.set_defaults(func=parse_deploy_settings)
+    # parse-inventory
+    inventory = subparsers.add_parser('parse-inventory',
+                                      help='Parse inventory file')
+    inventory.add_argument('-f', '--file',
+                           default='deploy_settings.yaml',
+                           help='path to deploy settings file')
+    inventory.add_argument('--ha',
+                           default=False,
+                           action='store_true',
+                           help='Indicate if deployment is HA or not')
+    inventory.add_argument('--virtual',
+                           default=False,
+                           action='store_true',
+                           help='Indicate if deployment inventory is virtual')
+    inventory.set_defaults(func=parse_inventory)
 
     clean = subparsers.add_parser('clean',
                                   help='Parse deploy settings file')
diff --git a/lib/undercloud-functions.sh b/lib/undercloud-functions.sh
index 5e9a5caa..177fe443 100755
--- a/lib/undercloud-functions.sh
+++ b/lib/undercloud-functions.sh
@@ -165,11 +165,6 @@ EOI
       # root's auth keys so that Undercloud can control
       # vm power on the hypervisor
       ssh ${SSH_OPTIONS[@]} "stack@$UNDERCLOUD" "cat /home/stack/.ssh/id_rsa.pub" >> /root/.ssh/authorized_keys
-
-      INSTACKENV=$CONFIG/instackenv-virt.json
-
-      # upload instackenv file to Undercloud for virtual deployment
-      scp ${SSH_OPTIONS[@]} $INSTACKENV "stack@$UNDERCLOUD":instackenv.json
   fi
 
   # allow stack to control power management on the hypervisor via sshkey
diff --git a/lib/virtual-setup-functions.sh b/lib/virtual-setup-functions.sh
index 903e3bcd..61dc6799 100755
--- a/lib/virtual-setup-functions.sh
+++ b/lib/virtual-setup-functions.sh
@@ -22,10 +22,9 @@ function setup_virtual_baremetal {
     vcpus=$1
     ramsize=$(($2*1024))
   fi
-  #start by generating the opening json for instackenv.json
-  cat > $CONFIG/instackenv-virt.json << EOF
-{
-  "nodes": [
+  #start by generating the opening yaml for the inventory-virt.yaml file
+  cat > /tmp/inventory-virt.yaml << EOF
+nodes:
 EOF
 
   # next create the virtual machines and add their definitions to the file
@@ -60,44 +59,26 @@ EOF
         fi
       done
     else
-      echo "Found Baremetal ${i} VM, using existing VM"
+      echo "Found baremetal${i} VM, using existing VM"
     fi
     #virsh vol-list default | grep baremetal${i} 2>&1> /dev/null || virsh vol-create-as default baremetal${i}.qcow2 41G --format qcow2
     mac=$(virsh domiflist baremetal${i} | grep admin_network | awk '{ print $5 }')
 
-    cat >> $CONFIG/instackenv-virt.json << EOF
-    {
-      "pm_addr": "192.168.122.1",
-      "pm_user": "root",
-      "pm_password": "INSERT_STACK_USER_PRIV_KEY",
-      "pm_type": "pxe_ssh",
-      "mac": [
-        "$mac"
-      ],
-      "cpu": "$vcpus",
-      "memory": "$ramsize",
-      "disk": "41",
-      "arch": "x86_64",
-      "capabilities": "$capability"
-    },
+    cat >> /tmp/inventory-virt.yaml << EOF
+  node${i}:
+    mac_address: "$mac"
+    ipmi_ip: 192.168.122.1
+    ipmi_user: root
+    ipmi_pass: "INSERT_STACK_USER_PRIV_KEY"
+    pm_type: "pxe_ssh"
+    cpus: $vcpus
+    memory: $ramsize
+    disk: 41
+    arch: "x86_64"
+    capabilities: "$capability"
 EOF
   done
 
-  #truncate the last line to remove the comma behind the bracket
-  tail -n 1 $CONFIG/instackenv-virt.json | wc -c | xargs -I {} truncate $CONFIG/instackenv-virt.json -s -{}
-
-  #finally reclose the bracket and close the instackenv.json file
-  cat >> $CONFIG/instackenv-virt.json << EOF
-    }
-  ],
-  "arch": "x86_64",
-  "host-ip": "192.168.122.1",
-  "power_manager": "nova.virt.baremetal.virtual_power_driver.VirtualPowerManager",
-  "seed-ip": "",
-  "ssh-key": "INSERT_STACK_USER_PRIV_KEY",
-  "ssh-user": "root"
-}
-EOF
   #Overwrite the tripleo-inclubator domain.xml with our own, keeping a backup.
   if [ ! -f /usr/share/tripleo/templates/domain.xml.bak ]; then
     /usr/bin/mv -f /usr/share/tripleo/templates/domain.xml /usr/share/tripleo/templates/domain.xml.bak
diff --git a/tests/test_apex_inventory.py b/tests/test_apex_inventory.py
new file mode 100644
index 00000000..08a34152
--- /dev/null
+++ b/tests/test_apex_inventory.py
@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2016 Dan Radez (Red Hat)
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from apex.inventory import Inventory
+from apex.inventory import InventoryException
+
+from nose.tools import assert_is_instance
+from nose.tools import assert_raises
+from nose.tools import assert_equal
+
+inventory_files = ('intel_pod2_settings.yaml',
+                   'nokia_pod1_settings.yaml',
+                   'pod_example_settings.yaml')
+
+
+class TestInventory(object):
+    @classmethod
+    def setup_class(klass):
+        """This method is run once for each class before any tests are run"""
+
+    @classmethod
+    def teardown_class(klass):
+        """This method is run once for each class _after_ all tests are run"""
+
+    def setUp(self):
+        """This method is run once before _each_ test method is executed"""
+
+    def teardown(self):
+        """This method is run once after _each_ test method is executed"""
+
+    def test_init(self):
+        for f in inventory_files:
+            i = Inventory('../config/inventory/{}'.format(f))
+            assert_equal(i.dump_instackenv_json(), None)
+
+        # test virtual
+        i = Inventory(i, virtual=True)
+        assert_equal(i.dump_instackenv_json(), None)
+
+        # Remove nodes to violate HA node count
+        while len(i['nodes']) >= 5:
+            i['nodes'].pop()
+        assert_raises(InventoryException,
+                      Inventory, i)
+
+        # Remove nodes to violate non-HA node count
+        while len(i['nodes']) >= 2:
+            i['nodes'].pop()
+        assert_raises(InventoryException,
+                      Inventory, i, ha=False)
+
+    def test_exception(sefl):
+        e = InventoryException("test")
+        print(e)
+        assert_is_instance(e, InventoryException)
diff --git a/tests/test_apex_python_utils_py.py b/tests/test_apex_python_utils_py.py
index 1f280356..237c5589 100644
--- a/tests/test_apex_python_utils_py.py
+++ b/tests/test_apex_python_utils_py.py
@@ -16,6 +16,7 @@ from apex_python_utils import parse_net_settings
 from apex_python_utils import parse_deploy_settings
 from apex_python_utils import find_ip
 from apex_python_utils import build_nic_template
+from apex_python_utils import parse_inventory
 
 from nose.tools import assert_equal
 from nose.tools import assert_raises
@@ -25,6 +26,7 @@ net_sets = '../config/network/network_settings.yaml'
 net_env = '../build/network-environment.yaml'
 deploy_sets = '../config/deploy/deploy_settings.yaml'
 nic_template = '../build/nics-template.yaml.jinja2'
+inventory = '../config/inventory/pod_example_settings.yaml'
 
 
 class TestCommonUtils(object):
@@ -77,3 +79,8 @@ class TestCommonUtils(object):
                                        '-r', 'compute',
                                        '-t', nic_template])
         assert_equal(build_nic_template(args), None)
+
+    def test_parse_inventory(self):
+        args = self.parser.parse_args(['parse-inventory',
+                                       '-f', inventory])
+        assert_equal(parse_inventory(args), None)