Dynamic Inventory

JIRA: COMPASS-556

1. using dynamic ansible inventory
2. modify Class AnsibleInstaller in compass-tasks
3. modify compass conf to support this behavior
4. specify docker image in /deploy/conf/compass.conf
5. remove clusterhost status update in playbook_done.py

Change-Id: I04079547c8b251571ae4e5b165d3bf425b8913b7
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

9 files changed, 130 insertions, 128 deletions

diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index 1e721aa6..3691a921 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -27,10 +27,11 @@
 - hosts: localhost
   remote_user: root
   roles:
-    - role: config-osa
-    - role: setup-host
-    - role: setup-infrastructure
-    - role: setup-openstack
-    - role: setup-openvswitch
-    - role: setup-opendaylight
-    - role: post-openstack
+    - config-osa
+    - setup-host
+    - setup-infrastructure
+    - setup-openstack
+    - sync-inventory
+    - setup-openvswitch
+    - setup-opendaylight
+    - post-openstack
diff --git a/deploy/adapters/ansible/roles/sync-inventory/tasks/main.yml b/deploy/adapters/ansible/roles/sync-inventory/tasks/main.yml
new file mode 100644
index 00000000..0db1c7cc
--- /dev/null
+++ b/deploy/adapters/ansible/roles/sync-inventory/tasks/main.yml
@@ -0,0 +1,15 @@
+# #############################################################################
+# # Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+# #
+# # All rights reserved. This program and the accompanying materials
+# # are made available under the terms of the Apache License, Version 2.0
+# # which accompanies this distribution, and is available at
+# # http://www.apache.org/licenses/LICENSE-2.0
+# # #############################################################################
+---
+- name: sync inventory
+  shell: >
+    "{{ run_dir }}"/inventories/inventory.py --merge
+    /etc/openstack_deploy/openstack_inventory.json
+
+- meta: refresh_inventory
diff --git a/deploy/compass_conf/package_installer/ansible-ocata.conf b/deploy/compass_conf/package_installer/ansible-ocata.conf
index 45e7cbe5..0286b4eb 100755
--- a/deploy/compass_conf/package_installer/ansible-ocata.conf
+++ b/deploy/compass_conf/package_installer/ansible-ocata.conf
@@ -5,7 +5,9 @@ SETTINGS = {
     'ansible_run_dir': '/var/ansible/run',
     'ansible_config': 'ansible.cfg',
     'playbook_file': 'site.yml',
-    'inventory_file': 'inventory.yml',
+    'inventory_file': 'inventory.py',
+    'inventory_json_file': 'inventory.json',
+    'inventory_group': ['controller', 'compute', 'ha', 'odl', 'onos', 'opencontrail', 'ceph_adm', 'ceph_mon', 'ceph_osd'],
     'group_variable': 'all',
     'etc_hosts_path': 'roles/pre-openstack/templates/hosts',
     'runner_dirs': ['roles','openstack_ocata/templates','openstack_ocata/roles']
diff --git a/deploy/compass_conf/templates/ansible_installer/openstack_ocata/inventories/HA-ansible-multinodes.tmpl b/deploy/compass_conf/templates/ansible_installer/openstack_ocata/inventories/HA-ansible-multinodes.tmpl
index 7cbd66d6..7184d21d 100755
--- a/deploy/compass_conf/templates/ansible_installer/openstack_ocata/inventories/HA-ansible-multinodes.tmpl
+++ b/deploy/compass_conf/templates/ansible_installer/openstack_ocata/inventories/HA-ansible-multinodes.tmpl
@@ -1,100 +1,74 @@
-#set controllers = $getVar('controller', [])
-#set computes = $getVar('compute', [])
-#set has = $getVar('ha', [])
-#set odls = $getVar('odl', [])
-#set onoss = $getVar('onos', [])
-#set opencontrails = $getVar('opencontrail', [])
-#set ceph_adm_list = $getVar('ceph_adm',[])
-#set ceph_mon_list = $getVar('ceph_mon',[])
-#set ceph_osd_list = $getVar('ceph_osd',[])
+#set inventory_json = $getVar('inventory_json', [])
+#!/usr/bin/env python
 
-#if not $isinstance($controllers, list)
-    #set controllers = [$controllers]
-#end if
-#if not $isinstance($computes, list)
-    #set computes = [$computes]
-#end if
-#if not $isinstance(has, list)
-    #set has = [has]
-#end if
-#if not $isinstance(odls, list)
-    #set odls = [odls]
-#end if
-#if not $isinstance(onoss, list)
-    #set onoss = [onoss]
-#end if
-#if not $isinstance(opencontrails, list)
-    #set opencontrails = [opencontrails]
-#end if
-#if not $isinstance(ceph_adm_list, list)
-    #set ceph_adm_list = [ceph_adm_list]
-#end if
-#if not $isinstance(ceph_mon_list, list)
-    #set ceph_mon_list = [ceph_mon_list]
-#end if
-#if not $isinstance(ceph_osd_list, list)
-    #set ceph_osd_list = [ceph_osd_list]
-#end if
+import os
+import sys
+import copy
+import argparse
 
-#set credentials = $getVar('server_credentials', {})
-#set username = $credentials.get('username', 'root')
-#set password = $credentials.get('password', 'root')
-[controller]
-#for controller in $controllers
-    #set controller_ip = $controller.install.ip
-    #set controller_hostname = $controller.hostname
-$controller_hostname ansible_ssh_host=$controller_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[compute]
-#for compute in $computes
-    #set compute_ip = $compute.install.ip
-    #set compute_hostname = $compute.hostname
-$compute_hostname ansible_ssh_host=$compute_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[ha]
-#for ha in $has
-    #set ha_ip = $ha.install.ip
-    #set ha_hostname = $ha.hostname
-$ha_hostname ansible_ssh_host=$ha_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[odl]
-#for odl in $odls
-    #set odl_ip = $odl.install.ip
-    #set odl_hostname = $odl.hostname
-$odl_hostname ansible_ssh_host=$odl_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[onos]
-#for onos in $onoss
-    #set onos_ip = $onos.install.ip
-    #set onos_hostname = $onos.hostname
-$onos_hostname ansible_ssh_host=$onos_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[opencontrail]
-#for opencontrail in $opencontrails
-    #set opencontrail_ip = $opencontrail.install.ip
-    #set opencontrail_hostname = $opencontrail.hostname
-$opencontrail_hostname ansible_ssh_host=$opencontrail_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[ceph_adm]
-#for ceph_adm in $ceph_adm_list
-    #set ceph_adm_ip = $ceph_adm.install.ip
-    #set ceph_adm_hostname = $ceph_adm.hostname
-$ceph_adm_hostname ansible_ssh_host=$ceph_adm_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[ceph_mon]
-#for ceph_mon in $ceph_mon_list
-    #set ceph_mon_ip = $ceph_mon.install.ip
-    #set ceph_mon_hostname = $ceph_mon.hostname
-$ceph_mon_hostname ansible_ssh_host=$ceph_mon_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[ceph_osd]
-#for ceph_osd in $ceph_osd_list
-    #set ceph_osd_ip = $ceph_osd.install.ip
-    #set ceph_osd_hostname = $ceph_osd.hostname
-$ceph_osd_hostname ansible_ssh_host=$ceph_osd_ip ansible_ssh_user=$username ansible_ssh_pass=$password
-#end for
-[ceph:children]
-ceph_adm
-ceph_mon
-ceph_osd
+try:
+    import json
+except ImportError:
+    import simplejson as json
 
+local_inventory='$inventory_json'
+
+def _byteify(data, ignore_dicts = False):
+    if isinstance(data, unicode):
+        return data.encode('utf-8')
+    if isinstance(data, list):
+        return [ _byteify(item, ignore_dicts=True) for item in data ]
+    if isinstance(data, dict) and not ignore_dicts:
+        return {
+            _byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True)
+            for key, value in data.iteritems()
+        }
+    return data
+
+def merge_dict(ldict, rdict, overwrite=True):
+    if not (ldict and rdict):
+        return
+
+    if not isinstance(ldict, dict):
+        raise TypeError('ldict type is %s not dict' % type(ldict))
+
+    if not isinstance(rdict, dict):
+        raise TypeError('rdict type is %s not dict' % type(rdict))
+
+    for key, value in rdict.items():
+        if isinstance(value, dict) and key in ldict and isinstance(ldict[key],
+                                                                 dict):
+            merge_dict(ldict[key], value, overwrite)
+        else:
+            if overwrite or key not in ldict:
+                ldict[key] = copy.deepcopy(value)
+
+def load_inventory(inventory):
+    if not os.path.exists(inventory):
+        raise RuntimeError('file: %s not exist' % inventory)
+    with open(inventory, 'r') as fd:
+        return json.load(fd, object_hook=_byteify)
+
+def dump_inventory(inventory, data):
+    with open(inventory, 'w') as fd:
+        json.dump(data, fd, indent=4)
+
+def merge_inventory(linv, rinv):
+    ldata = load_inventory(linv)
+    rdata = load_inventory(rinv)
+    merge_dict(ldata, rdata, overwrite=True)
+    dump_inventory(linv, ldata)
+
+def read_cli_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--list', action = 'store_true')
+    parser.add_argument('--merge', action = 'store')
+    return parser.parse_args()
+
+if __name__ == '__main__':
+    get_args = read_cli_args()
+    new_inventory = get_args.merge
+    if get_args.list:
+        print load_inventory(local_inventory)
+    elif new_inventory:
+        merge_inventory(local_inventory, new_inventory)
diff --git a/deploy/compass_conf/templates/ansible_installer/openstack_ocata/vars/HA-ansible-multinodes.tmpl b/deploy/compass_conf/templates/ansible_installer/openstack_ocata/vars/HA-ansible-multinodes.tmpl
index 7b0f4ab0..b7b6b91e 100755
--- a/deploy/compass_conf/templates/ansible_installer/openstack_ocata/vars/HA-ansible-multinodes.tmpl
+++ b/deploy/compass_conf/templates/ansible_installer/openstack_ocata/vars/HA-ansible-multinodes.tmpl
@@ -34,7 +34,6 @@ $res
     #set $host_info[$host['hostname']] = $info
     #set $inc = $inc + 1
 #end for
-
 host_info: $host_info
 
 #set ip_settings={}
@@ -51,6 +50,7 @@ host_info: $host_info
 
 #set has = $getVar('ha', [])
 #set ha_vip = $getVar('ha_vip', [])
+run_dir: $getVar('run_dir', '')
 
 enable_secgroup: $getVar('enable_secgroup', True)
 enable_fwaas: $getVar('enable_fwaas', True)
@@ -187,13 +187,6 @@ CONGRESS_PASS: $congress_pass
 DEMO_PASS: $demo_pass
 ADMIN_PASS: $admin_pass
 
-#set plugins = $getVar('plugins', [])
-#for item in plugins
-#set keys = $item.keys()
-#set values = $item.values()
-$keys[0]: $values[0]
-#end for
-
 #set neutron_service_plugins=['router']
 
 #if $getVar('enable_fwaas', True)
diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh
index 971db056..58173455 100755
--- a/deploy/compass_vm.sh
+++ b/deploy/compass_vm.sh
@@ -131,6 +131,11 @@ function wait_ok() {
 function launch_compass() {
     local group_vars=$WORK_DIR/installer/compass-docker-compose/group_vars/all
     sed -i "s#^\(compass_dir:\).*#\1 $COMPASS_DIR#g" $group_vars
+    sed -i "s#^\(compass_deck:\).*#\1 $COMPASS_DECK#g" $group_vars
+    sed -i "s#^\(compass_tasks:\).*#\1 $COMPASS_TASKS#g" $group_vars
+    sed -i "s#^\(compass_cobbler:\).*#\1 $COMPASS_COBBLER#g" $group_vars
+    sed -i "s#^\(compass_db:\).*#\1 $COMPASS_DB#g" $group_vars
+    sed -i "s#^\(compass_mq:\).*#\1 $COMPASS_MQ#g" $group_vars
 
     ansible-playbook $WORK_DIR/installer/compass-docker-compose/bring_up_compass.yml
 }
diff --git a/deploy/conf/compass.conf b/deploy/conf/compass.conf
index 2395965e..dafab732 100644
--- a/deploy/conf/compass.conf
+++ b/deploy/conf/compass.conf
@@ -14,3 +14,9 @@ export LANGUAGE="EN"
 export TIMEZONE="America/Los_Angeles"
 export NTP_SERVER="$COMPASS_SERVER"
 export NAMESERVERS="$COMPASS_SERVER"
+
+export COMPASS_DECK="compass4nfv/compass-deck"
+export COMPASS_TASKS="wtwde/compass-tasks-osa:v0.3"
+export COMPASS_COBBLER="compass4nfv/compass-cobbler"
+export COMPASS_DB="compass4nfv/compass-db"
+export COMPASS_MQ="compass4nfv/compass-mq"
diff --git a/deploy/playbook_done.py b/deploy/playbook_done.py
index ddb8e8d8..24c8c55b 100644
--- a/deploy/playbook_done.py
+++ b/deploy/playbook_done.py
@@ -105,11 +105,3 @@ class CallbackModule(CallbackBase):
 
         if failures or unreachable:
             return
-
-        self._login(self.client)
-
-        for host in hosts:
-            if host == "localhost":
-                continue
-            clusterhost_name = host + "." + cluster_name
-            self.client.clusterhost_ready(clusterhost_name)
diff --git a/deploy/status_callback.py b/deploy/status_callback.py
index 47df1d30..4bbbc321 100644
--- a/deploy/status_callback.py
+++ b/deploy/status_callback.py
@@ -13,6 +13,8 @@ import sys  # noqa:F401
 
 from ansible.plugins.callback import CallbackBase
 
+COMPASS_HOST = "compass-deck"
+
 
 def task_error(display, host, data):
     display.display("task_error: host=%s,data=%s" % (host, data))
@@ -20,7 +22,7 @@ def task_error(display, host, data):
 #    if isinstance(data, dict):
 #        invocation = data.pop('invocation', {})
 
-    notify_host(display, "compass-deck", host, "failed")
+    notify_host(display, COMPASS_HOST, host, "failed")
 
 
 class CallbackModule(CallbackBase):
@@ -42,7 +44,7 @@ class CallbackModule(CallbackBase):
         # task_error(self._display, host, res)
         pass
 
-    def v2_runner_on_ok(self, host, res):
+    def v2_runner_on_ok(self, res):
         pass
 
     def v2_runner_on_skipped(self, host, item=None):
@@ -112,14 +114,26 @@ class CallbackModule(CallbackBase):
             if summary['unreachable'] > 0:
                 unreachable = True
 
-        clusterhosts = set(hosts) - set(['localhost'])
+        headers = {"Content-type": "application/json",
+                   "Accept": "*/*"}
+
+        conn = httplib.HTTPConnection(COMPASS_HOST, 80)
+        token = auth(conn)
+        headers["X-Auth-Token"] = token
+        get_url = "/api/hosts"
+        conn.request("GET", get_url, "", headers)
+        resp = conn.getresponse()
+        raise_for_status(resp)
+        host_data = json.loads(resp.read())
+        clusterhosts = [item["name"] for item in host_data]
+
         if failures or unreachable:
-            for host in clusterhosts:
-                notify_host(self._display, "compass-deck", host, "error")
-            return
+            host_status = "error"
+        else:
+            host_status = "succ"
 
         for host in clusterhosts:
-            notify_host(self._display, "compass-deck", host, "succ")
+            notify_host(self._display, "compass-deck", host, host_status)
 
 
 def raise_for_status(resp):