Add compass-docker-compose

1. Add ansible for docker compose containerized
compass
2. deploy/conf/compass.conf no longer specify
image tags for runtime. build.yaml now is added
into compass tarball to be the only place determine
image tags
3. This patch is based on patch:
https://gerrit.opnfv.org/gerrit/#/c/47393/
use merged compass-tasks
4. Tempoarily using private docker repo for
compass-tasks. Switch to opnfv repo when patch
https://gerrit.opnfv.org/gerrit/#/c/47329/
is merged
5. Modify ansible callback module accrodingly

Change-Id: I0728a9002a1cc7804c9e8f13bd1afdccb684a159
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

4 files changed, 50 insertions, 83 deletions

diff --git a/deploy/compass_vm.sh b/deploy/compass_vm.sh
index cc0471ae..7689f41c 100755
--- a/deploy/compass_vm.sh
+++ b/deploy/compass_vm.sh
@@ -27,11 +27,11 @@ function install_compass_core() {
 }
 
 function set_compass_machine() {
-    local config_file=$WORK_DIR/installer/compass-docker-compose/group_vars/all
+    local config_file=$WORK_DIR/installer/docker-compose/group_vars/all
     sed -i '/pxe_boot_macs/d' $config_file
     echo "pxe_boot_macs: [${machines}]" >> $config_file
 
-    ansible-playbook $WORK_DIR/installer/compass-docker-compose/add_machine.yml
+    ansible-playbook $WORK_DIR/installer/docker-compose/add_machine.yml
 }
 
 function install_compass() {
@@ -129,13 +129,9 @@ function wait_ok() {
 }
 
 function launch_compass() {
-    local group_vars=$WORK_DIR/installer/compass-docker-compose/group_vars/all
+    local group_vars=$WORK_DIR/installer/docker-compose/group_vars/all
     sed -i "s#^\(compass_dir:\).*#\1 $COMPASS_DIR#g" $group_vars
-    sed -i "s#^\(compass_deck:\).*#\1 $COMPASS_DECK#g" $group_vars
-    sed -i "s#^\(compass_tasks:\).*#\1 $COMPASS_TASKS#g" $group_vars
-    sed -i "s#^\(compass_cobbler:\).*#\1 $COMPASS_COBBLER#g" $group_vars
-    sed -i "s#^\(compass_db:\).*#\1 $COMPASS_DB#g" $group_vars
-    sed -i "s#^\(compass_mq:\).*#\1 $COMPASS_MQ#g" $group_vars
+    sed -i "s#^\(compose_images:\).*#\1 $COMPOSE_IMAGES#g" $group_vars
 
     if [[ $OFFLINE_DEPLOY == "Enable" ]]; then
         sed -i "s#.*\(compass_repo:\).*#\1 $COMPASS_REPO#g" $group_vars
@@ -150,7 +146,7 @@ function launch_compass() {
 
     sed -i "s#^\(deck_port:\).*#\1 $COMPASS_DECK_PORT#g" $group_vars
     sed -i "s#^\(repo_port:\).*#\1 $COMPASS_REPO_PORT#g" $group_vars
-    ansible-playbook $WORK_DIR/installer/compass-docker-compose/bring_up_compass.yml
+    ansible-playbook $WORK_DIR/installer/docker-compose/bring_up_compass.yml
 }
 
 function recover_compass() {
diff --git a/deploy/conf/compass.conf b/deploy/conf/compass.conf
index 78dd141b..9d9145f1 100644
--- a/deploy/conf/compass.conf
+++ b/deploy/conf/compass.conf
@@ -17,38 +17,4 @@ export NTP_SERVER="$COMPASS_SERVER"
 export NAMESERVERS=${USER_NAMESERVER:-"$COMPASS_SERVER"}
 export COMPASS_REPO_PORT="5151"
 export OFFLINE_DEPLOY=${OFFLINE_DEPLOY:-'Disable'}
-
-if [ "$COMPASS_ARCH" = "aarch64" ]; then
-  # Docker images for aarch64
-
-  # Arm images are temporarily held at dockerhub linaro repo
-  COMPASS_DOCKER_REPO=${COMPASS_DOCKER_REPO:-linaro}
-
-  export COMPASS_DECK="$COMPASS_DOCKER_REPO/compass-deck"
-  export COMPASS_TASKS="$COMPASS_DOCKER_REPO/compass-tasks-osa"
-
-  if [[ "x"$KUBERNETES_VERSION != "x" ]]; then
-    export COMPASS_TASKS="$COMPASS_DOCKER_REPO/compass-tasks-k8s"
-  fi
-
-  export COMPASS_DB="$COMPASS_DOCKER_REPO/compass-db"
-  export COMPASS_MQ="$COMPASS_DOCKER_REPO/compass-mq"
-  export COMPASS_REPO="$COMPASS_DOCKER_REPO/compass-repo-osa-ubuntu:euphrates"
-  export COMPASS_COBBLER="$COMPASS_DOCKER_REPO/compass-cobbler"
-
-else
-  # Docker images for x86_64
-
-  export COMPASS_DECK="compass4nfv/compass-deck"
-  export COMPASS_TASKS="yifei0van0xue/compass-tasks-osa"
-
-  if [[ "x"$KUBERNETES_VERSION != "x" ]]; then
-    export COMPASS_TASKS="compass4nfv/compass-tasks-k8s"
-  fi
-
-  export COMPASS_DB="compass4nfv/compass-db"
-  export COMPASS_MQ="compass4nfv/compass-mq"
-  export COMPASS_REPO="compass4nfv/compass-repo-osa-ubuntu:euphrates"
-  export COMPASS_COBBLER="yifei0van0xue/compass-cobbler"
-
-fi
+export COMPOSE_IMAGES="[compass-db,compass-mq,compass-deck,compass-tasks,compass-cobbler]"
diff --git a/deploy/playbook_done.py b/deploy/playbook_done.py
index 24c8c55b..6b1043d4 100644
--- a/deploy/playbook_done.py
+++ b/deploy/playbook_done.py
@@ -15,16 +15,15 @@
 # limitations under the License.
 
 """Ansible playbook callback after a playbook run has completed."""
-import os
 import sys
 
 from ansible.plugins.callback import CallbackBase
 
-current_dir = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(current_dir + '/..')
+compass_bin = "/opt/compass/bin"
+sys.path.append(compass_bin)
 
+import switch_virtualenv  # noqa: F401
 
-# import switch_virtualenv  # noqa
 from compass.apiclient.restful import Client  # noqa: E402
 from compass.utils import flags  # noqa: E402
 
diff --git a/deploy/status_callback.py b/deploy/status_callback.py
index 4bbbc321..f0615f3f 100644
--- a/deploy/status_callback.py
+++ b/deploy/status_callback.py
@@ -16,13 +16,13 @@ from ansible.plugins.callback import CallbackBase
 COMPASS_HOST = "compass-deck"
 
 
-def task_error(display, host, data):
-    display.display("task_error: host=%s,data=%s" % (host, data))
-
-#    if isinstance(data, dict):
-#        invocation = data.pop('invocation', {})
-
-    notify_host(display, COMPASS_HOST, host, "failed")
+# def task_error(display, host, data):
+#     display.display("task_error: host=%s,data=%s" % (host, data))
+#
+#     if isinstance(data, dict):
+#         invocation = data.pop('invocation', {})
+#
+#     notify_host(display, COMPASS_HOST, host, "failed")
 
 
 class CallbackModule(CallbackBase):
@@ -101,39 +101,47 @@ class CallbackModule(CallbackBase):
 
     def v2_playbook_on_stats(self, stats):
         self._display.display("playbook_on_stats enter")
+        all_vars = self.play.get_variable_manager().get_vars(self.loader)
+        host_vars = all_vars["hostvars"]
         hosts = sorted(stats.processed.keys())
-        failures = False
-        unreachable = False
-
-        for host in hosts:
-            summary = stats.summarize(host)
-            # self._display.display("host: %s \nsummary: %s\n" % (host, summary)) # noqa
-
-            if summary['failures'] > 0:
-                failures = True
-            if summary['unreachable'] > 0:
-                unreachable = True
+        cluster_name = host_vars[hosts[0]]['cluster_name']
 
         headers = {"Content-type": "application/json",
                    "Accept": "*/*"}
-
         conn = httplib.HTTPConnection(COMPASS_HOST, 80)
         token = auth(conn)
         headers["X-Auth-Token"] = token
-        get_url = "/api/hosts"
+        get_url = "/api/clusterhosts"
         conn.request("GET", get_url, "", headers)
         resp = conn.getresponse()
         raise_for_status(resp)
-        host_data = json.loads(resp.read())
-        clusterhosts = [item["name"] for item in host_data]
-
-        if failures or unreachable:
-            host_status = "error"
-        else:
-            host_status = "succ"
+        clusterhost_data = json.loads(resp.read())
+        clusterhost_mapping = {}
+        for item in clusterhost_data:
+            if item["clustername"] == cluster_name:
+                clusterhost_mapping.update({item["hostname"]:
+                                           item["clusterhost_id"]})
+
+        force_error = False
+        if "localhost" in hosts:
+            summary = stats.summarize("localhost")
+            if summary['failures'] > 0 or summary['unreachable'] > 0:
+                force_error = True
+
+        for hostname, hostid in clusterhost_mapping.iteritems():
+            if hostname not in hosts:
+                continue
+
+            summary = stats.summarize(hostname)
+            # self._display.display("host: %s \nsummary: %s\n" % (host, summary)) # noqa
 
-        for host in clusterhosts:
-            notify_host(self._display, "compass-deck", host, host_status)
+            if summary['failures'] > 0 or summary['unreachable'] > 0 \
+               or force_error:
+                status = "error"
+            else:
+                status = "succ"
+            self._display.display("hostname: %s" % hostname)
+            notify_host(self._display, COMPASS_HOST, hostid, status)
 
 
 def raise_for_status(resp):
@@ -157,17 +165,15 @@ def auth(conn):
     return json.loads(resp.read())["token"]
 
 
-def notify_host(display, compass_host, host, status):
-    display.display("hostname: %s" % host)
-    host = host.strip("host")
-    url = "/api/clusterhosts/%s/state" % host
+def notify_host(display, compass_host, hostid, status):
+    url = "/api/clusterhosts/%s/state" % hostid
     if status == "succ":
         body = {"state": "SUCCESSFUL"}
     elif status == "error":
         body = {"state": "ERROR"}
     else:
-        display.error("notify_host: host %s with status %s is not supported"
-                      % (host, status))
+        display.error("notify_host: hostid %s with status %s is not supported"
+                      % (hostid, status))
         return
 
     headers = {"Content-type": "application/json",