Add compass-deck

RESTful API and DB Handlers for Compass

Change-Id: I1ce411f279943764c286ea48dca9185d453cf254
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

3 files changed, 414 insertions, 0 deletions

diff --git a/compass-deck/deployment/installers/pk_installers/__init__.py b/compass-deck/deployment/installers/pk_installers/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-deck/deployment/installers/pk_installers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-deck/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-deck/deployment/installers/pk_installers/ansible_installer/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-deck/deployment/installers/pk_installers/ansible_installer/__init__.py
diff --git a/compass-deck/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-deck/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
new file mode 100644
index 0000000..345f4e0
--- /dev/null
+++ b/compass-deck/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
@@ -0,0 +1,401 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__auther__ = "Compass Dev Team (dev-team@syscompass.org)"
+
+"""package installer: ansible plugin."""
+
+from Cheetah.Template import Template
+from copy import deepcopy
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+
+from compass.deployment.installers.installer import PKInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+NAME = "AnsibleInstaller"
+
+
+def byteify(input):
+    if isinstance(input, dict):
+        return dict([(byteify(key), byteify(value))
+                    for key, value in input.iteritems()])
+    elif isinstance(input, list):
+        return [byteify(element) for element in input]
+    elif isinstance(input, unicode):
+        return input.encode('utf-8')
+    else:
+        return input
+
+
+class AnsibleInstaller(PKInstaller):
+    INVENTORY_TMPL_DIR = 'inventories'
+    GROUPVARS_TMPL_DIR = 'vars'
+    INVENTORY_PATCH_TEMPALTE_DIR = 'inventories'
+
+    # keywords in package installer settings
+    ANSIBLE_DIR = 'ansible_dir'
+    ANSIBLE_RUN_DIR = 'ansible_run_dir'
+    LOG_FILE = 'ansible_log_file'
+    ANSIBLE_CONFIG = 'ansible_config'
+    INVENTORY = 'inventory_file'
+    GROUP_VARIABLE = 'group_variable'
+    HOSTS_PATH = 'etc_hosts_path'
+    RUNNER_DIRS = 'runner_dirs'
+
+    def __init__(self, config_manager):
+        super(AnsibleInstaller, self).__init__()
+
+        self.config_manager = config_manager
+        self.tmpl_name = self.config_manager.get_cluster_flavor_template()
+        self.installer_settings = (
+            self.config_manager.get_pk_installer_settings()
+        )
+        settings = self.installer_settings
+        self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None)
+        self.ansible_run_dir = (
+            settings.setdefault(self.ANSIBLE_RUN_DIR, None)
+        )
+        self.log_file = settings.setdefault(self.LOG_FILE, None)
+        self.ansible_config = (
+            settings.setdefault(self.ANSIBLE_CONFIG, None)
+        )
+        self.inventory = settings.setdefault(self.INVENTORY, None)
+        self.group_variable = (
+            settings.setdefault(self.GROUP_VARIABLE, None)
+        )
+        self.hosts_path = (
+            settings.setdefault(self.HOSTS_PATH, None)
+        )
+        self.runner_dirs = (
+            settings.setdefault(self.RUNNER_DIRS, None)
+        )
+        self.playbook = self.tmpl_name.replace('tmpl', 'yml')
+        self.runner_files = [self.playbook]
+
+        adapter_name = self.config_manager.get_dist_system_name()
+        self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name)
+        self.adapter_dir = os.path.join(self.ansible_dir, adapter_name)
+        logging.debug('%s instance created', self)
+
+    @classmethod
+    def get_tmpl_path(cls, adapter_name):
+        tmpl_path = os.path.join(
+            os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'),
+            adapter_name
+        )
+        return tmpl_path
+
+    def __repr__(self):
+        return '%s[name=%s,installer_url=%s]' % (
+            self.__class__.__name__, self.NAME, self.installer_url)
+
+    def generate_installer_config(self):
+        """Render ansible config file by OS installing.
+
+        The output format:
+        {
+           '1'($host_id/clusterhost_id):{
+               'tool': 'ansible',
+           },
+           .....
+        }
+        """
+        host_ids = self.config_manager.get_host_id_list()
+        os_installer_configs = {}
+        for host_id in host_ids:
+            temp = {
+                "tool": "ansible",
+            }
+            os_installer_configs[host_id] = temp
+
+        return os_installer_configs
+
+    def get_env_name(self, dist_sys_name, cluster_name):
+        return "-".join((dist_sys_name, cluster_name))
+
+    def _get_cluster_tmpl_vars(self):
+        """Generate template variables dict
+
+        Generates based on cluster level config.
+        The vars_dict will be:
+        {
+            "baseinfo": {
+                "id":1,
+                "name": "cluster01",
+                ...
+            },
+            "package_config": {
+                .... //mapped from original package config based on metadata
+            },
+            "role_mapping": {
+                ....
+            }
+        }
+        """
+        cluster_vars_dict = {}
+        # set cluster basic information to vars_dict
+        cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
+        cluster_vars_dict[const.BASEINFO] = cluster_baseinfo
+
+        # get and set template variables from cluster package config.
+        pk_metadata = self.config_manager.get_pk_config_meatadata()
+        pk_config = self.config_manager.get_cluster_package_config()
+
+        # get os config as ansible needs them
+        os_metadata = self.config_manager.get_os_config_metadata()
+        os_config = self.config_manager.get_cluster_os_config()
+
+        pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
+        os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config)
+        util.merge_dict(pk_meta_dict, os_meta_dict)
+
+        cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict
+
+        # get and set roles_mapping to vars_dict
+        mapping = self.config_manager.get_cluster_roles_mapping()
+        logging.info("cluster role mapping is %s", mapping)
+        cluster_vars_dict[const.ROLES_MAPPING] = mapping
+
+        # get ip settings to vars_dict
+        hosts_ip_settings = self.config_manager.get_hosts_ip_settings(
+            pk_meta_dict["network_cfg"]["ip_settings"],
+            pk_meta_dict["network_cfg"]["sys_intf_mappings"]
+        )
+        logging.info("hosts_ip_settings is %s", hosts_ip_settings)
+        cluster_vars_dict["ip_settings"] = hosts_ip_settings
+
+        return byteify(cluster_vars_dict)
+
+    def _generate_inventory_attributes(self, global_vars_dict):
+        inventory_tmpl_path = os.path.join(
+            os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR),
+            self.tmpl_name
+        )
+        if not os.path.exists(inventory_tmpl_path):
+            logging.error(
+                "Inventory template '%s' does not exist", self.tmpl_name
+            )
+            raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+        return self.get_config_from_template(
+            inventory_tmpl_path, global_vars_dict
+        )
+
+    def _generate_group_vars_attributes(self, global_vars_dict):
+        logging.info("global vars dict is %s", global_vars_dict)
+        group_vars_tmpl_path = os.path.join(
+            os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR),
+            self.tmpl_name
+        )
+        if not os.path.exists(group_vars_tmpl_path):
+            logging.error("Vars template '%s' does not exist",
+                          self.tmpl_name)
+            raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+        return self.get_config_from_template(
+            group_vars_tmpl_path, global_vars_dict
+        )
+
+    def _generate_hosts_attributes(self, global_vars_dict):
+        hosts_tmpl_path = os.path.join(
+            os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name
+        )
+        if not os.path.exists(hosts_tmpl_path):
+            logging.error("Hosts template '%s' does not exist", self.tmpl_name)
+            raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+        return self.get_config_from_template(hosts_tmpl_path, global_vars_dict)
+
+    def _generate_ansible_cfg_attributes(self, global_vars_dict):
+        ansible_cfg_tmpl_path = os.path.join(
+            os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name
+        )
+        if not os.path.exists(ansible_cfg_tmpl_path):
+            logging.error("cfg template '%s' does not exist", self.tmpl_name)
+            raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+        return self.get_config_from_template(
+            ansible_cfg_tmpl_path,
+            global_vars_dict
+        )
+
+    def get_config_from_template(self, tmpl_path, vars_dict):
+        logging.debug("vars_dict is %s", vars_dict)
+
+        if not os.path.exists(tmpl_path) or not vars_dict:
+            logging.info("Template dir or vars_dict is None!")
+            return {}
+
+        searchList = []
+        copy_vars_dict = deepcopy(vars_dict)
+        for key, value in vars_dict.iteritems():
+            if isinstance(value, dict):
+                temp = copy_vars_dict[key]
+                del copy_vars_dict[key]
+                searchList.append(temp)
+        searchList.append(copy_vars_dict)
+
+        # Load specific template for current adapter
+        tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList)
+        return tmpl.respond()
+
+    def _create_ansible_run_env(self, env_name, ansible_run_destination):
+        if os.path.exists(ansible_run_destination):
+            shutil.rmtree(ansible_run_destination, True)
+
+        os.mkdir(ansible_run_destination)
+
+        # copy roles to run env
+        dirs = self.runner_dirs
+        files = self.runner_files
+        for dir in dirs:
+            if not os.path.exists(os.path.join(self.ansible_dir, dir)):
+                continue
+            os.system(
+                "cp -rf %s %s" % (
+                    os.path.join(self.ansible_dir, dir),
+                    ansible_run_destination
+                )
+            )
+        for file in files:
+            logging.info('file is %s', file)
+            shutil.copy(
+                os.path.join(self.adapter_dir, file),
+                os.path.join(
+                    ansible_run_destination,
+                    file
+                )
+            )
+
+    def prepare_ansible(self, env_name, global_vars_dict):
+        ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+        if os.path.exists(ansible_run_destination):
+            ansible_run_destination += "-expansion"
+        self._create_ansible_run_env(env_name, ansible_run_destination)
+        inv_config = self._generate_inventory_attributes(global_vars_dict)
+        inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+
+        vars_config = self._generate_group_vars_attributes(global_vars_dict)
+        vars_dir = os.path.join(ansible_run_destination, 'group_vars')
+
+        hosts_config = self._generate_hosts_attributes(global_vars_dict)
+        hosts_destination = os.path.join(
+            ansible_run_destination, self.hosts_path
+        )
+
+        cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict)
+        cfg_destination = os.path.join(
+            ansible_run_destination,
+            self.ansible_config
+        )
+
+        os.mkdir(inventory_dir)
+        os.mkdir(vars_dir)
+
+        inventory_destination = os.path.join(inventory_dir, self.inventory)
+        group_vars_destination = os.path.join(vars_dir, self.group_variable)
+        self.serialize_config(inv_config, inventory_destination)
+        self.serialize_config(vars_config, group_vars_destination)
+        self.serialize_config(hosts_config, hosts_destination)
+        self.serialize_config(cfg_config, cfg_destination)
+
+    def deploy(self):
+        """Start to deploy a distributed system.
+
+        Return both cluster and hosts deployed configs.
+        The return format:
+        {
+            "cluster": {
+                "id": 1,
+                "deployed_package_config": {
+                    "roles_mapping": {...},
+                    "service_credentials": {...},
+                    ....
+                }
+            },
+            "hosts": {
+                1($clusterhost_id): {
+                    "deployed_package_config": {...}
+                },
+                ....
+            }
+        }
+        """
+        host_list = self.config_manager.get_host_id_list()
+        if not host_list:
+            return {}
+
+        adapter_name = self.config_manager.get_adapter_name()
+        cluster_name = self.config_manager.get_clustername()
+        env_name = self.get_env_name(adapter_name, cluster_name)
+
+        global_vars_dict = self._get_cluster_tmpl_vars()
+        logging.info(
+            '%s var dict: %s', self.__class__.__name__, global_vars_dict
+        )
+        # Create ansible related files
+        self.prepare_ansible(env_name, global_vars_dict)
+
+    def patch(self, patched_role_mapping):
+        adapter_name = self.config_manager.get_adapter_name()
+        cluster_name = self.config_manager.get_clustername()
+        env_name = self.get_env_name(adapter_name, cluster_name)
+        ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+        inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+        patched_global_vars_dict = self._get_cluster_tmpl_vars()
+        mapping = self.config_manager.get_cluster_patched_roles_mapping()
+        patched_global_vars_dict['roles_mapping'] = mapping
+        patched_inv = self._generate_inventory_attributes(
+            patched_global_vars_dict)
+        inv_file = os.path.join(inventory_dir, 'patched_inventory.yml')
+        self.serialize_config(patched_inv, inv_file)
+        config_file = os.path.join(
+            ansible_run_destination, self.ansible_config
+        )
+        playbook_file = os.path.join(ansible_run_destination, self.playbook)
+        log_file = os.path.join(ansible_run_destination, 'patch.log')
+        cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+                                                               inv_file,
+                                                               playbook_file)
+        with open(log_file, 'w') as logfile:
+            subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)
+        return patched_role_mapping
+
+    def cluster_os_ready(self):
+        adapter_name = self.config_manager.get_adapter_name()
+        cluster_name = self.config_manager.get_clustername()
+        env_name = self.get_env_name(adapter_name, cluster_name)
+        ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+        expansion_dir = ansible_run_destination + "-expansion"
+        if os.path.exists(expansion_dir):
+            ansible_run_destination = expansion_dir
+        inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+        inventory_file = os.path.join(inventory_dir, self.inventory)
+        playbook_file = os.path.join(ansible_run_destination, self.playbook)
+        log_file = os.path.join(ansible_run_destination, 'run.log')
+        config_file = os.path.join(
+            ansible_run_destination, self.ansible_config
+        )
+        cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+                                                               inventory_file,
+                                                               playbook_file)
+        with open(log_file, 'w') as logfile:
+            subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)