summaryrefslogtreecommitdiffstats
path: root/compass-tasks/deployment/installers/pk_installers
diff options
context:
space:
mode:
Diffstat (limited to 'compass-tasks/deployment/installers/pk_installers')
-rw-r--r--compass-tasks/deployment/installers/pk_installers/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py0
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py441
3 files changed, 454 insertions, 0 deletions
diff --git a/compass-tasks/deployment/installers/pk_installers/__init__.py b/compass-tasks/deployment/installers/pk_installers/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py
diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
new file mode 100644
index 0000000..0a86be4
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
@@ -0,0 +1,441 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__auther__ = "Compass Dev Team (dev-team@syscompass.org)"
+
+"""package installer: ansible plugin."""
+
+from Cheetah.Template import Template
+from copy import deepcopy
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+
+from compass.deployment.installers.installer import PKInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+NAME = "AnsibleInstaller"
+
+
+def byteify(input):
+ if isinstance(input, dict):
+ return dict([(byteify(key), byteify(value))
+ for key, value in input.iteritems()])
+ elif isinstance(input, list):
+ return [byteify(element) for element in input]
+ elif isinstance(input, unicode):
+ return input.encode('utf-8')
+ else:
+ return input
+
+
+class AnsibleInstaller(PKInstaller):
+ INVENTORY_TMPL_DIR = 'inventories'
+ GROUPVARS_TMPL_DIR = 'vars'
+ INVENTORY_PATCH_TEMPALTE_DIR = 'inventories'
+
+ # keywords in package installer settings
+ ANSIBLE_DIR = 'ansible_dir'
+ ANSIBLE_RUN_DIR = 'ansible_run_dir'
+ LOG_FILE = 'ansible_log_file'
+ ANSIBLE_CONFIG = 'ansible_config'
+ INVENTORY = 'inventory_file'
+ INVENTORY_JSON = 'inventory_json_file'
+ INVENTORY_GROUP = 'inventory_group'
+ GROUP_VARIABLE = 'group_variable'
+ HOSTS_PATH = 'etc_hosts_path'
+ RUNNER_DIRS = 'runner_dirs'
+
+ def __init__(self, config_manager):
+ super(AnsibleInstaller, self).__init__()
+
+ self.config_manager = config_manager
+ self.tmpl_name = self.config_manager.get_cluster_flavor_template()
+ self.installer_settings = (
+ self.config_manager.get_pk_installer_settings()
+ )
+ settings = self.installer_settings
+ self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None)
+ self.ansible_run_dir = (
+ settings.setdefault(self.ANSIBLE_RUN_DIR, None)
+ )
+ self.log_file = settings.setdefault(self.LOG_FILE, None)
+ self.ansible_config = (
+ settings.setdefault(self.ANSIBLE_CONFIG, None)
+ )
+ self.inventory = settings.setdefault(self.INVENTORY, None)
+ self.inventory_json = settings.setdefault(self.INVENTORY_JSON, None)
+ self.inventory_group = settings.setdefault(self.INVENTORY_GROUP, None)
+ self.group_variable = (
+ settings.setdefault(self.GROUP_VARIABLE, None)
+ )
+ self.hosts_path = (
+ settings.setdefault(self.HOSTS_PATH, None)
+ )
+ self.runner_dirs = (
+ settings.setdefault(self.RUNNER_DIRS, None)
+ )
+ self.playbook = self.tmpl_name.replace('tmpl', 'yml')
+ self.runner_files = [self.playbook]
+
+ adapter_name = self.config_manager.get_dist_system_name()
+ self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name)
+ self.adapter_dir = os.path.join(self.ansible_dir, adapter_name)
+ logging.debug('%s instance created', self)
+
+ @classmethod
+ def get_tmpl_path(cls, adapter_name):
+ tmpl_path = os.path.join(
+ os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'),
+ adapter_name
+ )
+ return tmpl_path
+
+ def __repr__(self):
+ return '%s[name=%s,installer_url=%s]' % (
+ self.__class__.__name__, self.NAME, self.installer_url)
+
+ def dump_inventory(self, data, inventory):
+ with open(inventory, "w") as f:
+ json.dump(data, f, indent=4)
+
+ def _generate_inventory_data(self, global_vars_dict):
+ vars_dict = global_vars_dict['roles_mapping']
+ inventory_data = {}
+ inventory_data['_meta'] = {'hostvars': {}}
+ for item in self.inventory_group:
+ if item in vars_dict:
+ inventory_data[item] = {'hosts': []}
+ for host in vars_dict[item]:
+ hostname = host['hostname']
+ if hostname not in inventory_data['_meta']['hostvars']:
+ host_dict = {}
+ host_dict['ansible_ssh_host'] = host['install']['ip']
+ host_dict['ansible_ssh_user'] = 'root'
+ host_dict['ansible_ssh_pass'] = 'root'
+ inventory_data['_meta']['hostvars'].update(
+ {hostname: host_dict})
+ inventory_data[item]['hosts'].append(hostname)
+
+ inventory_data['ceph'] = {'children':
+ ['ceph_adm', 'ceph_mon', 'ceph_osd']}
+ return inventory_data
+
+ def generate_installer_config(self):
+ """Render ansible config file by OS installing.
+
+ The output format:
+ {
+ '1'($host_id/clusterhost_id):{
+ 'tool': 'ansible',
+ },
+ .....
+ }
+ """
+ host_ids = self.config_manager.get_host_id_list()
+ os_installer_configs = {}
+ for host_id in host_ids:
+ temp = {
+ "tool": "ansible",
+ }
+ os_installer_configs[host_id] = temp
+
+ return os_installer_configs
+
+ def get_env_name(self, dist_sys_name, cluster_name):
+ return "-".join((dist_sys_name, cluster_name))
+
+ def _get_cluster_tmpl_vars(self):
+ """Generate template variables dict
+
+ Generates based on cluster level config.
+ The vars_dict will be:
+ {
+ "baseinfo": {
+ "id":1,
+ "name": "cluster01",
+ ...
+ },
+ "package_config": {
+ .... //mapped from original package config based on metadata
+ },
+ "role_mapping": {
+ ....
+ }
+ }
+ """
+ cluster_vars_dict = {}
+ # set cluster basic information to vars_dict
+ cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
+ cluster_vars_dict[const.BASEINFO] = cluster_baseinfo
+
+ # get and set template variables from cluster package config.
+ pk_metadata = self.config_manager.get_pk_config_meatadata()
+ pk_config = self.config_manager.get_cluster_package_config()
+
+ # get os config as ansible needs them
+ os_metadata = self.config_manager.get_os_config_metadata()
+ os_config = self.config_manager.get_cluster_os_config()
+
+ pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
+ os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config)
+ util.merge_dict(pk_meta_dict, os_meta_dict)
+
+ cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict
+
+ # get and set roles_mapping to vars_dict
+ mapping = self.config_manager.get_cluster_roles_mapping()
+ logging.info("cluster role mapping is %s", mapping)
+ cluster_vars_dict[const.ROLES_MAPPING] = mapping
+
+ # get ip settings to vars_dict
+ hosts_ip_settings = self.config_manager.get_hosts_ip_settings(
+ pk_meta_dict["network_cfg"]["ip_settings"],
+ pk_meta_dict["network_cfg"]["sys_intf_mappings"]
+ )
+ logging.info("hosts_ip_settings is %s", hosts_ip_settings)
+ cluster_vars_dict["ip_settings"] = hosts_ip_settings
+
+ return byteify(cluster_vars_dict)
+
+ def _generate_inventory_attributes(self, global_vars_dict):
+ inventory_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR),
+ self.tmpl_name
+ )
+ if not os.path.exists(inventory_tmpl_path):
+ logging.error(
+ "Inventory template '%s' does not exist", self.tmpl_name
+ )
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+ inventory_dir = os.path.join(global_vars_dict['run_dir'], 'inventories')
+ inventory_json = os.path.join(inventory_dir, self.inventory_json)
+ vars_dict = {'inventory_json': inventory_json}
+ return self.get_config_from_template(
+ inventory_tmpl_path, vars_dict
+ )
+
+ def _generate_group_vars_attributes(self, global_vars_dict):
+ logging.info("global vars dict is %s", global_vars_dict)
+ group_vars_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR),
+ self.tmpl_name
+ )
+ if not os.path.exists(group_vars_tmpl_path):
+ logging.error("Vars template '%s' does not exist",
+ self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(
+ group_vars_tmpl_path, global_vars_dict
+ )
+
+ def _generate_hosts_attributes(self, global_vars_dict):
+ hosts_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name
+ )
+ if not os.path.exists(hosts_tmpl_path):
+ logging.error("Hosts template '%s' does not exist", self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(hosts_tmpl_path, global_vars_dict)
+
+ def _generate_ansible_cfg_attributes(self, global_vars_dict):
+ ansible_cfg_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name
+ )
+ if not os.path.exists(ansible_cfg_tmpl_path):
+ logging.error("cfg template '%s' does not exist", self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(
+ ansible_cfg_tmpl_path,
+ global_vars_dict
+ )
+
+ def get_config_from_template(self, tmpl_path, vars_dict):
+ logging.debug("vars_dict is %s", vars_dict)
+
+ if not os.path.exists(tmpl_path) or not vars_dict:
+ logging.info("Template dir or vars_dict is None!")
+ return {}
+
+ searchList = []
+ copy_vars_dict = deepcopy(vars_dict)
+ for key, value in vars_dict.iteritems():
+ if isinstance(value, dict):
+ temp = copy_vars_dict[key]
+ del copy_vars_dict[key]
+ searchList.append(temp)
+ searchList.append(copy_vars_dict)
+
+ # Load specific template for current adapter
+ tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList)
+ return tmpl.respond()
+
+ def _create_ansible_run_env(self, env_name, ansible_run_destination):
+ if os.path.exists(ansible_run_destination):
+ shutil.rmtree(ansible_run_destination, True)
+
+ os.mkdir(ansible_run_destination)
+
+ # copy roles to run env
+ dirs = self.runner_dirs
+ files = self.runner_files
+ for dir in dirs:
+ if not os.path.exists(os.path.join(self.ansible_dir, dir)):
+ continue
+ os.system(
+ "cp -rf %s %s" % (
+ os.path.join(self.ansible_dir, dir),
+ ansible_run_destination
+ )
+ )
+ for file in files:
+ logging.info('file is %s', file)
+ shutil.copy(
+ os.path.join(self.adapter_dir, file),
+ os.path.join(
+ ansible_run_destination,
+ file
+ )
+ )
+
+ def prepare_ansible(self, env_name, global_vars_dict):
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ if os.path.exists(ansible_run_destination):
+ ansible_run_destination += "-expansion"
+ self._create_ansible_run_env(env_name, ansible_run_destination)
+ global_vars_dict.update({'run_dir': ansible_run_destination})
+
+ inv_config = self._generate_inventory_attributes(global_vars_dict)
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+
+ vars_config = self._generate_group_vars_attributes(global_vars_dict)
+ vars_dir = os.path.join(ansible_run_destination, 'group_vars')
+
+ hosts_config = self._generate_hosts_attributes(global_vars_dict)
+ hosts_destination = os.path.join(
+ ansible_run_destination, self.hosts_path
+ )
+
+ cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict)
+ cfg_destination = os.path.join(
+ ansible_run_destination,
+ self.ansible_config
+ )
+
+ inventory_data = self._generate_inventory_data(global_vars_dict)
+ inventory_json_destination = os.path.join(inventory_dir,
+ self.inventory_json)
+
+ os.mkdir(inventory_dir)
+ os.mkdir(vars_dir)
+
+ inventory_destination = os.path.join(inventory_dir, self.inventory)
+ group_vars_destination = os.path.join(vars_dir, self.group_variable)
+ self.dump_inventory(inventory_data, inventory_json_destination)
+ self.serialize_config(inv_config, inventory_destination)
+ self.serialize_config(vars_config, group_vars_destination)
+ self.serialize_config(hosts_config, hosts_destination)
+ self.serialize_config(cfg_config, cfg_destination)
+
+ def deploy(self):
+ """Start to deploy a distributed system.
+
+ Return both cluster and hosts deployed configs.
+ The return format:
+ {
+ "cluster": {
+ "id": 1,
+ "deployed_package_config": {
+ "roles_mapping": {...},
+ "service_credentials": {...},
+ ....
+ }
+ },
+ "hosts": {
+ 1($clusterhost_id): {
+ "deployed_package_config": {...}
+ },
+ ....
+ }
+ }
+ """
+ host_list = self.config_manager.get_host_id_list()
+ if not host_list:
+ return {}
+
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+
+ global_vars_dict = self._get_cluster_tmpl_vars()
+ logging.info(
+ '%s var dict: %s', self.__class__.__name__, global_vars_dict
+ )
+ # Create ansible related files
+ self.prepare_ansible(env_name, global_vars_dict)
+
+ def patch(self, patched_role_mapping):
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+ patched_global_vars_dict = self._get_cluster_tmpl_vars()
+ mapping = self.config_manager.get_cluster_patched_roles_mapping()
+ patched_global_vars_dict['roles_mapping'] = mapping
+ patched_inv = self._generate_inventory_attributes(
+ patched_global_vars_dict)
+ inv_file = os.path.join(inventory_dir, 'patched_inventory.yml')
+ self.serialize_config(patched_inv, inv_file)
+ config_file = os.path.join(
+ ansible_run_destination, self.ansible_config
+ )
+ playbook_file = os.path.join(ansible_run_destination, self.playbook)
+ log_file = os.path.join(ansible_run_destination, 'patch.log')
+ cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+ inv_file,
+ playbook_file)
+ with open(log_file, 'w') as logfile:
+ subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)
+ return patched_role_mapping
+
+ def cluster_os_ready(self):
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ expansion_dir = ansible_run_destination + "-expansion"
+ if os.path.exists(expansion_dir):
+ ansible_run_destination = expansion_dir
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+ inventory_file = os.path.join(inventory_dir, self.inventory)
+ playbook_file = os.path.join(ansible_run_destination, self.playbook)
+ log_file = os.path.join(ansible_run_destination, 'run.log')
+ config_file = os.path.join(
+ ansible_run_destination, self.ansible_config
+ )
+ os.system("chmod +x %s" % inventory_file)
+ cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+ inventory_file,
+ playbook_file)
+ with open(log_file, 'w') as logfile:
+ subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)