summaryrefslogtreecommitdiffstats
path: root/compass-tasks/deployment/installers
diff options
context:
space:
mode:
Diffstat (limited to 'compass-tasks/deployment/installers')
-rw-r--r--compass-tasks/deployment/installers/__init__.py21
-rw-r--r--compass-tasks/deployment/installers/config_manager.py527
-rw-r--r--compass-tasks/deployment/installers/installer.py291
-rw-r--r--compass-tasks/deployment/installers/os_installers/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/os_installers/cobbler/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py449
-rw-r--r--compass-tasks/deployment/installers/pk_installers/__init__.py13
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py0
-rw-r--r--compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py441
9 files changed, 1768 insertions, 0 deletions
diff --git a/compass-tasks/deployment/installers/__init__.py b/compass-tasks/deployment/installers/__init__.py
new file mode 100644
index 0000000..0296be5
--- /dev/null
+++ b/compass-tasks/deployment/installers/__init__.py
@@ -0,0 +1,21 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""modules to read/write cluster/host config from installers.
+
+ .. moduleauthor:: Grace Yu <grace.yu@huawei.com>
+"""
diff --git a/compass-tasks/deployment/installers/config_manager.py b/compass-tasks/deployment/installers/config_manager.py
new file mode 100644
index 0000000..597c3a6
--- /dev/null
+++ b/compass-tasks/deployment/installers/config_manager.py
@@ -0,0 +1,527 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "baigk baiguoku@huawei.com)"
+
+from collections import defaultdict
+from copy import deepcopy
+import json
+import logging
+import netaddr
+
+from compass.deployment.utils import constants as const
+
+ip_generator_map = {}
+
+
+def get_ip_addr(ip_ranges):
+ def _get_ip_addr():
+ for ip_range in ip_ranges:
+ for ip in netaddr.iter_iprange(*ip_range):
+ yield str(ip)
+
+ s = json.dumps(ip_ranges)
+ if s not in ip_generator_map:
+ ip_generator_map[s] = _get_ip_addr()
+ return ip_generator_map[s]
+ else:
+ return ip_generator_map[s]
+
+
+class AdapterInfo(object):
+ def __init__(self, adapter_info):
+ self.adapter_info = adapter_info
+ self.name = self.adapter_info.get(const.NAME)
+ self.dist_system_name = self.name
+ self.health_check_cmd = self.adapter_info.get(const.HEALTH_CHECK_CMD)
+
+ self.os_installer = self.adapter_info.setdefault(
+ const.OS_INSTALLER, {}
+ )
+ self.os_installer.setdefault(const.INSTALLER_SETTINGS, {})
+
+ self.package_installer = self.adapter_info.setdefault(
+ const.PK_INSTALLER, {}
+ )
+ self.package_installer.setdefault(const.INSTALLER_SETTINGS, {})
+
+ self.metadata = self.adapter_info.setdefault(const.METADATA, {})
+ self.os_metadata = self.metadata.setdefault(const.OS_CONFIG, {})
+ self.package_metadata = self.metadata.setdefault(const.PK_CONFIG, {})
+
+ self.flavors = dict([(f[const.FLAVOR_NAME], f)
+ for f in self.adapter_info.get(const.FLAVOR, [])])
+
+ @property
+ def flavor_list(self):
+ return self.flavors.values()
+
+ def get_flavor(self, flavor_name):
+ return self.flavors.get(flavor_name)
+
+
+class ClusterInfo(object):
+ def __init__(self, cluster_info):
+ self.cluster_info = cluster_info
+ self.id = self.cluster_info.get(const.ID)
+ self.name = self.cluster_info.get(const.NAME)
+ self.os_version = self.cluster_info.get(const.OS_VERSION)
+ self.flavor = self.cluster_info.setdefault(
+ const.FLAVOR, {}
+ )
+ self.os_config = self.cluster_info.setdefault(
+ const.OS_CONFIG, {}
+ )
+ self.package_config = self.cluster_info.setdefault(
+ const.PK_CONFIG, {}
+ )
+ self.deployed_os_config = self.cluster_info.setdefault(
+ const.DEPLOYED_OS_CONFIG, {}
+ )
+ self.deployed_package_config = self.cluster_info.setdefault(
+ const.DEPLOYED_PK_CONFIG, {}
+ )
+ self.network_mapping = self.package_config.setdefault(
+ const.NETWORK_MAPPING, {}
+ )
+
+ os_config_general = self.os_config.setdefault(
+ const.OS_CONFIG_GENERAL, {}
+ )
+ self.domain = os_config_general.setdefault(const.DOMAIN, None)
+ self.hosts = []
+
+ def add_host(self, host):
+ self.hosts.append(host)
+
+ @property
+ def roles_mapping(self):
+ deploy_config = self.deployed_package_config
+ return deploy_config.setdefault(
+ const.ROLES_MAPPING, self._get_cluster_roles_mapping()
+ )
+
+ def _get_cluster_roles_mapping(self):
+ """The ouput format will be as below, for example:
+
+ {
+ "controller": [{
+ "hostname": "xxx",
+ "management": {
+ "interface": "eth0",
+ "ip": "192.168.1.10",
+ "netmask": "255.255.255.0",
+ "subnet": "192.168.1.0/24",
+ "is_mgmt": True,
+ "is_promiscuous": False
+ },
+ ...
+ }],
+ ...
+ }
+ """
+ mapping = defaultdict(list)
+ for host in self.hosts:
+ for role, value in host.roles_mapping.iteritems():
+ mapping[role].append(value)
+
+ return dict(mapping)
+
+ def _get_cluster_patched_roles_mapping(self):
+ mapping = defaultdict(list)
+ for host in self.hosts:
+ for role, value in host.patched_roles_mapping.iteritems():
+ mapping[role].append(value)
+
+ return dict(mapping)
+
+ @property
+ def base_info(self):
+ return {
+ const.ID: self.id,
+ const.NAME: self.name,
+ const.OS_VERSION: self.os_version
+ }
+
+
+class HostInfo(object):
+ def __init__(self, host_info, cluster_info):
+ self.host_info = host_info
+ self.cluster_info = cluster_info
+ self.id = self.host_info.get(const.ID)
+ self.name = self.host_info.get(const.NAME)
+ self.mac = self.host_info.get(const.MAC_ADDR)
+ self.hostname = self.host_info.get(const.HOSTNAME)
+ self.networks = self.host_info.setdefault(const.NETWORKS, {})
+ self.os_config = self.host_info.setdefault(const.OS_CONFIG, {})
+
+ self.package_config = self.host_info.setdefault(const.PK_CONFIG, {})
+ self.roles = self.host_info.setdefault(const.ROLES, [])
+ self.patched_roles = self.host_info.setdefault(const.PATCHED_ROLES, [])
+ self.ipmi = deepcopy(self.host_info.setdefault(const.IPMI, {}))
+ self.reinstall_os_flag = self.host_info.get(const.REINSTALL_OS_FLAG)
+ self.deployed_os_config = self.host_info.setdefault(
+ const.DEPLOYED_OS_CONFIG, {}
+ )
+ self.deployed_package_config = self.host_info.setdefault(
+ const.DEPLOYED_PK_CONFIG, {}
+ )
+
+ os_general_config = self.os_config.setdefault(
+ const.OS_CONFIG_GENERAL, {}
+ )
+ domain = os_general_config.setdefault(const.DOMAIN, None)
+ if domain is None:
+ self.domain = self.cluster_info.domain
+ else:
+ self.domain = domain
+
+ if const.DNS in host_info:
+ self.dns = host_info[const.DNS]
+ else:
+ self.dns = '.'.join((self.hostname, self.domain))
+
+ if const.NETWORK_MAPPING not in self.package_config:
+ self.network_mapping = self.cluster_info.network_mapping
+ else:
+ self.network_mapping = self.package_config[const.NETWORK_MAPPING]
+
+ if const.ROLES_MAPPING not in self.deployed_package_config:
+ self.roles_mapping = self._get_host_roles_mapping()
+ self.deployed_package_config[
+ const.ROLES_MAPPING
+ ] = self.roles_mapping
+ else:
+ self.roles_mapping = \
+ self.deployed_package_config[const.ROLES_MAPPING]
+
+ self.patched_roles_mapping = self._get_host_patched_roles_mapping()
+
+ self.cluster_info.add_host(self)
+
+ def valid_interface(self, interface):
+ if interface not in self.networks:
+ raise RuntimeError("interface %s is invalid" % interface)
+
+ def get_interface(self, interface):
+ self.valid_interface(interface)
+ return self.networks[interface]
+
+ def get_interface_ip(self, interface):
+ return self.get_interface(interface).get(const.IP_ADDR)
+
+ def get_interface_netmask(self, interface):
+ return self.get_interface(interface).get(const.NETMASK)
+
+ def get_interface_subnet(self, interface):
+ return self.get_interface(interface).get(const.SUBNET)
+
+ def is_interface_promiscuous(self, interface):
+ return self.get_interface(interface).get(const.PROMISCUOUS_FLAG)
+
+ def is_interface_mgmt(self, interface):
+ return self.get_interface(interface).get(const.MGMT_NIC_FLAG)
+
+ def _get_host_roles_mapping(self):
+ if not self.network_mapping:
+ return {}
+
+ net_info = {const.HOSTNAME: self.hostname}
+ for k, v in self.network_mapping.items():
+ try:
+ net_info[k] = self.networks[v[const.NIC]]
+ net_info[k][const.NIC] = v[const.NIC]
+ except Exception:
+ pass
+
+ mapping = {}
+ for role in self.roles:
+ role = role.replace("-", "_")
+ mapping[role] = net_info
+
+ return mapping
+
+ def _get_host_patched_roles_mapping(self):
+ if not self.network_mapping:
+ return {}
+
+ net_info = {const.HOSTNAME: self.hostname}
+ for k, v in self.network_mapping.items():
+ try:
+ net_info[k] = self.networks[v[const.NIC]]
+ net_info[k][const.NIC] = v[const.NIC]
+ except Exception:
+ pass
+
+ mapping = {}
+ for role in self.patched_roles:
+ role = role['name'].replace("-", "_")
+ mapping[role] = net_info
+
+ return mapping
+
+ @property
+ def baseinfo(self):
+ return {
+ const.REINSTALL_OS_FLAG: self.reinstall_os_flag,
+ const.MAC_ADDR: self.mac,
+ const.NAME: self.name,
+ const.HOSTNAME: self.hostname,
+ const.DNS: self.dns,
+ const.NETWORKS: deepcopy(self.networks)
+ }
+
+
+class BaseConfigManager(object):
+ def __init__(self, adapter_info={}, cluster_info={}, hosts_info={}):
+ assert(adapter_info and isinstance(adapter_info, dict))
+ assert(cluster_info and isinstance(cluster_info, dict))
+ assert(hosts_info and isinstance(hosts_info, dict))
+
+ self.adapter_info = AdapterInfo(adapter_info)
+ self.cluster_info = ClusterInfo(cluster_info)
+ self.hosts_info = dict([(k, HostInfo(v, self.cluster_info))
+ for k, v in hosts_info.iteritems()])
+
+ def get_adapter_name(self):
+ return self.adapter_info.name
+
+ def get_dist_system_name(self):
+ return self.adapter_info.dist_system_name
+
+ def get_adapter_health_check_cmd(self):
+ return self.adapter_info.health_check_cmd
+
+ def get_os_installer_settings(self):
+ return self.adapter_info.os_installer[const.INSTALLER_SETTINGS]
+
+ def get_pk_installer_settings(self):
+ return self.adapter_info.package_installer[const.INSTALLER_SETTINGS]
+
+ def get_os_config_metadata(self):
+ return self.adapter_info.metadata[const.OS_CONFIG]
+
+ def get_pk_config_meatadata(self):
+ return self.adapter_info.metadata[const.PK_CONFIG]
+
+ def get_adapter_all_flavors(self):
+ return self.adapter_info.flavor_list
+
+ def get_adapter_flavor(self, flavor_name):
+ return self.adapter_info.get_flavor(flavor_name)
+
+ def get_cluster_id(self):
+ return self.cluster_info.id
+
+ def get_clustername(self):
+ return self.cluster_info.name
+
+ def get_os_version(self):
+ return self.cluster_info.os_version
+
+ def get_cluster_os_config(self):
+ return self.cluster_info.os_config
+
+ def get_cluster_baseinfo(self):
+ return self.cluster_info.base_info
+
+ def get_cluster_flavor_name(self):
+ return self.cluster_info.flavor.get(const.FLAVOR_NAME)
+
+ def get_cluster_flavor_roles(self):
+ return self.cluster_info.flavor.get(const.ROLES, [])
+
+ def get_cluster_flavor_template(self):
+ return self.cluster_info.flavor.get(const.TMPL)
+
+ def get_cluster_package_config(self):
+ return self.cluster_info.package_config
+
+ def get_cluster_network_mapping(self):
+ mapping = self.cluster_info.network_mapping
+ logging.info("Network mapping in the config is '%s'!", mapping)
+ return mapping
+
+ def get_cluster_deployed_os_config(self):
+ return self.cluster_info.deployed_os_config
+
+ def get_cluster_deployed_package_config(self):
+ return self.cluster_info.deployed_package_config
+
+ def get_cluster_roles_mapping(self):
+ return self.cluster_info.roles_mapping
+
+ def get_cluster_patched_roles_mapping(self):
+ return self.cluster_info._get_cluster_patched_roles_mapping()
+
+ def validate_host(self, host_id):
+ if host_id not in self.hosts_info:
+ raise RuntimeError("host_id %s is invalid" % host_id)
+
+ def get_host_id_list(self):
+ return self.hosts_info.keys()
+
+ def get_hosts_id_list_for_os_installation(self):
+ """Get info of hosts which need to install/reinstall OS."""
+ return [
+ id for id, info in self.hosts_info.items()
+ if info.reinstall_os_flag
+ ]
+
+ def get_server_credentials(self):
+ cluster_os_config = self.get_cluster_os_config()
+ if not cluster_os_config:
+ logging.info("cluster os_config is None!")
+ return ()
+
+ username = cluster_os_config[const.SERVER_CREDS][const.USERNAME]
+ password = cluster_os_config[const.SERVER_CREDS][const.PASSWORD]
+ return (username, password)
+
+ def _get_host_info(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id]
+
+ def get_host_baseinfo(self, host_id):
+ self.validate_host(host_id)
+ host_info = self.hosts_info[host_id]
+ return host_info.baseinfo
+
+ def get_host_fullname(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].name
+
+ def get_host_dns(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].dns
+
+ def get_host_mac_address(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].mac
+
+ def get_hostname(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].hostname
+
+ def get_host_networks(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].networks
+
+ def get_host_interfaces(self, host_id):
+ # get interface names
+ return self.get_host_networks(host_id).keys()
+
+ def get_host_interface_ip(self, host_id, interface):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].get_interface_ip(interface)
+
+ def get_host_interface_netmask(self, host_id, interface):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].get_interface_netmask(interface)
+
+ def get_host_interface_subnet(self, host_id, interface):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].get_interface_subnet(interface)
+
+ def is_interface_promiscuous(self, host_id, interface):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].is_interface_promiscuous(interface)
+
+ def is_interface_mgmt(self, host_id, interface):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].is_interface_mgmt(interface)
+
+ def get_host_os_config(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].os_config
+
+ def get_host_domain(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].domain
+
+ def get_host_network_mapping(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].network_mapping
+
+ def get_host_package_config(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].package_config
+
+ def get_host_deployed_os_config(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].deployed_os_config
+
+ def get_host_deployed_package_config(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].deployed_package_config
+
+ def get_host_roles(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].roles
+
+ def get_all_hosts_roles(self, hosts_id_list=None):
+ roles = []
+ for host_id, host_info in self.hosts_info.iteritems():
+ roles.extend(host_info.roles)
+
+ return list(set(roles))
+
+ def get_hosts_ip_settings(self, ip_settings, sys_intf_mappings):
+ logging.info(
+ "get_hosts_ip_settings:ip_settings=%s, sys_intf_mappings=%s" %
+ (ip_settings, sys_intf_mappings)
+ )
+
+ intf_alias = {}
+ for m in sys_intf_mappings:
+ if "vlan_tag" in m:
+ intf_alias[m["name"]] = m["name"]
+ else:
+ intf_alias[m["name"]] = m["interface"]
+
+ mappings = {}
+ hosts_id_list = self.get_host_id_list()
+ for host_id in hosts_id_list:
+ hostname = self.get_hostname(host_id)
+ mappings[hostname] = []
+ for ip_info in ip_settings:
+ logging.info("ip_info=%s" % ip_info)
+ new_ip_info = deepcopy(ip_info)
+ del new_ip_info["ip_ranges"]
+
+ ip_ranges = ip_info["ip_ranges"]
+ new_ip_info["netmask"] = netaddr.IPNetwork(
+ ip_info["cidr"]
+ ).netmask.bin.count("1")
+ new_ip_info["ip"] = get_ip_addr(ip_ranges).next()
+ new_ip_info["alias"] = intf_alias[ip_info["name"]]
+ mappings[hostname].append(new_ip_info)
+
+ return {"ip_settings": mappings}
+
+ def get_host_roles_mapping(self, host_id):
+ self.validate_host(host_id)
+ return self.hosts_info[host_id].roles_mapping
+
+ def get_host_ipmi_info(self, host_id):
+ self.validate_host(host_id)
+ if self.hosts_info[host_id].ipmi:
+ return (
+ self.hosts_info[host_id].ipmi[const.IP_ADDR],
+ self.hosts_info[host_id].ipmi
+ [const.IPMI_CREDS][const.USERNAME],
+ self.hosts_info[host_id].ipmi
+ [const.IPMI_CREDS][const.USERNAME])
+ else:
+ return (None, None, None)
diff --git a/compass-tasks/deployment/installers/installer.py b/compass-tasks/deployment/installers/installer.py
new file mode 100644
index 0000000..cfeb9e8
--- /dev/null
+++ b/compass-tasks/deployment/installers/installer.py
@@ -0,0 +1,291 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "Grace Yu (grace.yu@huawei.com)"
+
+
+"""Module to provider installer interface.
+"""
+from Cheetah.Template import Template
+from copy import deepcopy
+import imp
+import logging
+import os
+import simplejson as json
+
+from compass.deployment.installers.config_manager import BaseConfigManager
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+
+CURRENT_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class BaseInstaller(object):
+ """Interface for installer."""
+ NAME = 'installer'
+
+ def __repr__(self):
+ return '%r[%r]' % (self.__class__.__name__, self.NAME)
+
+ def deploy(self, **kwargs):
+ """virtual method to start installing process."""
+ raise NotImplementedError
+
+ def clean_progress(self, **kwargs):
+ raise NotImplementedError
+
+ def delete_hosts(self, **kwargs):
+ """Delete hosts from installer server."""
+ raise NotImplementedError
+
+ def redeploy(self, **kwargs):
+ raise NotImplementedError
+
+ def ready(self, **kwargs):
+ pass
+
+ def cluster_ready(self, **kwargs):
+ pass
+
+ def get_tmpl_vars_from_metadata(self, metadata, config):
+ """Get variables dictionary for rendering templates from metadata.
+
+ :param dict metadata: The metadata dictionary.
+ :param dict config: The
+ """
+ template_vars = {}
+ self._get_tmpl_vars_helper(metadata, config, template_vars)
+
+ return template_vars
+
+ def _get_key_mapping(self, metadata, key, is_regular_key):
+ """Get the keyword which the input key maps to.
+
+ This keyword will be added to dictionary used to render templates.
+
+ If the key in metadata has a mapping to another keyword which is
+ used for templates, then return this keyword. If the key is started
+ with '$', which is a variable in metadata, return the key itself as
+ the mapping keyword. If the key has no mapping, return None.
+
+ :param dict metadata: metadata/submetadata dictionary.
+ :param str key: The keyword defined in metadata.
+ :param bool is_regular_key: False when the key defined in metadata
+ is a variable(starting with '$').
+ """
+ mapping_to = key
+ if is_regular_key:
+ try:
+ mapping_to = metadata['_self']['mapping_to']
+ except Exception:
+ mapping_to = None
+
+ return mapping_to
+
+ def _get_submeta_by_key(self, metadata, key):
+ """Get submetadata dictionary.
+
+ Based on current metadata key. And
+ determines the input key is a regular string keyword or a variable
+ keyword defined in metadata, which starts with '$'.
+
+ :param dict metadata: The metadata dictionary.
+ :param str key: The keyword defined in the metadata.
+ """
+ if key in metadata:
+ return (True, metadata[key])
+
+ temp = deepcopy(metadata)
+ if '_self' in temp:
+ del temp['_self']
+ meta_key = temp.keys()[0]
+ if meta_key.startswith("$"):
+ return (False, metadata[meta_key])
+
+ raise KeyError("'%s' is invalid in metadata '%s'!" % (key, metadata))
+
+ def _get_tmpl_vars_helper(self, metadata, config, output):
+ for key, config_value in sorted(config.iteritems()):
+ is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key)
+ mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key)
+
+ if isinstance(config_value, dict):
+ if mapping_to:
+ new_output = output[mapping_to] = {}
+ else:
+ new_output = output
+
+ self._get_tmpl_vars_helper(sub_meta, config_value, new_output)
+
+ elif mapping_to:
+ output[mapping_to] = config_value
+
+ def get_config_from_template(self, tmpl_path, vars_dict):
+ logging.debug("template path is %s", tmpl_path)
+ logging.debug("vars_dict is %s", vars_dict)
+
+ if not os.path.exists(tmpl_path) or not vars_dict:
+ logging.info("Template dir or vars_dict is None!")
+ return {}
+
+ searchList = []
+ copy_vars_dict = deepcopy(vars_dict)
+ for key, value in vars_dict.iteritems():
+ if isinstance(value, dict):
+ temp = copy_vars_dict[key]
+ del copy_vars_dict[key]
+ searchList.append(temp)
+ searchList.append(copy_vars_dict)
+
+ # Load base template first if it exists
+ base_config = {}
+ base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl')
+ if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path:
+ base_tmpl = Template(file=base_tmpl_path, searchList=searchList)
+ base_config = json.loads(base_tmpl.respond(), encoding='utf-8')
+ base_config = json.loads(json.dumps(base_config), encoding='utf-8')
+
+ # Load specific template for current adapter
+ tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList)
+ config = json.loads(tmpl.respond(), encoding='utf-8')
+ config = json.loads(json.dumps(config), encoding='utf-8')
+
+ # Merge the two outputs
+ config = util.merge_dict(base_config, config)
+
+ logging.debug("get_config_from_template resulting %s", config)
+ return config
+
+ @classmethod
+ def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info):
+ try:
+ mod_file, path, descr = imp.find_module(name, [path])
+ if mod_file:
+ mod = imp.load_module(name, mod_file, path, descr)
+ config_manager = BaseConfigManager(adapter_info, cluster_info,
+ hosts_info)
+ return getattr(mod, mod.NAME)(config_manager)
+
+ except ImportError as exc:
+ logging.error('No such module found: %s', name)
+ logging.exception(exc)
+
+ return None
+
+
+class OSInstaller(BaseInstaller):
+ """Interface for os installer."""
+ NAME = 'OSInstaller'
+ INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers')
+
+ def get_oses(self):
+ """virtual method to get supported oses.
+
+ :returns: list of str, each is the supported os version.
+ """
+ return []
+
+ @classmethod
+ def get_installer(cls, name, adapter_info, cluster_info, hosts_info):
+ if name is None:
+ logging.info("Installer name is None! No OS installer loaded!")
+ return None
+
+ path = os.path.join(cls.INSTALLER_BASE_DIR, name)
+ installer = super(OSInstaller, cls).get_installer(name, path,
+ adapter_info,
+ cluster_info,
+ hosts_info)
+
+ if not isinstance(installer, OSInstaller):
+ logging.info("Installer '%s' is not an OS installer!" % name)
+ return None
+
+ return installer
+
+ def poweron(self, host_id):
+ pass
+
+ def poweroff(self, host_id):
+ pass
+
+ def reset(self, host_id):
+ pass
+
+
+class PKInstaller(BaseInstaller):
+ """Interface for package installer."""
+ NAME = 'PKInstaller'
+ INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers')
+
+ def generate_installer_config(self):
+ raise NotImplementedError(
+ 'generate_installer_config is not defined in %s',
+ self.__class__.__name__
+ )
+
+ def get_target_systems(self):
+ """virtual method to get available target_systems for each os.
+
+ :param oses: supported os versions.
+ :type oses: list of st
+
+ :returns: dict of os_version to target systems as list of str.
+ """
+ return {}
+
+ def get_roles(self, target_system):
+ """virtual method to get all roles of given target system.
+
+ :param target_system: target distributed system such as OpenStack.
+ :type target_system: str
+
+ :returns: dict of role to role description as str.
+ """
+ return {}
+
+ def os_ready(self, **kwargs):
+ pass
+
+ def cluster_os_ready(self, **kwargs):
+ pass
+
+ def serialize_config(self, config, destination):
+ with open(destination, "w") as f:
+ f.write(config)
+
+ @classmethod
+ def get_installer(cls, name, adapter_info, cluster_info, hosts_info):
+ if name is None:
+ logging.info("Install name is None. No package installer loaded!")
+ return None
+
+ path = os.path.join(cls.INSTALLER_BASE_DIR, name)
+ if not os.path.exists(path):
+ path = os.path.join(os.path.join(os.path.join(
+ compass_setting.PLUGINS_DIR, name), "implementation"), name)
+ if not os.path.exists(path):
+ logging.info("Installer '%s' does not exist!" % name)
+ return None
+ installer = super(PKInstaller, cls).get_installer(name, path,
+ adapter_info,
+ cluster_info,
+ hosts_info)
+
+ if not isinstance(installer, PKInstaller):
+ logging.info("Installer '%s' is not a package installer!" % name)
+ return None
+
+ return installer
diff --git a/compass-tasks/deployment/installers/os_installers/__init__.py b/compass-tasks/deployment/installers/os_installers/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-tasks/deployment/installers/os_installers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py b/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py b/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py
new file mode 100644
index 0000000..9c2a935
--- /dev/null
+++ b/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py
@@ -0,0 +1,449 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""os installer cobbler plugin.
+"""
+import logging
+import os
+import shutil
+import xmlrpclib
+
+from compass.deployment.installers.installer import OSInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+from copy import deepcopy
+
+
+NAME = 'CobblerInstaller'
+
+
+class CobblerInstaller(OSInstaller):
+ """cobbler installer"""
+ CREDENTIALS = "credentials"
+ USERNAME = 'username'
+ PASSWORD = 'password'
+
+ INSTALLER_URL = "cobbler_url"
+ TMPL_DIR = 'tmpl_dir'
+ SYS_TMPL = 'system.tmpl'
+ SYS_TMPL_NAME = 'system.tmpl'
+ SYS_PROFILE_NAME = 'profile.tmpl'
+ PROFILE = 'profile'
+
+ POWER_TYPE = 'power_type'
+ POWER_ADDR = 'power_address'
+ POWER_USER = 'power_user'
+ POWER_PASS = 'power_pass'
+
+ def __init__(self, config_manager):
+ super(CobblerInstaller, self).__init__()
+
+ self.config_manager = config_manager
+ installer_settings = self.config_manager.get_os_installer_settings()
+ try:
+ username = installer_settings[self.CREDENTIALS][self.USERNAME]
+ password = installer_settings[self.CREDENTIALS][self.PASSWORD]
+ cobbler_url = installer_settings[self.INSTALLER_URL]
+ self.tmpl_dir = CobblerInstaller.get_tmpl_path()
+
+ except KeyError as ex:
+ raise KeyError(ex.message)
+
+ # The connection is created when cobbler installer is initialized.
+ self.remote = self._get_cobbler_server(cobbler_url)
+ self.token = self._get_token(username, password)
+ self.pk_installer_config = None
+
+ logging.debug('%s instance created', 'CobblerInstaller')
+
+ @classmethod
+ def get_tmpl_path(cls):
+ return os.path.join(compass_setting.TMPL_DIR, 'cobbler')
+
+ def __repr__(self):
+ return '%r[remote=%r,token=%r' % (
+ self.__class__.__name__, self.remote, self.token)
+
+ def _get_cobbler_server(self, cobbler_url):
+ if not cobbler_url:
+ logging.error("Cobbler URL is None!")
+ raise Exception("Cobbler URL cannot be None!")
+
+ return xmlrpclib.Server(cobbler_url)
+
+ def _get_token(self, username, password):
+ if self.remote is None:
+ raise Exception("Cobbler remote instance is None!")
+ return self.remote.login(username, password)
+
+ def get_supported_oses(self):
+ """get supported os versions.
+
+ note::
+ In cobbler, we treat profile name as the indicator
+ of os version. It is just a simple indicator
+ and not accurate.
+ """
+ profiles = self.remote.get_profiles()
+ oses = []
+ for profile in profiles:
+ oses.append(profile['name'])
+ return oses
+
+ def deploy(self):
+ """Sync cobbler to catch up the latest update config and start to
+
+ install OS. Return both cluster and hosts deploy configs. The return
+ format:
+ {
+ "cluster": {
+ "id": 1,
+ "deployed_os_config": {},
+ },
+ "hosts": {
+ 1($clusterhost_id): {
+ "deployed_os_config": {...},
+ },
+ ....
+ }
+ }
+ """
+ host_ids = self.config_manager.get_hosts_id_list_for_os_installation()
+ if not host_ids:
+ # No hosts need to install OS
+ logging.info("Cobbler: No host needs to install OS.")
+ return {}
+
+ os_version = self.config_manager.get_os_version()
+ profile = self._get_profile_from_server(os_version)
+
+ global_vars_dict = self._get_cluster_tmpl_vars_dict()
+
+ self.update_profile_config_to_cobbler(profile, global_vars_dict)
+
+ hosts_deploy_config = {}
+
+ for host_id in host_ids:
+ hostname = self.config_manager.get_hostname(host_id)
+ vars_dict = self._get_host_tmpl_vars_dict(host_id,
+ global_vars_dict,
+ hostname=hostname,
+ profile=profile)
+
+ self.update_host_config_to_cobbler(host_id, hostname, vars_dict)
+
+ # set host deploy config
+ host_config = {}
+ host_config[const.DEPLOYED_OS_CONFIG] = vars_dict[const.OS_CONFIG]
+ hosts_deploy_config[host_id] = host_config
+
+ # sync to cobbler and trigger installtion.
+ self._sync()
+
+ cluster_config = global_vars_dict.setdefault(const.OS_CONFIG, {})
+
+ return {
+ const.CLUSTER: {
+ const.ID: self.config_manager.get_cluster_id(),
+ const.DEPLOYED_OS_CONFIG: cluster_config
+ },
+ const.HOSTS: hosts_deploy_config
+ }
+
+ def clean_progress(self):
+ """clean log files and config for hosts which to deploy."""
+ clusterhost_list = self.config_manager.get_host_id_list()
+ log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME]
+
+ for host_id in clusterhost_list:
+ hostname = self.config_manager.get_hostname(host_id)
+ self._clean_log(log_dir_prefix, hostname)
+
+ def redeploy(self):
+ """redeploy hosts."""
+ host_ids = self.config_manager.get_host_id_list()
+ if not host_ids:
+ logging.info("Cobbler: hostlist is None, no host is redeployed")
+ return
+ for host_id in host_ids:
+ hostname = self.config_manager.get_hostname(host_id)
+ sys_id = self._get_create_system(hostname)
+ if sys_id:
+ # enable netboot for this host
+ self._netboot_enabled(sys_id)
+
+ self._sync()
+
+ def set_package_installer_config(self, package_configs):
+ """Cobbler can install and configure package installer right after
+
+ OS installation compelets by setting package_config info provided
+ by package installer.
+
+ :param dict package_configs: The dict of config generated by package
+ installer for each clusterhost. The IDs
+ of clusterhosts are the keys of
+ package_configs.
+ """
+ self.pk_installer_config = package_configs
+
+ def _sync(self):
+ """Sync the updated config to cobbler and trigger installation."""
+ try:
+ self.remote.sync(self.token)
+ os.system('sudo service rsyslog restart')
+ except Exception as ex:
+ logging.debug("Failed to sync cobbler server! Error: %s" % ex)
+ raise ex
+
+ def dump_system_info(self, host_id):
+
+ hostname = self.config_manager.get_hostname(host_id)
+ if self.remote is None or not hostname:
+ logging.info("[dump_system_info]Remote or hostname is None.")
+ return {}
+
+ return self.remote.get_system_as_rendered(hostname)
+
+ def _generate_system_config(self, host_id, host_vars_dict):
+ """Generate updated system config from the template.
+
+ :param host_vars_dict: dict of variables for the system template to
+ generate system config dict for each host.
+ """
+ os_version = self.config_manager.get_os_version()
+
+ tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME
+ )
+ if not os.path.exists(tmpl_path):
+ err_msg = "Template '%s' does not exists!" % tmpl_path
+ logging.error(err_msg)
+ raise Exception(err_msg)
+ host_vars_dict[const.BASEINFO]['host_id'] = host_id
+ system_config = self.get_config_from_template(tmpl_path,
+ host_vars_dict)
+
+ # update package config info to cobbler ksmeta
+ if self.pk_installer_config and host_id in self.pk_installer_config:
+ pk_config = self.pk_installer_config[host_id]
+ ksmeta = system_config.setdefault("ksmeta", {})
+ util.merge_dict(ksmeta, pk_config)
+ system_config["ksmeta"] = ksmeta
+
+ return system_config
+
+ def _generate_profile_config(self, cluster_vars_dict):
+ os_version = self.config_manager.get_os_version()
+ tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, os_version), self.SYS_PROFILE_NAME
+ )
+
+ return self.get_config_from_template(tmpl_path, cluster_vars_dict)
+
+ def _get_profile_from_server(self, os_version):
+ """Get profile from cobbler server."""
+ result = self.remote.find_profile({'name': os_version})
+ if not result:
+ raise Exception("Cannot find profile for '%s'", os_version)
+
+ profile = result[0]
+ return profile
+
+ def _get_create_system(self, hostname):
+ """get system reference id for the host."""
+ sys_name = hostname
+ sys_id = None
+ system_info = self.remote.find_system({"name": hostname})
+
+ if not system_info:
+ # Create a new system
+ sys_id = self.remote.new_system(self.token)
+ self.remote.modify_system(sys_id, "name", hostname, self.token)
+ logging.debug('create new system %s for %s', sys_id, sys_name)
+ else:
+ sys_id = self.remote.get_system_handle(sys_name, self.token)
+
+ return sys_id
+
+ def _get_profile_id(self, profilename):
+ """get profile reference id for the cluster."""
+ return self.remote.get_profile_handle(profilename, self.token)
+
+ def _clean_system(self, hostname):
+ """clean system."""
+ sys_name = hostname
+ try:
+ self.remote.remove_system(sys_name, self.token)
+ logging.debug('system %s is removed', sys_name)
+ except Exception:
+ logging.debug('no system %s found to remove', sys_name)
+
+ def _update_system_config(self, sys_id, system_config):
+ """update modify system."""
+ for key, value in system_config.iteritems():
+ self.remote.modify_system(sys_id, str(key), value, self.token)
+
+ self.remote.save_system(sys_id, self.token)
+
+ def _update_profile_config(self, profile_id, profile_config):
+ for key, value in profile_config.iteritems():
+ self.remote.modify_profile(profile_id, str(key), value, self.token)
+
+ self.remote.save_profile(profile_id, self.token)
+
+ def _netboot_enabled(self, sys_id):
+ """enable netboot."""
+ self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token)
+ self.remote.save_system(sys_id, self.token)
+
+ def _clean_log(self, log_dir_prefix, system_name):
+ """clean log."""
+ log_dir = os.path.join(log_dir_prefix, system_name)
+ shutil.rmtree(log_dir, True)
+
+ def update_host_config_to_cobbler(self, host_id, hostname, host_vars_dict):
+ """update host config and upload to cobbler server."""
+ sys_id = self._get_create_system(hostname)
+
+ system_config = self._generate_system_config(host_id, host_vars_dict)
+ logging.debug('%s system config to update: %s', host_id, system_config)
+
+ self._update_system_config(sys_id, system_config)
+ self._netboot_enabled(sys_id)
+
+ def update_profile_config_to_cobbler(self, profilename, cluster_vars_dict):
+ """update profile config and upload to cobbler server."""
+
+ profile_id = self._get_profile_id(profilename)
+
+ profile_config = self._generate_profile_config(cluster_vars_dict)
+ logging.debug(
+ '%s profile config to update: %s', profilename, profile_config
+ )
+
+ self._update_profile_config(profile_id, profile_config)
+
+ def delete_hosts(self):
+ hosts_id_list = self.config_manager.get_host_id_list()
+ logging.debug('delete hosts %s', hosts_id_list)
+ for host_id in hosts_id_list:
+ self.delete_single_host(host_id)
+ self._sync()
+
+ def delete_single_host(self, host_id):
+ """Delete the host from cobbler server and clean up the installation
+
+ progress.
+ """
+ hostname = self.config_manager.get_hostname(host_id)
+ try:
+ log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME]
+ self._clean_system(hostname)
+ self._clean_log(log_dir_prefix, hostname)
+ except Exception as ex:
+ logging.error("Deleting host got exception: %s", ex)
+ logging.exception(ex)
+
+ def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs):
+ """Generate template variables dictionary."""
+ vars_dict = {}
+ if global_vars_dict:
+ # Set cluster template vars_dict from cluster os_config.
+ vars_dict = deepcopy(global_vars_dict)
+
+ # Set hostname, MAC address and hostname, networks, dns and so on.
+ host_baseinfo = self.config_manager.get_host_baseinfo(host_id)
+ vars_dict[const.BASEINFO] = host_baseinfo
+
+ # Set profile
+ if self.PROFILE in kwargs:
+ profile = kwargs[self.PROFILE]
+ else:
+ os_version = self.config_manager.get_os_version()
+ profile = self._get_profile_from_server(os_version)
+
+ vars_dict[const.BASEINFO][self.PROFILE] = profile
+
+ metadata = self.config_manager.get_os_config_metadata()
+ os_config = self.config_manager.get_host_os_config(host_id)
+
+ # Get template variables values from host os_config
+ host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config)
+ util.merge_dict(
+ vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict
+ )
+ return vars_dict
+
+ def _get_cluster_tmpl_vars_dict(self):
+ metadata = self.config_manager.get_os_config_metadata()
+ os_config = self.config_manager.get_cluster_os_config()
+
+ cluster_vas_dict = {}
+ cluster_vas_dict[const.OS_CONFIG] = \
+ self.get_tmpl_vars_from_metadata(metadata, os_config)
+
+ return cluster_vas_dict
+
+ def _check_and_set_system_impi(self, host_id, sys_id):
+ if not sys_id:
+ logging.info("System is None!")
+ return False
+
+ system = self.dump_system_info(host_id)
+ if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]:
+ # Set sytem power type to ipmilan if needs and set IPMI info
+ ipmi_info = self.config_manager.get_host_ipmi_info(host_id)
+ if not ipmi_info:
+ logging.info('No IPMI information found! Failed power on.')
+ return False
+
+ ipmi_ip, ipmi_user, ipmi_pass = ipmi_info
+ power_opts = {}
+ power_opts[self.POWER_TYPE] = 'ipmilan'
+ power_opts[self.POWER_ADDR] = ipmi_ip
+ power_opts[self.POWER_USER] = ipmi_user
+ power_opts[self.POWER_PASS] = ipmi_pass
+
+ self._update_system_config(sys_id, power_opts)
+
+ return True
+
+ def poweron(self, host_id):
+ hostname = self.config_manager.get_hostname(host_id)
+ sys_id = self._get_create_system(hostname)
+ if not self._check_and_set_system_impi(sys_id):
+ return
+
+ self.remote.power_system(sys_id, self.token, power='on')
+ logging.info("Host with ID=%d starts to power on!" % host_id)
+
+ def poweroff(self, host_id):
+ hostname = self.config_manager.get_hostname(host_id)
+ sys_id = self._get_create_system(hostname)
+ if not self._check_and_set_system_impi(sys_id):
+ return
+
+ self.remote.power_system(sys_id, self.token, power='off')
+ logging.info("Host with ID=%d starts to power off!" % host_id)
+
+ def reset(self, host_id):
+ hostname = self.config_manager.get_hostname(host_id)
+ sys_id = self._get_create_system(hostname)
+ if not self._check_and_set_system_impi(sys_id):
+ return
+
+ self.remote.power_system(sys_id, self.token, power='reboot')
+ logging.info("Host with ID=%d starts to reboot!" % host_id)
diff --git a/compass-tasks/deployment/installers/pk_installers/__init__.py b/compass-tasks/deployment/installers/pk_installers/__init__.py
new file mode 100644
index 0000000..5e42ae9
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py
diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
new file mode 100644
index 0000000..0a86be4
--- /dev/null
+++ b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py
@@ -0,0 +1,441 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__auther__ = "Compass Dev Team (dev-team@syscompass.org)"
+
+"""package installer: ansible plugin."""
+
+from Cheetah.Template import Template
+from copy import deepcopy
+import json
+import logging
+import os
+import re
+import shutil
+import subprocess
+
+from compass.deployment.installers.installer import PKInstaller
+from compass.deployment.utils import constants as const
+from compass.utils import setting_wrapper as compass_setting
+from compass.utils import util
+
+NAME = "AnsibleInstaller"
+
+
+def byteify(input):
+ if isinstance(input, dict):
+ return dict([(byteify(key), byteify(value))
+ for key, value in input.iteritems()])
+ elif isinstance(input, list):
+ return [byteify(element) for element in input]
+ elif isinstance(input, unicode):
+ return input.encode('utf-8')
+ else:
+ return input
+
+
+class AnsibleInstaller(PKInstaller):
+ INVENTORY_TMPL_DIR = 'inventories'
+ GROUPVARS_TMPL_DIR = 'vars'
+ INVENTORY_PATCH_TEMPALTE_DIR = 'inventories'
+
+ # keywords in package installer settings
+ ANSIBLE_DIR = 'ansible_dir'
+ ANSIBLE_RUN_DIR = 'ansible_run_dir'
+ LOG_FILE = 'ansible_log_file'
+ ANSIBLE_CONFIG = 'ansible_config'
+ INVENTORY = 'inventory_file'
+ INVENTORY_JSON = 'inventory_json_file'
+ INVENTORY_GROUP = 'inventory_group'
+ GROUP_VARIABLE = 'group_variable'
+ HOSTS_PATH = 'etc_hosts_path'
+ RUNNER_DIRS = 'runner_dirs'
+
+ def __init__(self, config_manager):
+ super(AnsibleInstaller, self).__init__()
+
+ self.config_manager = config_manager
+ self.tmpl_name = self.config_manager.get_cluster_flavor_template()
+ self.installer_settings = (
+ self.config_manager.get_pk_installer_settings()
+ )
+ settings = self.installer_settings
+ self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None)
+ self.ansible_run_dir = (
+ settings.setdefault(self.ANSIBLE_RUN_DIR, None)
+ )
+ self.log_file = settings.setdefault(self.LOG_FILE, None)
+ self.ansible_config = (
+ settings.setdefault(self.ANSIBLE_CONFIG, None)
+ )
+ self.inventory = settings.setdefault(self.INVENTORY, None)
+ self.inventory_json = settings.setdefault(self.INVENTORY_JSON, None)
+ self.inventory_group = settings.setdefault(self.INVENTORY_GROUP, None)
+ self.group_variable = (
+ settings.setdefault(self.GROUP_VARIABLE, None)
+ )
+ self.hosts_path = (
+ settings.setdefault(self.HOSTS_PATH, None)
+ )
+ self.runner_dirs = (
+ settings.setdefault(self.RUNNER_DIRS, None)
+ )
+ self.playbook = self.tmpl_name.replace('tmpl', 'yml')
+ self.runner_files = [self.playbook]
+
+ adapter_name = self.config_manager.get_dist_system_name()
+ self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name)
+ self.adapter_dir = os.path.join(self.ansible_dir, adapter_name)
+ logging.debug('%s instance created', self)
+
+ @classmethod
+ def get_tmpl_path(cls, adapter_name):
+ tmpl_path = os.path.join(
+ os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'),
+ adapter_name
+ )
+ return tmpl_path
+
+ def __repr__(self):
+ return '%s[name=%s,installer_url=%s]' % (
+ self.__class__.__name__, self.NAME, self.installer_url)
+
+ def dump_inventory(self, data, inventory):
+ with open(inventory, "w") as f:
+ json.dump(data, f, indent=4)
+
+ def _generate_inventory_data(self, global_vars_dict):
+ vars_dict = global_vars_dict['roles_mapping']
+ inventory_data = {}
+ inventory_data['_meta'] = {'hostvars': {}}
+ for item in self.inventory_group:
+ if item in vars_dict:
+ inventory_data[item] = {'hosts': []}
+ for host in vars_dict[item]:
+ hostname = host['hostname']
+ if hostname not in inventory_data['_meta']['hostvars']:
+ host_dict = {}
+ host_dict['ansible_ssh_host'] = host['install']['ip']
+ host_dict['ansible_ssh_user'] = 'root'
+ host_dict['ansible_ssh_pass'] = 'root'
+ inventory_data['_meta']['hostvars'].update(
+ {hostname: host_dict})
+ inventory_data[item]['hosts'].append(hostname)
+
+ inventory_data['ceph'] = {'children':
+ ['ceph_adm', 'ceph_mon', 'ceph_osd']}
+ return inventory_data
+
+ def generate_installer_config(self):
+ """Render ansible config file by OS installing.
+
+ The output format:
+ {
+ '1'($host_id/clusterhost_id):{
+ 'tool': 'ansible',
+ },
+ .....
+ }
+ """
+ host_ids = self.config_manager.get_host_id_list()
+ os_installer_configs = {}
+ for host_id in host_ids:
+ temp = {
+ "tool": "ansible",
+ }
+ os_installer_configs[host_id] = temp
+
+ return os_installer_configs
+
+ def get_env_name(self, dist_sys_name, cluster_name):
+ return "-".join((dist_sys_name, cluster_name))
+
+ def _get_cluster_tmpl_vars(self):
+ """Generate template variables dict
+
+ Generates based on cluster level config.
+ The vars_dict will be:
+ {
+ "baseinfo": {
+ "id":1,
+ "name": "cluster01",
+ ...
+ },
+ "package_config": {
+ .... //mapped from original package config based on metadata
+ },
+ "role_mapping": {
+ ....
+ }
+ }
+ """
+ cluster_vars_dict = {}
+ # set cluster basic information to vars_dict
+ cluster_baseinfo = self.config_manager.get_cluster_baseinfo()
+ cluster_vars_dict[const.BASEINFO] = cluster_baseinfo
+
+ # get and set template variables from cluster package config.
+ pk_metadata = self.config_manager.get_pk_config_meatadata()
+ pk_config = self.config_manager.get_cluster_package_config()
+
+ # get os config as ansible needs them
+ os_metadata = self.config_manager.get_os_config_metadata()
+ os_config = self.config_manager.get_cluster_os_config()
+
+ pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config)
+ os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config)
+ util.merge_dict(pk_meta_dict, os_meta_dict)
+
+ cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict
+
+ # get and set roles_mapping to vars_dict
+ mapping = self.config_manager.get_cluster_roles_mapping()
+ logging.info("cluster role mapping is %s", mapping)
+ cluster_vars_dict[const.ROLES_MAPPING] = mapping
+
+ # get ip settings to vars_dict
+ hosts_ip_settings = self.config_manager.get_hosts_ip_settings(
+ pk_meta_dict["network_cfg"]["ip_settings"],
+ pk_meta_dict["network_cfg"]["sys_intf_mappings"]
+ )
+ logging.info("hosts_ip_settings is %s", hosts_ip_settings)
+ cluster_vars_dict["ip_settings"] = hosts_ip_settings
+
+ return byteify(cluster_vars_dict)
+
+ def _generate_inventory_attributes(self, global_vars_dict):
+ inventory_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR),
+ self.tmpl_name
+ )
+ if not os.path.exists(inventory_tmpl_path):
+ logging.error(
+ "Inventory template '%s' does not exist", self.tmpl_name
+ )
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+ inventory_dir = os.path.join(global_vars_dict['run_dir'], 'inventories')
+ inventory_json = os.path.join(inventory_dir, self.inventory_json)
+ vars_dict = {'inventory_json': inventory_json}
+ return self.get_config_from_template(
+ inventory_tmpl_path, vars_dict
+ )
+
+ def _generate_group_vars_attributes(self, global_vars_dict):
+ logging.info("global vars dict is %s", global_vars_dict)
+ group_vars_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR),
+ self.tmpl_name
+ )
+ if not os.path.exists(group_vars_tmpl_path):
+ logging.error("Vars template '%s' does not exist",
+ self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(
+ group_vars_tmpl_path, global_vars_dict
+ )
+
+ def _generate_hosts_attributes(self, global_vars_dict):
+ hosts_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name
+ )
+ if not os.path.exists(hosts_tmpl_path):
+ logging.error("Hosts template '%s' does not exist", self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(hosts_tmpl_path, global_vars_dict)
+
+ def _generate_ansible_cfg_attributes(self, global_vars_dict):
+ ansible_cfg_tmpl_path = os.path.join(
+ os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name
+ )
+ if not os.path.exists(ansible_cfg_tmpl_path):
+ logging.error("cfg template '%s' does not exist", self.tmpl_name)
+ raise Exception("Template '%s' does not exist!" % self.tmpl_name)
+
+ return self.get_config_from_template(
+ ansible_cfg_tmpl_path,
+ global_vars_dict
+ )
+
+ def get_config_from_template(self, tmpl_path, vars_dict):
+ logging.debug("vars_dict is %s", vars_dict)
+
+ if not os.path.exists(tmpl_path) or not vars_dict:
+ logging.info("Template dir or vars_dict is None!")
+ return {}
+
+ searchList = []
+ copy_vars_dict = deepcopy(vars_dict)
+ for key, value in vars_dict.iteritems():
+ if isinstance(value, dict):
+ temp = copy_vars_dict[key]
+ del copy_vars_dict[key]
+ searchList.append(temp)
+ searchList.append(copy_vars_dict)
+
+ # Load specific template for current adapter
+ tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList)
+ return tmpl.respond()
+
+ def _create_ansible_run_env(self, env_name, ansible_run_destination):
+ if os.path.exists(ansible_run_destination):
+ shutil.rmtree(ansible_run_destination, True)
+
+ os.mkdir(ansible_run_destination)
+
+ # copy roles to run env
+ dirs = self.runner_dirs
+ files = self.runner_files
+ for dir in dirs:
+ if not os.path.exists(os.path.join(self.ansible_dir, dir)):
+ continue
+ os.system(
+ "cp -rf %s %s" % (
+ os.path.join(self.ansible_dir, dir),
+ ansible_run_destination
+ )
+ )
+ for file in files:
+ logging.info('file is %s', file)
+ shutil.copy(
+ os.path.join(self.adapter_dir, file),
+ os.path.join(
+ ansible_run_destination,
+ file
+ )
+ )
+
+ def prepare_ansible(self, env_name, global_vars_dict):
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ if os.path.exists(ansible_run_destination):
+ ansible_run_destination += "-expansion"
+ self._create_ansible_run_env(env_name, ansible_run_destination)
+ global_vars_dict.update({'run_dir': ansible_run_destination})
+
+ inv_config = self._generate_inventory_attributes(global_vars_dict)
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+
+ vars_config = self._generate_group_vars_attributes(global_vars_dict)
+ vars_dir = os.path.join(ansible_run_destination, 'group_vars')
+
+ hosts_config = self._generate_hosts_attributes(global_vars_dict)
+ hosts_destination = os.path.join(
+ ansible_run_destination, self.hosts_path
+ )
+
+ cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict)
+ cfg_destination = os.path.join(
+ ansible_run_destination,
+ self.ansible_config
+ )
+
+ inventory_data = self._generate_inventory_data(global_vars_dict)
+ inventory_json_destination = os.path.join(inventory_dir,
+ self.inventory_json)
+
+ os.mkdir(inventory_dir)
+ os.mkdir(vars_dir)
+
+ inventory_destination = os.path.join(inventory_dir, self.inventory)
+ group_vars_destination = os.path.join(vars_dir, self.group_variable)
+ self.dump_inventory(inventory_data, inventory_json_destination)
+ self.serialize_config(inv_config, inventory_destination)
+ self.serialize_config(vars_config, group_vars_destination)
+ self.serialize_config(hosts_config, hosts_destination)
+ self.serialize_config(cfg_config, cfg_destination)
+
+ def deploy(self):
+ """Start to deploy a distributed system.
+
+ Return both cluster and hosts deployed configs.
+ The return format:
+ {
+ "cluster": {
+ "id": 1,
+ "deployed_package_config": {
+ "roles_mapping": {...},
+ "service_credentials": {...},
+ ....
+ }
+ },
+ "hosts": {
+ 1($clusterhost_id): {
+ "deployed_package_config": {...}
+ },
+ ....
+ }
+ }
+ """
+ host_list = self.config_manager.get_host_id_list()
+ if not host_list:
+ return {}
+
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+
+ global_vars_dict = self._get_cluster_tmpl_vars()
+ logging.info(
+ '%s var dict: %s', self.__class__.__name__, global_vars_dict
+ )
+ # Create ansible related files
+ self.prepare_ansible(env_name, global_vars_dict)
+
+ def patch(self, patched_role_mapping):
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+ patched_global_vars_dict = self._get_cluster_tmpl_vars()
+ mapping = self.config_manager.get_cluster_patched_roles_mapping()
+ patched_global_vars_dict['roles_mapping'] = mapping
+ patched_inv = self._generate_inventory_attributes(
+ patched_global_vars_dict)
+ inv_file = os.path.join(inventory_dir, 'patched_inventory.yml')
+ self.serialize_config(patched_inv, inv_file)
+ config_file = os.path.join(
+ ansible_run_destination, self.ansible_config
+ )
+ playbook_file = os.path.join(ansible_run_destination, self.playbook)
+ log_file = os.path.join(ansible_run_destination, 'patch.log')
+ cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+ inv_file,
+ playbook_file)
+ with open(log_file, 'w') as logfile:
+ subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)
+ return patched_role_mapping
+
+ def cluster_os_ready(self):
+ adapter_name = self.config_manager.get_adapter_name()
+ cluster_name = self.config_manager.get_clustername()
+ env_name = self.get_env_name(adapter_name, cluster_name)
+ ansible_run_destination = os.path.join(self.ansible_run_dir, env_name)
+ expansion_dir = ansible_run_destination + "-expansion"
+ if os.path.exists(expansion_dir):
+ ansible_run_destination = expansion_dir
+ inventory_dir = os.path.join(ansible_run_destination, 'inventories')
+ inventory_file = os.path.join(inventory_dir, self.inventory)
+ playbook_file = os.path.join(ansible_run_destination, self.playbook)
+ log_file = os.path.join(ansible_run_destination, 'run.log')
+ config_file = os.path.join(
+ ansible_run_destination, self.ansible_config
+ )
+ os.system("chmod +x %s" % inventory_file)
+ cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file,
+ inventory_file,
+ playbook_file)
+ with open(log_file, 'w') as logfile:
+ subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile)