diff options
Diffstat (limited to 'compass-tasks/deployment')
13 files changed, 0 insertions, 2119 deletions
diff --git a/compass-tasks/deployment/__init__.py b/compass-tasks/deployment/__init__.py deleted file mode 100644 index cbd36e0..0000000 --- a/compass-tasks/deployment/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks/deployment/deploy_manager.py b/compass-tasks/deployment/deploy_manager.py deleted file mode 100644 index baf7cd6..0000000 --- a/compass-tasks/deployment/deploy_manager.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - -"""Module to get configs from provider and isntallers and update - them to provider and installers. -""" -from compass.deployment.installers.installer import OSInstaller -from compass.deployment.installers.installer import PKInstaller -from compass.deployment.utils import constants as const -from compass.utils import util - - -import logging - - -class DeployManager(object): - """Deploy manager module.""" - def __init__(self, adapter_info, cluster_info, hosts_info): - """Init deploy manager.""" - self.os_installer = None - self.pk_installer = None - - # Get OS installer - os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] - self.os_installer = DeployManager._get_installer(OSInstaller, - os_installer_name, - adapter_info, - cluster_info, - hosts_info) - - # Get package installer - pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) - if pk_info: - pk_installer_name = pk_info[const.NAME] - self.pk_installer = DeployManager._get_installer(PKInstaller, - pk_installer_name, - adapter_info, - cluster_info, - hosts_info) - - @staticmethod - def _get_installer(installer_type, name, adapter_info, cluster_info, - hosts_info): - """Get installer instance.""" - callback = getattr(installer_type, 'get_installer') - installer = callback(name, adapter_info, cluster_info, hosts_info) - - return installer - - def deploy(self): - """Deploy the cluster.""" - deployed_config = self.deploy_os() - package_deployed_config = self.deploy_target_system() - - util.merge_dict(deployed_config, package_deployed_config) - - return deployed_config - - def check_cluster_health(self, callback_url): - logging.info("DeployManager check_cluster_health...........") - self.pk_installer.check_cluster_health(callback_url) - - def clean_progress(self): - """Clean previous installation log and progress.""" - self.clean_os_installtion_progress() - self.clean_package_installation_progress() - - def clean_os_installtion_progress(self): - # OS installer cleans previous installing progress. - if self.os_installer: - self.os_installer.clean_progress() - - def clean_package_installation_progress(self): - # Package installer cleans previous installing progress. - if self.pk_installer: - self.pk_installer.clean_progress() - - def prepare_for_deploy(self): - self.clean_progress() - - def deploy_os(self): - """Deploy OS to hosts which need to in the cluster. - - Return OS deployed config. - """ - if not self.os_installer: - return {} - - pk_installer_config = {} - if self.pk_installer: - # generate target system config which will be installed by OS - # installer right after OS installation is completed. - pk_installer_config = self.pk_installer.generate_installer_config() - logging.debug('[DeployManager]package installer config is %s', - pk_installer_config) - - # Send package installer config info to OS installer. - self.os_installer.set_package_installer_config(pk_installer_config) - - # start to deploy OS - return self.os_installer.deploy() - - def deploy_target_system(self): - """Deploy target system to all hosts in the cluster. - - Return package deployed config. - """ - if not self.pk_installer: - return {} - - return self.pk_installer.deploy() - - def redeploy_os(self): - """Redeploy OS for this cluster without changing configurations.""" - if not self.os_installer: - logging.info("Redeploy_os: No OS installer found!") - return - - self.os_installer.redeploy() - logging.info("Start to redeploy OS for cluster.") - - def redeploy_target_system(self): - """Redeploy target system for the cluster without changing config.""" - if not self.pk_installer: - logging.info("Redeploy_target_system: No installer found!") - return - - self.pk_installer.deploy() - logging.info("Start to redeploy target system.") - - def redeploy(self): - """Redeploy the cluster without changing configurations.""" - self.redeploy_os() - self.redeploy_target_system() - - def remove_hosts(self, package_only=False, delete_cluster=False): - """Remove hosts from both OS and/or package installlers server side.""" - if self.os_installer and not package_only: - self.os_installer.delete_hosts() - - if self.pk_installer: - self.pk_installer.delete_hosts(delete_cluster=delete_cluster) - - def os_installed(self): - if self.os_installer: - self.os_installer.ready() - if self.pk_installer: - self.pk_installer.os_ready() - - def cluster_os_installed(self): - if self.os_installer: - self.os_installer.cluster_ready() - if self.pk_installer: - self.pk_installer.cluster_os_ready() - - def package_installed(self): - if self.pk_installer: - self.pk_installer.ready() - - def cluster_installed(self): - if self.pk_installer: - self.pk_installer.cluster_ready() - - -class Patcher(DeployManager): - """Patcher Module.""" - def __init__(self, adapter_info, cluster_info, hosts_info, cluster_hosts): - self.pk_installer = None - self.cluster_info = cluster_info - registered_roles = cluster_info['flavor']['roles'] - - pk_info = adapter_info.setdefault(const.PK_INSTALLER, {}) - if pk_info: - pk_installer_name = pk_info[const.NAME] - self.pk_installer = Patcher._get_installer(PKInstaller, - pk_installer_name, - adapter_info, - cluster_info, - hosts_info) - - patched_role_mapping = {} - for role in registered_roles: - patched_role_mapping[role] = [] - for host in cluster_hosts: - if len(host['patched_roles']) == 0: - continue - for role in host['patched_roles']: - patched_role_mapping[role['name']].append(host) - self.patched_role_mapping = patched_role_mapping - - def patch(self): - patched_config = self.pk_installer.patch(self.patched_role_mapping) - - return patched_config - - -class PowerManager(object): - """Manage host to power on, power off, and reset.""" - - def __init__(self, adapter_info, cluster_info, hosts_info): - os_installer_name = adapter_info[const.OS_INSTALLER][const.NAME] - self.os_installer = DeployManager._get_installer(OSInstaller, - os_installer_name, - adapter_info, - cluster_info, - hosts_info) - - def poweron(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.poweron() - - def poweroff(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.poweroff() - - def reset(self): - if not self.os_installer: - logging.info("No OS installer found, cannot power on machine!") - return - self.os_installer.reset() diff --git a/compass-tasks/deployment/installers/__init__.py b/compass-tasks/deployment/installers/__init__.py deleted file mode 100644 index 0296be5..0000000 --- a/compass-tasks/deployment/installers/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""modules to read/write cluster/host config from installers. - - .. moduleauthor:: Grace Yu <grace.yu@huawei.com> -""" diff --git a/compass-tasks/deployment/installers/config_manager.py b/compass-tasks/deployment/installers/config_manager.py deleted file mode 100644 index 597c3a6..0000000 --- a/compass-tasks/deployment/installers/config_manager.py +++ /dev/null @@ -1,527 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "baigk baiguoku@huawei.com)" - -from collections import defaultdict -from copy import deepcopy -import json -import logging -import netaddr - -from compass.deployment.utils import constants as const - -ip_generator_map = {} - - -def get_ip_addr(ip_ranges): - def _get_ip_addr(): - for ip_range in ip_ranges: - for ip in netaddr.iter_iprange(*ip_range): - yield str(ip) - - s = json.dumps(ip_ranges) - if s not in ip_generator_map: - ip_generator_map[s] = _get_ip_addr() - return ip_generator_map[s] - else: - return ip_generator_map[s] - - -class AdapterInfo(object): - def __init__(self, adapter_info): - self.adapter_info = adapter_info - self.name = self.adapter_info.get(const.NAME) - self.dist_system_name = self.name - self.health_check_cmd = self.adapter_info.get(const.HEALTH_CHECK_CMD) - - self.os_installer = self.adapter_info.setdefault( - const.OS_INSTALLER, {} - ) - self.os_installer.setdefault(const.INSTALLER_SETTINGS, {}) - - self.package_installer = self.adapter_info.setdefault( - const.PK_INSTALLER, {} - ) - self.package_installer.setdefault(const.INSTALLER_SETTINGS, {}) - - self.metadata = self.adapter_info.setdefault(const.METADATA, {}) - self.os_metadata = self.metadata.setdefault(const.OS_CONFIG, {}) - self.package_metadata = self.metadata.setdefault(const.PK_CONFIG, {}) - - self.flavors = dict([(f[const.FLAVOR_NAME], f) - for f in self.adapter_info.get(const.FLAVOR, [])]) - - @property - def flavor_list(self): - return self.flavors.values() - - def get_flavor(self, flavor_name): - return self.flavors.get(flavor_name) - - -class ClusterInfo(object): - def __init__(self, cluster_info): - self.cluster_info = cluster_info - self.id = self.cluster_info.get(const.ID) - self.name = self.cluster_info.get(const.NAME) - self.os_version = self.cluster_info.get(const.OS_VERSION) - self.flavor = self.cluster_info.setdefault( - const.FLAVOR, {} - ) - self.os_config = self.cluster_info.setdefault( - const.OS_CONFIG, {} - ) - self.package_config = self.cluster_info.setdefault( - const.PK_CONFIG, {} - ) - self.deployed_os_config = self.cluster_info.setdefault( - const.DEPLOYED_OS_CONFIG, {} - ) - self.deployed_package_config = self.cluster_info.setdefault( - const.DEPLOYED_PK_CONFIG, {} - ) - self.network_mapping = self.package_config.setdefault( - const.NETWORK_MAPPING, {} - ) - - os_config_general = self.os_config.setdefault( - const.OS_CONFIG_GENERAL, {} - ) - self.domain = os_config_general.setdefault(const.DOMAIN, None) - self.hosts = [] - - def add_host(self, host): - self.hosts.append(host) - - @property - def roles_mapping(self): - deploy_config = self.deployed_package_config - return deploy_config.setdefault( - const.ROLES_MAPPING, self._get_cluster_roles_mapping() - ) - - def _get_cluster_roles_mapping(self): - """The ouput format will be as below, for example: - - { - "controller": [{ - "hostname": "xxx", - "management": { - "interface": "eth0", - "ip": "192.168.1.10", - "netmask": "255.255.255.0", - "subnet": "192.168.1.0/24", - "is_mgmt": True, - "is_promiscuous": False - }, - ... - }], - ... - } - """ - mapping = defaultdict(list) - for host in self.hosts: - for role, value in host.roles_mapping.iteritems(): - mapping[role].append(value) - - return dict(mapping) - - def _get_cluster_patched_roles_mapping(self): - mapping = defaultdict(list) - for host in self.hosts: - for role, value in host.patched_roles_mapping.iteritems(): - mapping[role].append(value) - - return dict(mapping) - - @property - def base_info(self): - return { - const.ID: self.id, - const.NAME: self.name, - const.OS_VERSION: self.os_version - } - - -class HostInfo(object): - def __init__(self, host_info, cluster_info): - self.host_info = host_info - self.cluster_info = cluster_info - self.id = self.host_info.get(const.ID) - self.name = self.host_info.get(const.NAME) - self.mac = self.host_info.get(const.MAC_ADDR) - self.hostname = self.host_info.get(const.HOSTNAME) - self.networks = self.host_info.setdefault(const.NETWORKS, {}) - self.os_config = self.host_info.setdefault(const.OS_CONFIG, {}) - - self.package_config = self.host_info.setdefault(const.PK_CONFIG, {}) - self.roles = self.host_info.setdefault(const.ROLES, []) - self.patched_roles = self.host_info.setdefault(const.PATCHED_ROLES, []) - self.ipmi = deepcopy(self.host_info.setdefault(const.IPMI, {})) - self.reinstall_os_flag = self.host_info.get(const.REINSTALL_OS_FLAG) - self.deployed_os_config = self.host_info.setdefault( - const.DEPLOYED_OS_CONFIG, {} - ) - self.deployed_package_config = self.host_info.setdefault( - const.DEPLOYED_PK_CONFIG, {} - ) - - os_general_config = self.os_config.setdefault( - const.OS_CONFIG_GENERAL, {} - ) - domain = os_general_config.setdefault(const.DOMAIN, None) - if domain is None: - self.domain = self.cluster_info.domain - else: - self.domain = domain - - if const.DNS in host_info: - self.dns = host_info[const.DNS] - else: - self.dns = '.'.join((self.hostname, self.domain)) - - if const.NETWORK_MAPPING not in self.package_config: - self.network_mapping = self.cluster_info.network_mapping - else: - self.network_mapping = self.package_config[const.NETWORK_MAPPING] - - if const.ROLES_MAPPING not in self.deployed_package_config: - self.roles_mapping = self._get_host_roles_mapping() - self.deployed_package_config[ - const.ROLES_MAPPING - ] = self.roles_mapping - else: - self.roles_mapping = \ - self.deployed_package_config[const.ROLES_MAPPING] - - self.patched_roles_mapping = self._get_host_patched_roles_mapping() - - self.cluster_info.add_host(self) - - def valid_interface(self, interface): - if interface not in self.networks: - raise RuntimeError("interface %s is invalid" % interface) - - def get_interface(self, interface): - self.valid_interface(interface) - return self.networks[interface] - - def get_interface_ip(self, interface): - return self.get_interface(interface).get(const.IP_ADDR) - - def get_interface_netmask(self, interface): - return self.get_interface(interface).get(const.NETMASK) - - def get_interface_subnet(self, interface): - return self.get_interface(interface).get(const.SUBNET) - - def is_interface_promiscuous(self, interface): - return self.get_interface(interface).get(const.PROMISCUOUS_FLAG) - - def is_interface_mgmt(self, interface): - return self.get_interface(interface).get(const.MGMT_NIC_FLAG) - - def _get_host_roles_mapping(self): - if not self.network_mapping: - return {} - - net_info = {const.HOSTNAME: self.hostname} - for k, v in self.network_mapping.items(): - try: - net_info[k] = self.networks[v[const.NIC]] - net_info[k][const.NIC] = v[const.NIC] - except Exception: - pass - - mapping = {} - for role in self.roles: - role = role.replace("-", "_") - mapping[role] = net_info - - return mapping - - def _get_host_patched_roles_mapping(self): - if not self.network_mapping: - return {} - - net_info = {const.HOSTNAME: self.hostname} - for k, v in self.network_mapping.items(): - try: - net_info[k] = self.networks[v[const.NIC]] - net_info[k][const.NIC] = v[const.NIC] - except Exception: - pass - - mapping = {} - for role in self.patched_roles: - role = role['name'].replace("-", "_") - mapping[role] = net_info - - return mapping - - @property - def baseinfo(self): - return { - const.REINSTALL_OS_FLAG: self.reinstall_os_flag, - const.MAC_ADDR: self.mac, - const.NAME: self.name, - const.HOSTNAME: self.hostname, - const.DNS: self.dns, - const.NETWORKS: deepcopy(self.networks) - } - - -class BaseConfigManager(object): - def __init__(self, adapter_info={}, cluster_info={}, hosts_info={}): - assert(adapter_info and isinstance(adapter_info, dict)) - assert(cluster_info and isinstance(cluster_info, dict)) - assert(hosts_info and isinstance(hosts_info, dict)) - - self.adapter_info = AdapterInfo(adapter_info) - self.cluster_info = ClusterInfo(cluster_info) - self.hosts_info = dict([(k, HostInfo(v, self.cluster_info)) - for k, v in hosts_info.iteritems()]) - - def get_adapter_name(self): - return self.adapter_info.name - - def get_dist_system_name(self): - return self.adapter_info.dist_system_name - - def get_adapter_health_check_cmd(self): - return self.adapter_info.health_check_cmd - - def get_os_installer_settings(self): - return self.adapter_info.os_installer[const.INSTALLER_SETTINGS] - - def get_pk_installer_settings(self): - return self.adapter_info.package_installer[const.INSTALLER_SETTINGS] - - def get_os_config_metadata(self): - return self.adapter_info.metadata[const.OS_CONFIG] - - def get_pk_config_meatadata(self): - return self.adapter_info.metadata[const.PK_CONFIG] - - def get_adapter_all_flavors(self): - return self.adapter_info.flavor_list - - def get_adapter_flavor(self, flavor_name): - return self.adapter_info.get_flavor(flavor_name) - - def get_cluster_id(self): - return self.cluster_info.id - - def get_clustername(self): - return self.cluster_info.name - - def get_os_version(self): - return self.cluster_info.os_version - - def get_cluster_os_config(self): - return self.cluster_info.os_config - - def get_cluster_baseinfo(self): - return self.cluster_info.base_info - - def get_cluster_flavor_name(self): - return self.cluster_info.flavor.get(const.FLAVOR_NAME) - - def get_cluster_flavor_roles(self): - return self.cluster_info.flavor.get(const.ROLES, []) - - def get_cluster_flavor_template(self): - return self.cluster_info.flavor.get(const.TMPL) - - def get_cluster_package_config(self): - return self.cluster_info.package_config - - def get_cluster_network_mapping(self): - mapping = self.cluster_info.network_mapping - logging.info("Network mapping in the config is '%s'!", mapping) - return mapping - - def get_cluster_deployed_os_config(self): - return self.cluster_info.deployed_os_config - - def get_cluster_deployed_package_config(self): - return self.cluster_info.deployed_package_config - - def get_cluster_roles_mapping(self): - return self.cluster_info.roles_mapping - - def get_cluster_patched_roles_mapping(self): - return self.cluster_info._get_cluster_patched_roles_mapping() - - def validate_host(self, host_id): - if host_id not in self.hosts_info: - raise RuntimeError("host_id %s is invalid" % host_id) - - def get_host_id_list(self): - return self.hosts_info.keys() - - def get_hosts_id_list_for_os_installation(self): - """Get info of hosts which need to install/reinstall OS.""" - return [ - id for id, info in self.hosts_info.items() - if info.reinstall_os_flag - ] - - def get_server_credentials(self): - cluster_os_config = self.get_cluster_os_config() - if not cluster_os_config: - logging.info("cluster os_config is None!") - return () - - username = cluster_os_config[const.SERVER_CREDS][const.USERNAME] - password = cluster_os_config[const.SERVER_CREDS][const.PASSWORD] - return (username, password) - - def _get_host_info(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id] - - def get_host_baseinfo(self, host_id): - self.validate_host(host_id) - host_info = self.hosts_info[host_id] - return host_info.baseinfo - - def get_host_fullname(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].name - - def get_host_dns(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].dns - - def get_host_mac_address(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].mac - - def get_hostname(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].hostname - - def get_host_networks(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].networks - - def get_host_interfaces(self, host_id): - # get interface names - return self.get_host_networks(host_id).keys() - - def get_host_interface_ip(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_ip(interface) - - def get_host_interface_netmask(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_netmask(interface) - - def get_host_interface_subnet(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].get_interface_subnet(interface) - - def is_interface_promiscuous(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].is_interface_promiscuous(interface) - - def is_interface_mgmt(self, host_id, interface): - self.validate_host(host_id) - return self.hosts_info[host_id].is_interface_mgmt(interface) - - def get_host_os_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].os_config - - def get_host_domain(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].domain - - def get_host_network_mapping(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].network_mapping - - def get_host_package_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].package_config - - def get_host_deployed_os_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].deployed_os_config - - def get_host_deployed_package_config(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].deployed_package_config - - def get_host_roles(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].roles - - def get_all_hosts_roles(self, hosts_id_list=None): - roles = [] - for host_id, host_info in self.hosts_info.iteritems(): - roles.extend(host_info.roles) - - return list(set(roles)) - - def get_hosts_ip_settings(self, ip_settings, sys_intf_mappings): - logging.info( - "get_hosts_ip_settings:ip_settings=%s, sys_intf_mappings=%s" % - (ip_settings, sys_intf_mappings) - ) - - intf_alias = {} - for m in sys_intf_mappings: - if "vlan_tag" in m: - intf_alias[m["name"]] = m["name"] - else: - intf_alias[m["name"]] = m["interface"] - - mappings = {} - hosts_id_list = self.get_host_id_list() - for host_id in hosts_id_list: - hostname = self.get_hostname(host_id) - mappings[hostname] = [] - for ip_info in ip_settings: - logging.info("ip_info=%s" % ip_info) - new_ip_info = deepcopy(ip_info) - del new_ip_info["ip_ranges"] - - ip_ranges = ip_info["ip_ranges"] - new_ip_info["netmask"] = netaddr.IPNetwork( - ip_info["cidr"] - ).netmask.bin.count("1") - new_ip_info["ip"] = get_ip_addr(ip_ranges).next() - new_ip_info["alias"] = intf_alias[ip_info["name"]] - mappings[hostname].append(new_ip_info) - - return {"ip_settings": mappings} - - def get_host_roles_mapping(self, host_id): - self.validate_host(host_id) - return self.hosts_info[host_id].roles_mapping - - def get_host_ipmi_info(self, host_id): - self.validate_host(host_id) - if self.hosts_info[host_id].ipmi: - return ( - self.hosts_info[host_id].ipmi[const.IP_ADDR], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME], - self.hosts_info[host_id].ipmi - [const.IPMI_CREDS][const.USERNAME]) - else: - return (None, None, None) diff --git a/compass-tasks/deployment/installers/installer.py b/compass-tasks/deployment/installers/installer.py deleted file mode 100644 index cfeb9e8..0000000 --- a/compass-tasks/deployment/installers/installer.py +++ /dev/null @@ -1,291 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""Module to provider installer interface. -""" -from Cheetah.Template import Template -from copy import deepcopy -import imp -import logging -import os -import simplejson as json - -from compass.deployment.installers.config_manager import BaseConfigManager -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util - - -CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) - - -class BaseInstaller(object): - """Interface for installer.""" - NAME = 'installer' - - def __repr__(self): - return '%r[%r]' % (self.__class__.__name__, self.NAME) - - def deploy(self, **kwargs): - """virtual method to start installing process.""" - raise NotImplementedError - - def clean_progress(self, **kwargs): - raise NotImplementedError - - def delete_hosts(self, **kwargs): - """Delete hosts from installer server.""" - raise NotImplementedError - - def redeploy(self, **kwargs): - raise NotImplementedError - - def ready(self, **kwargs): - pass - - def cluster_ready(self, **kwargs): - pass - - def get_tmpl_vars_from_metadata(self, metadata, config): - """Get variables dictionary for rendering templates from metadata. - - :param dict metadata: The metadata dictionary. - :param dict config: The - """ - template_vars = {} - self._get_tmpl_vars_helper(metadata, config, template_vars) - - return template_vars - - def _get_key_mapping(self, metadata, key, is_regular_key): - """Get the keyword which the input key maps to. - - This keyword will be added to dictionary used to render templates. - - If the key in metadata has a mapping to another keyword which is - used for templates, then return this keyword. If the key is started - with '$', which is a variable in metadata, return the key itself as - the mapping keyword. If the key has no mapping, return None. - - :param dict metadata: metadata/submetadata dictionary. - :param str key: The keyword defined in metadata. - :param bool is_regular_key: False when the key defined in metadata - is a variable(starting with '$'). - """ - mapping_to = key - if is_regular_key: - try: - mapping_to = metadata['_self']['mapping_to'] - except Exception: - mapping_to = None - - return mapping_to - - def _get_submeta_by_key(self, metadata, key): - """Get submetadata dictionary. - - Based on current metadata key. And - determines the input key is a regular string keyword or a variable - keyword defined in metadata, which starts with '$'. - - :param dict metadata: The metadata dictionary. - :param str key: The keyword defined in the metadata. - """ - if key in metadata: - return (True, metadata[key]) - - temp = deepcopy(metadata) - if '_self' in temp: - del temp['_self'] - meta_key = temp.keys()[0] - if meta_key.startswith("$"): - return (False, metadata[meta_key]) - - raise KeyError("'%s' is invalid in metadata '%s'!" % (key, metadata)) - - def _get_tmpl_vars_helper(self, metadata, config, output): - for key, config_value in sorted(config.iteritems()): - is_regular_key, sub_meta = self._get_submeta_by_key(metadata, key) - mapping_to = self._get_key_mapping(sub_meta, key, is_regular_key) - - if isinstance(config_value, dict): - if mapping_to: - new_output = output[mapping_to] = {} - else: - new_output = output - - self._get_tmpl_vars_helper(sub_meta, config_value, new_output) - - elif mapping_to: - output[mapping_to] = config_value - - def get_config_from_template(self, tmpl_path, vars_dict): - logging.debug("template path is %s", tmpl_path) - logging.debug("vars_dict is %s", vars_dict) - - if not os.path.exists(tmpl_path) or not vars_dict: - logging.info("Template dir or vars_dict is None!") - return {} - - searchList = [] - copy_vars_dict = deepcopy(vars_dict) - for key, value in vars_dict.iteritems(): - if isinstance(value, dict): - temp = copy_vars_dict[key] - del copy_vars_dict[key] - searchList.append(temp) - searchList.append(copy_vars_dict) - - # Load base template first if it exists - base_config = {} - base_tmpl_path = os.path.join(os.path.dirname(tmpl_path), 'base.tmpl') - if os.path.isfile(base_tmpl_path) and base_tmpl_path != tmpl_path: - base_tmpl = Template(file=base_tmpl_path, searchList=searchList) - base_config = json.loads(base_tmpl.respond(), encoding='utf-8') - base_config = json.loads(json.dumps(base_config), encoding='utf-8') - - # Load specific template for current adapter - tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) - config = json.loads(tmpl.respond(), encoding='utf-8') - config = json.loads(json.dumps(config), encoding='utf-8') - - # Merge the two outputs - config = util.merge_dict(base_config, config) - - logging.debug("get_config_from_template resulting %s", config) - return config - - @classmethod - def get_installer(cls, name, path, adapter_info, cluster_info, hosts_info): - try: - mod_file, path, descr = imp.find_module(name, [path]) - if mod_file: - mod = imp.load_module(name, mod_file, path, descr) - config_manager = BaseConfigManager(adapter_info, cluster_info, - hosts_info) - return getattr(mod, mod.NAME)(config_manager) - - except ImportError as exc: - logging.error('No such module found: %s', name) - logging.exception(exc) - - return None - - -class OSInstaller(BaseInstaller): - """Interface for os installer.""" - NAME = 'OSInstaller' - INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'os_installers') - - def get_oses(self): - """virtual method to get supported oses. - - :returns: list of str, each is the supported os version. - """ - return [] - - @classmethod - def get_installer(cls, name, adapter_info, cluster_info, hosts_info): - if name is None: - logging.info("Installer name is None! No OS installer loaded!") - return None - - path = os.path.join(cls.INSTALLER_BASE_DIR, name) - installer = super(OSInstaller, cls).get_installer(name, path, - adapter_info, - cluster_info, - hosts_info) - - if not isinstance(installer, OSInstaller): - logging.info("Installer '%s' is not an OS installer!" % name) - return None - - return installer - - def poweron(self, host_id): - pass - - def poweroff(self, host_id): - pass - - def reset(self, host_id): - pass - - -class PKInstaller(BaseInstaller): - """Interface for package installer.""" - NAME = 'PKInstaller' - INSTALLER_BASE_DIR = os.path.join(CURRENT_DIR, 'pk_installers') - - def generate_installer_config(self): - raise NotImplementedError( - 'generate_installer_config is not defined in %s', - self.__class__.__name__ - ) - - def get_target_systems(self): - """virtual method to get available target_systems for each os. - - :param oses: supported os versions. - :type oses: list of st - - :returns: dict of os_version to target systems as list of str. - """ - return {} - - def get_roles(self, target_system): - """virtual method to get all roles of given target system. - - :param target_system: target distributed system such as OpenStack. - :type target_system: str - - :returns: dict of role to role description as str. - """ - return {} - - def os_ready(self, **kwargs): - pass - - def cluster_os_ready(self, **kwargs): - pass - - def serialize_config(self, config, destination): - with open(destination, "w") as f: - f.write(config) - - @classmethod - def get_installer(cls, name, adapter_info, cluster_info, hosts_info): - if name is None: - logging.info("Install name is None. No package installer loaded!") - return None - - path = os.path.join(cls.INSTALLER_BASE_DIR, name) - if not os.path.exists(path): - path = os.path.join(os.path.join(os.path.join( - compass_setting.PLUGINS_DIR, name), "implementation"), name) - if not os.path.exists(path): - logging.info("Installer '%s' does not exist!" % name) - return None - installer = super(PKInstaller, cls).get_installer(name, path, - adapter_info, - cluster_info, - hosts_info) - - if not isinstance(installer, PKInstaller): - logging.info("Installer '%s' is not a package installer!" % name) - return None - - return installer diff --git a/compass-tasks/deployment/installers/os_installers/__init__.py b/compass-tasks/deployment/installers/os_installers/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/os_installers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py b/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/os_installers/cobbler/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py b/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py deleted file mode 100644 index 9c2a935..0000000 --- a/compass-tasks/deployment/installers/os_installers/cobbler/cobbler.py +++ /dev/null @@ -1,449 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""os installer cobbler plugin. -""" -import logging -import os -import shutil -import xmlrpclib - -from compass.deployment.installers.installer import OSInstaller -from compass.deployment.utils import constants as const -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util -from copy import deepcopy - - -NAME = 'CobblerInstaller' - - -class CobblerInstaller(OSInstaller): - """cobbler installer""" - CREDENTIALS = "credentials" - USERNAME = 'username' - PASSWORD = 'password' - - INSTALLER_URL = "cobbler_url" - TMPL_DIR = 'tmpl_dir' - SYS_TMPL = 'system.tmpl' - SYS_TMPL_NAME = 'system.tmpl' - SYS_PROFILE_NAME = 'profile.tmpl' - PROFILE = 'profile' - - POWER_TYPE = 'power_type' - POWER_ADDR = 'power_address' - POWER_USER = 'power_user' - POWER_PASS = 'power_pass' - - def __init__(self, config_manager): - super(CobblerInstaller, self).__init__() - - self.config_manager = config_manager - installer_settings = self.config_manager.get_os_installer_settings() - try: - username = installer_settings[self.CREDENTIALS][self.USERNAME] - password = installer_settings[self.CREDENTIALS][self.PASSWORD] - cobbler_url = installer_settings[self.INSTALLER_URL] - self.tmpl_dir = CobblerInstaller.get_tmpl_path() - - except KeyError as ex: - raise KeyError(ex.message) - - # The connection is created when cobbler installer is initialized. - self.remote = self._get_cobbler_server(cobbler_url) - self.token = self._get_token(username, password) - self.pk_installer_config = None - - logging.debug('%s instance created', 'CobblerInstaller') - - @classmethod - def get_tmpl_path(cls): - return os.path.join(compass_setting.TMPL_DIR, 'cobbler') - - def __repr__(self): - return '%r[remote=%r,token=%r' % ( - self.__class__.__name__, self.remote, self.token) - - def _get_cobbler_server(self, cobbler_url): - if not cobbler_url: - logging.error("Cobbler URL is None!") - raise Exception("Cobbler URL cannot be None!") - - return xmlrpclib.Server(cobbler_url) - - def _get_token(self, username, password): - if self.remote is None: - raise Exception("Cobbler remote instance is None!") - return self.remote.login(username, password) - - def get_supported_oses(self): - """get supported os versions. - - note:: - In cobbler, we treat profile name as the indicator - of os version. It is just a simple indicator - and not accurate. - """ - profiles = self.remote.get_profiles() - oses = [] - for profile in profiles: - oses.append(profile['name']) - return oses - - def deploy(self): - """Sync cobbler to catch up the latest update config and start to - - install OS. Return both cluster and hosts deploy configs. The return - format: - { - "cluster": { - "id": 1, - "deployed_os_config": {}, - }, - "hosts": { - 1($clusterhost_id): { - "deployed_os_config": {...}, - }, - .... - } - } - """ - host_ids = self.config_manager.get_hosts_id_list_for_os_installation() - if not host_ids: - # No hosts need to install OS - logging.info("Cobbler: No host needs to install OS.") - return {} - - os_version = self.config_manager.get_os_version() - profile = self._get_profile_from_server(os_version) - - global_vars_dict = self._get_cluster_tmpl_vars_dict() - - self.update_profile_config_to_cobbler(profile, global_vars_dict) - - hosts_deploy_config = {} - - for host_id in host_ids: - hostname = self.config_manager.get_hostname(host_id) - vars_dict = self._get_host_tmpl_vars_dict(host_id, - global_vars_dict, - hostname=hostname, - profile=profile) - - self.update_host_config_to_cobbler(host_id, hostname, vars_dict) - - # set host deploy config - host_config = {} - host_config[const.DEPLOYED_OS_CONFIG] = vars_dict[const.OS_CONFIG] - hosts_deploy_config[host_id] = host_config - - # sync to cobbler and trigger installtion. - self._sync() - - cluster_config = global_vars_dict.setdefault(const.OS_CONFIG, {}) - - return { - const.CLUSTER: { - const.ID: self.config_manager.get_cluster_id(), - const.DEPLOYED_OS_CONFIG: cluster_config - }, - const.HOSTS: hosts_deploy_config - } - - def clean_progress(self): - """clean log files and config for hosts which to deploy.""" - clusterhost_list = self.config_manager.get_host_id_list() - log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] - - for host_id in clusterhost_list: - hostname = self.config_manager.get_hostname(host_id) - self._clean_log(log_dir_prefix, hostname) - - def redeploy(self): - """redeploy hosts.""" - host_ids = self.config_manager.get_host_id_list() - if not host_ids: - logging.info("Cobbler: hostlist is None, no host is redeployed") - return - for host_id in host_ids: - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if sys_id: - # enable netboot for this host - self._netboot_enabled(sys_id) - - self._sync() - - def set_package_installer_config(self, package_configs): - """Cobbler can install and configure package installer right after - - OS installation compelets by setting package_config info provided - by package installer. - - :param dict package_configs: The dict of config generated by package - installer for each clusterhost. The IDs - of clusterhosts are the keys of - package_configs. - """ - self.pk_installer_config = package_configs - - def _sync(self): - """Sync the updated config to cobbler and trigger installation.""" - try: - self.remote.sync(self.token) - os.system('sudo service rsyslog restart') - except Exception as ex: - logging.debug("Failed to sync cobbler server! Error: %s" % ex) - raise ex - - def dump_system_info(self, host_id): - - hostname = self.config_manager.get_hostname(host_id) - if self.remote is None or not hostname: - logging.info("[dump_system_info]Remote or hostname is None.") - return {} - - return self.remote.get_system_as_rendered(hostname) - - def _generate_system_config(self, host_id, host_vars_dict): - """Generate updated system config from the template. - - :param host_vars_dict: dict of variables for the system template to - generate system config dict for each host. - """ - os_version = self.config_manager.get_os_version() - - tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, os_version), self.SYS_TMPL_NAME - ) - if not os.path.exists(tmpl_path): - err_msg = "Template '%s' does not exists!" % tmpl_path - logging.error(err_msg) - raise Exception(err_msg) - host_vars_dict[const.BASEINFO]['host_id'] = host_id - system_config = self.get_config_from_template(tmpl_path, - host_vars_dict) - - # update package config info to cobbler ksmeta - if self.pk_installer_config and host_id in self.pk_installer_config: - pk_config = self.pk_installer_config[host_id] - ksmeta = system_config.setdefault("ksmeta", {}) - util.merge_dict(ksmeta, pk_config) - system_config["ksmeta"] = ksmeta - - return system_config - - def _generate_profile_config(self, cluster_vars_dict): - os_version = self.config_manager.get_os_version() - tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, os_version), self.SYS_PROFILE_NAME - ) - - return self.get_config_from_template(tmpl_path, cluster_vars_dict) - - def _get_profile_from_server(self, os_version): - """Get profile from cobbler server.""" - result = self.remote.find_profile({'name': os_version}) - if not result: - raise Exception("Cannot find profile for '%s'", os_version) - - profile = result[0] - return profile - - def _get_create_system(self, hostname): - """get system reference id for the host.""" - sys_name = hostname - sys_id = None - system_info = self.remote.find_system({"name": hostname}) - - if not system_info: - # Create a new system - sys_id = self.remote.new_system(self.token) - self.remote.modify_system(sys_id, "name", hostname, self.token) - logging.debug('create new system %s for %s', sys_id, sys_name) - else: - sys_id = self.remote.get_system_handle(sys_name, self.token) - - return sys_id - - def _get_profile_id(self, profilename): - """get profile reference id for the cluster.""" - return self.remote.get_profile_handle(profilename, self.token) - - def _clean_system(self, hostname): - """clean system.""" - sys_name = hostname - try: - self.remote.remove_system(sys_name, self.token) - logging.debug('system %s is removed', sys_name) - except Exception: - logging.debug('no system %s found to remove', sys_name) - - def _update_system_config(self, sys_id, system_config): - """update modify system.""" - for key, value in system_config.iteritems(): - self.remote.modify_system(sys_id, str(key), value, self.token) - - self.remote.save_system(sys_id, self.token) - - def _update_profile_config(self, profile_id, profile_config): - for key, value in profile_config.iteritems(): - self.remote.modify_profile(profile_id, str(key), value, self.token) - - self.remote.save_profile(profile_id, self.token) - - def _netboot_enabled(self, sys_id): - """enable netboot.""" - self.remote.modify_system(sys_id, 'netboot_enabled', True, self.token) - self.remote.save_system(sys_id, self.token) - - def _clean_log(self, log_dir_prefix, system_name): - """clean log.""" - log_dir = os.path.join(log_dir_prefix, system_name) - shutil.rmtree(log_dir, True) - - def update_host_config_to_cobbler(self, host_id, hostname, host_vars_dict): - """update host config and upload to cobbler server.""" - sys_id = self._get_create_system(hostname) - - system_config = self._generate_system_config(host_id, host_vars_dict) - logging.debug('%s system config to update: %s', host_id, system_config) - - self._update_system_config(sys_id, system_config) - self._netboot_enabled(sys_id) - - def update_profile_config_to_cobbler(self, profilename, cluster_vars_dict): - """update profile config and upload to cobbler server.""" - - profile_id = self._get_profile_id(profilename) - - profile_config = self._generate_profile_config(cluster_vars_dict) - logging.debug( - '%s profile config to update: %s', profilename, profile_config - ) - - self._update_profile_config(profile_id, profile_config) - - def delete_hosts(self): - hosts_id_list = self.config_manager.get_host_id_list() - logging.debug('delete hosts %s', hosts_id_list) - for host_id in hosts_id_list: - self.delete_single_host(host_id) - self._sync() - - def delete_single_host(self, host_id): - """Delete the host from cobbler server and clean up the installation - - progress. - """ - hostname = self.config_manager.get_hostname(host_id) - try: - log_dir_prefix = compass_setting.INSTALLATION_LOGDIR[NAME] - self._clean_system(hostname) - self._clean_log(log_dir_prefix, hostname) - except Exception as ex: - logging.error("Deleting host got exception: %s", ex) - logging.exception(ex) - - def _get_host_tmpl_vars_dict(self, host_id, global_vars_dict, **kwargs): - """Generate template variables dictionary.""" - vars_dict = {} - if global_vars_dict: - # Set cluster template vars_dict from cluster os_config. - vars_dict = deepcopy(global_vars_dict) - - # Set hostname, MAC address and hostname, networks, dns and so on. - host_baseinfo = self.config_manager.get_host_baseinfo(host_id) - vars_dict[const.BASEINFO] = host_baseinfo - - # Set profile - if self.PROFILE in kwargs: - profile = kwargs[self.PROFILE] - else: - os_version = self.config_manager.get_os_version() - profile = self._get_profile_from_server(os_version) - - vars_dict[const.BASEINFO][self.PROFILE] = profile - - metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_host_os_config(host_id) - - # Get template variables values from host os_config - host_vars_dict = self.get_tmpl_vars_from_metadata(metadata, os_config) - util.merge_dict( - vars_dict.setdefault(const.OS_CONFIG, {}), host_vars_dict - ) - return vars_dict - - def _get_cluster_tmpl_vars_dict(self): - metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_cluster_os_config() - - cluster_vas_dict = {} - cluster_vas_dict[const.OS_CONFIG] = \ - self.get_tmpl_vars_from_metadata(metadata, os_config) - - return cluster_vas_dict - - def _check_and_set_system_impi(self, host_id, sys_id): - if not sys_id: - logging.info("System is None!") - return False - - system = self.dump_system_info(host_id) - if system[self.POWER_TYPE] != 'ipmilan' or not system[self.POWER_USER]: - # Set sytem power type to ipmilan if needs and set IPMI info - ipmi_info = self.config_manager.get_host_ipmi_info(host_id) - if not ipmi_info: - logging.info('No IPMI information found! Failed power on.') - return False - - ipmi_ip, ipmi_user, ipmi_pass = ipmi_info - power_opts = {} - power_opts[self.POWER_TYPE] = 'ipmilan' - power_opts[self.POWER_ADDR] = ipmi_ip - power_opts[self.POWER_USER] = ipmi_user - power_opts[self.POWER_PASS] = ipmi_pass - - self._update_system_config(sys_id, power_opts) - - return True - - def poweron(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='on') - logging.info("Host with ID=%d starts to power on!" % host_id) - - def poweroff(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='off') - logging.info("Host with ID=%d starts to power off!" % host_id) - - def reset(self, host_id): - hostname = self.config_manager.get_hostname(host_id) - sys_id = self._get_create_system(hostname) - if not self._check_and_set_system_impi(sys_id): - return - - self.remote.power_system(sys_id, self.token, power='reboot') - logging.info("Host with ID=%d starts to reboot!" % host_id) diff --git a/compass-tasks/deployment/installers/pk_installers/__init__.py b/compass-tasks/deployment/installers/pk_installers/__init__.py deleted file mode 100644 index 5e42ae9..0000000 --- a/compass-tasks/deployment/installers/pk_installers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/compass-tasks/deployment/installers/pk_installers/ansible_installer/__init__.py +++ /dev/null diff --git a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py b/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py deleted file mode 100644 index 0a86be4..0000000 --- a/compass-tasks/deployment/installers/pk_installers/ansible_installer/ansible_installer.py +++ /dev/null @@ -1,441 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__auther__ = "Compass Dev Team (dev-team@syscompass.org)" - -"""package installer: ansible plugin.""" - -from Cheetah.Template import Template -from copy import deepcopy -import json -import logging -import os -import re -import shutil -import subprocess - -from compass.deployment.installers.installer import PKInstaller -from compass.deployment.utils import constants as const -from compass.utils import setting_wrapper as compass_setting -from compass.utils import util - -NAME = "AnsibleInstaller" - - -def byteify(input): - if isinstance(input, dict): - return dict([(byteify(key), byteify(value)) - for key, value in input.iteritems()]) - elif isinstance(input, list): - return [byteify(element) for element in input] - elif isinstance(input, unicode): - return input.encode('utf-8') - else: - return input - - -class AnsibleInstaller(PKInstaller): - INVENTORY_TMPL_DIR = 'inventories' - GROUPVARS_TMPL_DIR = 'vars' - INVENTORY_PATCH_TEMPALTE_DIR = 'inventories' - - # keywords in package installer settings - ANSIBLE_DIR = 'ansible_dir' - ANSIBLE_RUN_DIR = 'ansible_run_dir' - LOG_FILE = 'ansible_log_file' - ANSIBLE_CONFIG = 'ansible_config' - INVENTORY = 'inventory_file' - INVENTORY_JSON = 'inventory_json_file' - INVENTORY_GROUP = 'inventory_group' - GROUP_VARIABLE = 'group_variable' - HOSTS_PATH = 'etc_hosts_path' - RUNNER_DIRS = 'runner_dirs' - - def __init__(self, config_manager): - super(AnsibleInstaller, self).__init__() - - self.config_manager = config_manager - self.tmpl_name = self.config_manager.get_cluster_flavor_template() - self.installer_settings = ( - self.config_manager.get_pk_installer_settings() - ) - settings = self.installer_settings - self.ansible_dir = settings.setdefault(self.ANSIBLE_DIR, None) - self.ansible_run_dir = ( - settings.setdefault(self.ANSIBLE_RUN_DIR, None) - ) - self.log_file = settings.setdefault(self.LOG_FILE, None) - self.ansible_config = ( - settings.setdefault(self.ANSIBLE_CONFIG, None) - ) - self.inventory = settings.setdefault(self.INVENTORY, None) - self.inventory_json = settings.setdefault(self.INVENTORY_JSON, None) - self.inventory_group = settings.setdefault(self.INVENTORY_GROUP, None) - self.group_variable = ( - settings.setdefault(self.GROUP_VARIABLE, None) - ) - self.hosts_path = ( - settings.setdefault(self.HOSTS_PATH, None) - ) - self.runner_dirs = ( - settings.setdefault(self.RUNNER_DIRS, None) - ) - self.playbook = self.tmpl_name.replace('tmpl', 'yml') - self.runner_files = [self.playbook] - - adapter_name = self.config_manager.get_dist_system_name() - self.tmpl_dir = AnsibleInstaller.get_tmpl_path(adapter_name) - self.adapter_dir = os.path.join(self.ansible_dir, adapter_name) - logging.debug('%s instance created', self) - - @classmethod - def get_tmpl_path(cls, adapter_name): - tmpl_path = os.path.join( - os.path.join(compass_setting.TMPL_DIR, 'ansible_installer'), - adapter_name - ) - return tmpl_path - - def __repr__(self): - return '%s[name=%s,installer_url=%s]' % ( - self.__class__.__name__, self.NAME, self.installer_url) - - def dump_inventory(self, data, inventory): - with open(inventory, "w") as f: - json.dump(data, f, indent=4) - - def _generate_inventory_data(self, global_vars_dict): - vars_dict = global_vars_dict['roles_mapping'] - inventory_data = {} - inventory_data['_meta'] = {'hostvars': {}} - for item in self.inventory_group: - if item in vars_dict: - inventory_data[item] = {'hosts': []} - for host in vars_dict[item]: - hostname = host['hostname'] - if hostname not in inventory_data['_meta']['hostvars']: - host_dict = {} - host_dict['ansible_ssh_host'] = host['install']['ip'] - host_dict['ansible_ssh_user'] = 'root' - host_dict['ansible_ssh_pass'] = 'root' - inventory_data['_meta']['hostvars'].update( - {hostname: host_dict}) - inventory_data[item]['hosts'].append(hostname) - - inventory_data['ceph'] = {'children': - ['ceph_adm', 'ceph_mon', 'ceph_osd']} - return inventory_data - - def generate_installer_config(self): - """Render ansible config file by OS installing. - - The output format: - { - '1'($host_id/clusterhost_id):{ - 'tool': 'ansible', - }, - ..... - } - """ - host_ids = self.config_manager.get_host_id_list() - os_installer_configs = {} - for host_id in host_ids: - temp = { - "tool": "ansible", - } - os_installer_configs[host_id] = temp - - return os_installer_configs - - def get_env_name(self, dist_sys_name, cluster_name): - return "-".join((dist_sys_name, cluster_name)) - - def _get_cluster_tmpl_vars(self): - """Generate template variables dict - - Generates based on cluster level config. - The vars_dict will be: - { - "baseinfo": { - "id":1, - "name": "cluster01", - ... - }, - "package_config": { - .... //mapped from original package config based on metadata - }, - "role_mapping": { - .... - } - } - """ - cluster_vars_dict = {} - # set cluster basic information to vars_dict - cluster_baseinfo = self.config_manager.get_cluster_baseinfo() - cluster_vars_dict[const.BASEINFO] = cluster_baseinfo - - # get and set template variables from cluster package config. - pk_metadata = self.config_manager.get_pk_config_meatadata() - pk_config = self.config_manager.get_cluster_package_config() - - # get os config as ansible needs them - os_metadata = self.config_manager.get_os_config_metadata() - os_config = self.config_manager.get_cluster_os_config() - - pk_meta_dict = self.get_tmpl_vars_from_metadata(pk_metadata, pk_config) - os_meta_dict = self.get_tmpl_vars_from_metadata(os_metadata, os_config) - util.merge_dict(pk_meta_dict, os_meta_dict) - - cluster_vars_dict[const.PK_CONFIG] = pk_meta_dict - - # get and set roles_mapping to vars_dict - mapping = self.config_manager.get_cluster_roles_mapping() - logging.info("cluster role mapping is %s", mapping) - cluster_vars_dict[const.ROLES_MAPPING] = mapping - - # get ip settings to vars_dict - hosts_ip_settings = self.config_manager.get_hosts_ip_settings( - pk_meta_dict["network_cfg"]["ip_settings"], - pk_meta_dict["network_cfg"]["sys_intf_mappings"] - ) - logging.info("hosts_ip_settings is %s", hosts_ip_settings) - cluster_vars_dict["ip_settings"] = hosts_ip_settings - - return byteify(cluster_vars_dict) - - def _generate_inventory_attributes(self, global_vars_dict): - inventory_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, self.INVENTORY_TMPL_DIR), - self.tmpl_name - ) - if not os.path.exists(inventory_tmpl_path): - logging.error( - "Inventory template '%s' does not exist", self.tmpl_name - ) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - inventory_dir = os.path.join(global_vars_dict['run_dir'], 'inventories') - inventory_json = os.path.join(inventory_dir, self.inventory_json) - vars_dict = {'inventory_json': inventory_json} - return self.get_config_from_template( - inventory_tmpl_path, vars_dict - ) - - def _generate_group_vars_attributes(self, global_vars_dict): - logging.info("global vars dict is %s", global_vars_dict) - group_vars_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, self.GROUPVARS_TMPL_DIR), - self.tmpl_name - ) - if not os.path.exists(group_vars_tmpl_path): - logging.error("Vars template '%s' does not exist", - self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template( - group_vars_tmpl_path, global_vars_dict - ) - - def _generate_hosts_attributes(self, global_vars_dict): - hosts_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, 'hosts'), self.tmpl_name - ) - if not os.path.exists(hosts_tmpl_path): - logging.error("Hosts template '%s' does not exist", self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template(hosts_tmpl_path, global_vars_dict) - - def _generate_ansible_cfg_attributes(self, global_vars_dict): - ansible_cfg_tmpl_path = os.path.join( - os.path.join(self.tmpl_dir, 'ansible_cfg'), self.tmpl_name - ) - if not os.path.exists(ansible_cfg_tmpl_path): - logging.error("cfg template '%s' does not exist", self.tmpl_name) - raise Exception("Template '%s' does not exist!" % self.tmpl_name) - - return self.get_config_from_template( - ansible_cfg_tmpl_path, - global_vars_dict - ) - - def get_config_from_template(self, tmpl_path, vars_dict): - logging.debug("vars_dict is %s", vars_dict) - - if not os.path.exists(tmpl_path) or not vars_dict: - logging.info("Template dir or vars_dict is None!") - return {} - - searchList = [] - copy_vars_dict = deepcopy(vars_dict) - for key, value in vars_dict.iteritems(): - if isinstance(value, dict): - temp = copy_vars_dict[key] - del copy_vars_dict[key] - searchList.append(temp) - searchList.append(copy_vars_dict) - - # Load specific template for current adapter - tmpl = Template(file=open(tmpl_path, "r"), searchList=searchList) - return tmpl.respond() - - def _create_ansible_run_env(self, env_name, ansible_run_destination): - if os.path.exists(ansible_run_destination): - shutil.rmtree(ansible_run_destination, True) - - os.mkdir(ansible_run_destination) - - # copy roles to run env - dirs = self.runner_dirs - files = self.runner_files - for dir in dirs: - if not os.path.exists(os.path.join(self.ansible_dir, dir)): - continue - os.system( - "cp -rf %s %s" % ( - os.path.join(self.ansible_dir, dir), - ansible_run_destination - ) - ) - for file in files: - logging.info('file is %s', file) - shutil.copy( - os.path.join(self.adapter_dir, file), - os.path.join( - ansible_run_destination, - file - ) - ) - - def prepare_ansible(self, env_name, global_vars_dict): - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - if os.path.exists(ansible_run_destination): - ansible_run_destination += "-expansion" - self._create_ansible_run_env(env_name, ansible_run_destination) - global_vars_dict.update({'run_dir': ansible_run_destination}) - - inv_config = self._generate_inventory_attributes(global_vars_dict) - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - - vars_config = self._generate_group_vars_attributes(global_vars_dict) - vars_dir = os.path.join(ansible_run_destination, 'group_vars') - - hosts_config = self._generate_hosts_attributes(global_vars_dict) - hosts_destination = os.path.join( - ansible_run_destination, self.hosts_path - ) - - cfg_config = self._generate_ansible_cfg_attributes(global_vars_dict) - cfg_destination = os.path.join( - ansible_run_destination, - self.ansible_config - ) - - inventory_data = self._generate_inventory_data(global_vars_dict) - inventory_json_destination = os.path.join(inventory_dir, - self.inventory_json) - - os.mkdir(inventory_dir) - os.mkdir(vars_dir) - - inventory_destination = os.path.join(inventory_dir, self.inventory) - group_vars_destination = os.path.join(vars_dir, self.group_variable) - self.dump_inventory(inventory_data, inventory_json_destination) - self.serialize_config(inv_config, inventory_destination) - self.serialize_config(vars_config, group_vars_destination) - self.serialize_config(hosts_config, hosts_destination) - self.serialize_config(cfg_config, cfg_destination) - - def deploy(self): - """Start to deploy a distributed system. - - Return both cluster and hosts deployed configs. - The return format: - { - "cluster": { - "id": 1, - "deployed_package_config": { - "roles_mapping": {...}, - "service_credentials": {...}, - .... - } - }, - "hosts": { - 1($clusterhost_id): { - "deployed_package_config": {...} - }, - .... - } - } - """ - host_list = self.config_manager.get_host_id_list() - if not host_list: - return {} - - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - - global_vars_dict = self._get_cluster_tmpl_vars() - logging.info( - '%s var dict: %s', self.__class__.__name__, global_vars_dict - ) - # Create ansible related files - self.prepare_ansible(env_name, global_vars_dict) - - def patch(self, patched_role_mapping): - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - patched_global_vars_dict = self._get_cluster_tmpl_vars() - mapping = self.config_manager.get_cluster_patched_roles_mapping() - patched_global_vars_dict['roles_mapping'] = mapping - patched_inv = self._generate_inventory_attributes( - patched_global_vars_dict) - inv_file = os.path.join(inventory_dir, 'patched_inventory.yml') - self.serialize_config(patched_inv, inv_file) - config_file = os.path.join( - ansible_run_destination, self.ansible_config - ) - playbook_file = os.path.join(ansible_run_destination, self.playbook) - log_file = os.path.join(ansible_run_destination, 'patch.log') - cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, - inv_file, - playbook_file) - with open(log_file, 'w') as logfile: - subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) - return patched_role_mapping - - def cluster_os_ready(self): - adapter_name = self.config_manager.get_adapter_name() - cluster_name = self.config_manager.get_clustername() - env_name = self.get_env_name(adapter_name, cluster_name) - ansible_run_destination = os.path.join(self.ansible_run_dir, env_name) - expansion_dir = ansible_run_destination + "-expansion" - if os.path.exists(expansion_dir): - ansible_run_destination = expansion_dir - inventory_dir = os.path.join(ansible_run_destination, 'inventories') - inventory_file = os.path.join(inventory_dir, self.inventory) - playbook_file = os.path.join(ansible_run_destination, self.playbook) - log_file = os.path.join(ansible_run_destination, 'run.log') - config_file = os.path.join( - ansible_run_destination, self.ansible_config - ) - os.system("chmod +x %s" % inventory_file) - cmd = "ANSIBLE_CONFIG=%s ansible-playbook -i %s %s" % (config_file, - inventory_file, - playbook_file) - with open(log_file, 'w') as logfile: - subprocess.Popen(cmd, shell=True, stdout=logfile, stderr=logfile) diff --git a/compass-tasks/deployment/utils/__init__.py b/compass-tasks/deployment/utils/__init__.py deleted file mode 100644 index cbd36e0..0000000 --- a/compass-tasks/deployment/utils/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" diff --git a/compass-tasks/deployment/utils/constants.py b/compass-tasks/deployment/utils/constants.py deleted file mode 100644 index e90b1b2..0000000 --- a/compass-tasks/deployment/utils/constants.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Grace Yu (grace.yu@huawei.com)" - - -"""All keywords variables in deployment are defined in this module.""" - - -# General keywords -BASEINFO = 'baseinfo' -CLUSTER = 'cluster' -HOST = 'host' -HOSTS = 'hosts' -ID = 'id' -NAME = 'name' -PASSWORD = 'password' -USERNAME = 'username' - - -# Adapter info related keywords -FLAVOR = 'flavor' -FLAVORS = 'flavors' -PLAYBOOK = 'playbook' -FLAVOR_NAME = 'flavor_name' -HEALTH_CHECK_CMD = 'health_check_cmd' -TMPL = 'template' -INSTALLER_SETTINGS = 'settings' -METADATA = 'metadata' -OS_INSTALLER = 'os_installer' -PK_INSTALLER = 'package_installer' -SUPPORT_OSES = 'supported_oses' - - -# Cluster info related keywords -ADAPTER_ID = 'adapter_id' -OS_VERSION = 'os_name' - - -# Host info related keywords -DNS = 'dns' -DOMAIN = 'domain' -HOST_ID = 'host_id' -HOSTNAME = 'hostname' -IP_ADDR = 'ip' -IPMI = 'ipmi' -IPMI_CREDS = 'ipmi_credentials' -MAC_ADDR = 'mac' -MGMT_NIC_FLAG = 'is_mgmt' -NETMASK = 'netmask' -NETWORKS = 'networks' -NIC = 'interface' -CLUSTER_ID = 'cluster_id' -ORIGIN_CLUSTER_ID = 'origin_cluster_id' -PROMISCUOUS_FLAG = 'is_promiscuous' -REINSTALL_OS_FLAG = 'reinstall_os' -SUBNET = 'subnet' - - -# Cluster/host config related keywords -COMPLETED_PK_CONFIG = 'completed_package_config' -COMPLETED_OS_CONFIG = 'completed_os_config' -DEPLOYED_OS_CONFIG = 'deployed_os_config' -DEPLOYED_PK_CONFIG = 'deployed_package_config' -NETWORK_MAPPING = 'network_mapping' -OS_CONFIG = 'os_config' -OS_CONFIG_GENERAL = 'general' -PK_CONFIG = 'package_config' -ROLES = 'roles' -PATCHED_ROLES = 'patched_roles' -ROLES_MAPPING = 'roles_mapping' -SERVER_CREDS = 'server_credentials' -TMPL_VARS_DICT = 'vars_dict' |